From 269d508ba568fbddf2c75984b148364f642f899e Mon Sep 17 00:00:00 2001 From: raver119 Date: Tue, 20 Aug 2019 18:52:41 +0300 Subject: [PATCH] [WIP] cross-device migrations (#134) * two more tests fixed Signed-off-by: raver119 * CUDA device afinity tweaks Signed-off-by: raver119 * minor tweaks Signed-off-by: raver119 * prepareAction/registerAction for CustomOps Signed-off-by: raver119 * lazy allocate host bufer before relocation Signed-off-by: raver119 * one special test for migration in cpp Signed-off-by: raver119 * tests update for msvc Signed-off-by: raver119 * logging Signed-off-by: raver119 * stick to old col2im impl Signed-off-by: raver119 * cudaStreams reorganization Signed-off-by: raver119 * buffer size fix Signed-off-by: raver119 * c++ data migration Signed-off-by: raver119 * fix CropAndResize test Signed-off-by: raver119 * - minor improvment Signed-off-by: Yurii --- libnd4j/blas/NDArray.h | 6 + libnd4j/blas/cuda/NDArray.cu | 14 +- libnd4j/blas/cuda/NativeOps.cu | 35 ++- libnd4j/include/array/DataBuffer.h | 2 + libnd4j/include/array/cpu/DataBuffer.cpp | 5 + libnd4j/include/array/cuda/DataBuffer.cu | 16 ++ libnd4j/include/execution/ContextBuffers.h | 13 +- libnd4j/include/execution/LaunchContext.h | 5 +- .../include/execution/cpu/ContextBuffers.cpp | 10 +- .../include/execution/cpu/LaunchContext.cpp | 4 + .../include/execution/cuda/AffinityManager.cu | 6 +- .../include/execution/cuda/ContextBuffers.cu | 40 ++++ .../include/execution/cuda/LaunchContext.cu | 43 +--- .../helpers/cuda/ConstantShapeHelper.cu | 7 +- .../include/helpers/cuda/ConstantTadHelper.cu | 5 +- .../ops/declarable/helpers/cuda/col2im.cu | 4 +- .../ops/declarable/helpers/cuda/dropout.cu | 5 +- .../layers_tests/ConvolutionTests1.cpp | 28 +-- .../layers_tests/ConvolutionTests2.cpp | 10 +- .../layers_tests/DeclarableOpsTests10.cpp | 52 ++--- .../layers_tests/DeclarableOpsTests8.cpp | 210 +++++++++--------- .../layers_tests/MultiDeviceTests.cpp | 68 ++++++ .../jita/allocator/impl/AtomicAllocator.java | 1 + .../jita/concurrency/CudaAffinityManager.java | 10 +- .../flow/impl/SynchronousFlowController.java | 9 +- .../jita/handler/impl/CudaZeroHandler.java | 30 ++- .../nd4j/linalg/jcublas/JCublasNDArray.java | 15 +- .../buffer/factory/CudaDataBufferFactory.java | 3 + .../ops/executioner/CudaExecutioner.java | 7 +- .../ops/executioner/CudaOpContext.java | 7 +- .../java/org/nd4j/nativeblas/Nd4jCuda.java | 69 ++++++ .../org/nd4j/nativeblas/Nd4jCudaPresets.java | 1 + .../java/org/nd4j/nativeblas/Nd4jCpu.java | 69 ++++++ .../org/nd4j/nativeblas/Nd4jCpuPresets.java | 1 + .../org/nd4j/linalg/crash/SpecialTests.java | 24 +- 35 files changed, 576 insertions(+), 258 deletions(-) create mode 100644 libnd4j/tests_cpu/layers_tests/MultiDeviceTests.cpp diff --git a/libnd4j/blas/NDArray.h b/libnd4j/blas/NDArray.h index edbf264d8..403f8d71d 100644 --- a/libnd4j/blas/NDArray.h +++ b/libnd4j/blas/NDArray.h @@ -39,6 +39,7 @@ #include #include #include +#include namespace nd4j { @@ -143,6 +144,11 @@ namespace nd4j { */ nd4j::DataType _dataType = FLOAT32; + /** + * deviceID where this NDArray belongs to + */ + int _deviceId = AffinityManager::currentDeviceId(); + template std::string toStringValue(T value); diff --git a/libnd4j/blas/cuda/NDArray.cu b/libnd4j/blas/cuda/NDArray.cu index 60498aaf7..38b6599bf 100644 --- a/libnd4j/blas/cuda/NDArray.cu +++ b/libnd4j/blas/cuda/NDArray.cu @@ -55,7 +55,19 @@ void* NDArray::getPlatformBuffer() const { return getSpecialBuffer(); } Nd4jLong* NDArray::getPlatformShapeInfo() const { return getSpecialShapeInfo(); } Nd4jLong* NDArray::platformShapeInfo() { return specialShapeInfo(); } -void NDArray::syncToDevice() const { _buffer->syncToSpecial(); } +void NDArray::syncToDevice() const { + auto currentDeviceId = AffinityManager::currentDeviceId(); + if (currentDeviceId != _deviceId) { + // first of all we update shapeInfo + const_cast(this)->setShapeInfo(this->getShapeInfo()); + + // now we actually migrate data buffer + _buffer->migrate(); + } + + _buffer->syncToSpecial(); +} + void NDArray::syncToHost() const { _buffer->syncToPrimary(getContext()); } void NDArray::tickWriteHost() const { _buffer->writePrimary(); } void NDArray::tickWriteDevice() const { _buffer->writeSpecial(); } diff --git a/libnd4j/blas/cuda/NativeOps.cu b/libnd4j/blas/cuda/NativeOps.cu index 11e2c0188..5c6dadbaf 100755 --- a/libnd4j/blas/cuda/NativeOps.cu +++ b/libnd4j/blas/cuda/NativeOps.cu @@ -31,6 +31,7 @@ #include #include +#include #include // FIXME: we need cuda-specific implementations #include @@ -965,11 +966,7 @@ int registerEvent(Nd4jPointer event, Nd4jPointer stream) { } int setDevice(int deviceId) { - auto dZ = cudaSetDevice(deviceId); - checkCudaErrors(dZ); - if (dZ != 0) - throw std::runtime_error("cudaSetDevice(...) failed"); - + AffinityManager::setCurrentDevice(deviceId); return 1; } @@ -1024,16 +1021,15 @@ Nd4jLong getDeviceTotalMemory(int device) { } int memcpySync(Nd4jPointer dst, Nd4jPointer src, Nd4jLong size, int flags, Nd4jPointer reserved) { - return memcpyAsync(dst, src, size, flags, reserved); } int memcpyAsync(Nd4jPointer dst, Nd4jPointer src, Nd4jLong size, int flags, Nd4jPointer reserved) { - cudaStream_t *pStream = reinterpret_cast(reserved); + auto pStream = reinterpret_cast(reserved); cudaMemcpyKind kind; - DEBUG_KERNEL(pStream, 0); + //nd4j::DebugHelper::checkErrorCode(pStream, "Preliminary sync failed"); switch (flags) { case 0: { @@ -1047,25 +1043,22 @@ int memcpyAsync(Nd4jPointer dst, Nd4jPointer src, Nd4jLong size, int flags, Nd4j case 2: { kind = cudaMemcpyDeviceToHost; } + break; case 3: { - kind = cudaMemcpyDeviceToDevice; - } + kind = cudaMemcpyDeviceToDevice; + } break; - default: { - - printf("UNDEFINED MEMCPY!\n"); - break; - } + default: + throw nd4j::cuda_exception::build("UNDEFINED MEMCPY!\n", 119); } - cudaError_t dZ = cudaMemcpyAsync(reinterpret_cast(dst), const_cast(reinterpret_cast(src)), static_cast(size), kind, *pStream); + auto dZ = cudaMemcpyAsync(reinterpret_cast(dst), const_cast(reinterpret_cast(src)), static_cast(size), kind, *pStream); + //auto dZ = cudaMemcpy(reinterpret_cast(dst), const_cast(reinterpret_cast(src)), static_cast(size), kind); if (dZ != 0) { - checkCudaErrors(dZ); - printf("Failed on [%lu] -> [%lu], size: [%i], direction: [%i], dZ: [%i]\n", src, dst, size, flags, static_cast(dZ)); + printf("Failed on [%lu] -> [%lu], size: [%i], direction: [%i], dZ: [%i]\n", src, dst, size, flags, static_cast(dZ)); fflush(stdout); fflush(stderr); - throw std::runtime_error("cudaMemcpyAsync(...) failed"); - //return 0L; + throw nd4j::cuda_exception::build("cudaMemcpyAsync(...) failed", dZ); } return 1; @@ -3256,7 +3249,7 @@ void tryPointer(Nd4jPointer extra, Nd4jPointer p, int len) { auto e = cudaStreamSynchronize(stream); if (e != 0) - throw std::runtime_error("tryPointer failed"); + throw nd4j::cuda_exception::build("tryPointer failed", e); cudaStreamDestroy(stream); } diff --git a/libnd4j/include/array/DataBuffer.h b/libnd4j/include/array/DataBuffer.h index 5bbc996f5..0b77289bd 100644 --- a/libnd4j/include/array/DataBuffer.h +++ b/libnd4j/include/array/DataBuffer.h @@ -105,6 +105,8 @@ class ND4J_EXPORT DataBuffer { bool isPrimaryActual() const; bool isSpecialActual() const; + void migrate(); + template FORCEINLINE T* primaryAsT(); template FORCEINLINE T* specialAsT(); diff --git a/libnd4j/include/array/cpu/DataBuffer.cpp b/libnd4j/include/array/cpu/DataBuffer.cpp index 2ec90f558..5d27bf9a1 100644 --- a/libnd4j/include/array/cpu/DataBuffer.cpp +++ b/libnd4j/include/array/cpu/DataBuffer.cpp @@ -96,6 +96,11 @@ void DataBuffer::allocateSpecial() { } +//////////////////////////////////////////////////////////////////////// +void DataBuffer::migrate() { + +} + //////////////////////////////////////////////////////////////////////// void DataBuffer::writePrimary() const { } void DataBuffer::writeSpecial() const { } diff --git a/libnd4j/include/array/cuda/DataBuffer.cu b/libnd4j/include/array/cuda/DataBuffer.cu index cf288d507..e71ed4b49 100644 --- a/libnd4j/include/array/cuda/DataBuffer.cu +++ b/libnd4j/include/array/cuda/DataBuffer.cu @@ -173,6 +173,22 @@ void DataBuffer::setToZeroBuffers(const bool both) { } } +//////////////////////////////////////////////////////////////////////// +void DataBuffer::migrate() { + memory::Workspace* newWorkspace = nullptr; + void* newBuffer; + ALLOCATE_SPECIAL(newBuffer, newWorkspace, getLenInBytes(), int8_t); + cudaMemcpy(newBuffer, _specialBuffer, getLenInBytes(), cudaMemcpyDeviceToDevice); + + if (_isOwnerSpecial) { + // now we're releasing original buffer + RELEASE_SPECIAL(_specialBuffer, _workspace); + } + + _isOwnerSpecial = true; + _specialBuffer = newBuffer; +} + //////////////////////////////////////////////////////////////////////// void DataBuffer::writePrimary() const { _writePrimary = ++_counter; } void DataBuffer::writeSpecial() const { _writeSpecial = ++_counter; } diff --git a/libnd4j/include/execution/ContextBuffers.h b/libnd4j/include/execution/ContextBuffers.h index 77b8d4ca3..39490d288 100644 --- a/libnd4j/include/execution/ContextBuffers.h +++ b/libnd4j/include/execution/ContextBuffers.h @@ -27,10 +27,12 @@ namespace nd4j { class ND4J_EXPORT ContextBuffers { private: - void* _reductionPointer; - void* _scalarPointer; - void* _allocationPointer; - bool _allocated = true; + void* _reductionPointer = nullptr; + void* _scalarPointer = nullptr; + void* _allocationPointer = nullptr; + void* _execStream = nullptr; + void* _specialStream = nullptr; + bool _allocated = false; int _deviceId = -1; @@ -44,6 +46,9 @@ namespace nd4j { void* scalarBuffer(); void* allocationBuffer(); + void* execStream(); + void* specialStream(); + void setReductionBuffer(void* pointer); void setScalarBuffer(void* pointer); void setAllocationBuffer(void* pointer); diff --git a/libnd4j/include/execution/LaunchContext.h b/libnd4j/include/execution/LaunchContext.h index 02b772415..f165e1297 100644 --- a/libnd4j/include/execution/LaunchContext.h +++ b/libnd4j/include/execution/LaunchContext.h @@ -52,8 +52,6 @@ class ND4J_EXPORT LaunchContext { #ifndef __JAVACPP_HACK__ - cudaStream_t* _cudaStream = nullptr; - cudaStream_t* _cudaSpecialStream = nullptr; void* _cublasHandle = nullptr; void* _cusolverHandle = nullptr; @@ -102,6 +100,9 @@ class ND4J_EXPORT LaunchContext { static LaunchContext* defaultContext(); + + static void swapContextBuffers(ContextBuffers &buffers); + }; } diff --git a/libnd4j/include/execution/cpu/ContextBuffers.cpp b/libnd4j/include/execution/cpu/ContextBuffers.cpp index d385548d0..19ceb1b36 100644 --- a/libnd4j/include/execution/cpu/ContextBuffers.cpp +++ b/libnd4j/include/execution/cpu/ContextBuffers.cpp @@ -71,4 +71,12 @@ namespace nd4j { int ContextBuffers::deviceId() { return _deviceId; } -} + + void* ContextBuffers::execStream() { + return _execStream; + } + + void* ContextBuffers::specialStream() { + return _specialStream; + } +} \ No newline at end of file diff --git a/libnd4j/include/execution/cpu/LaunchContext.cpp b/libnd4j/include/execution/cpu/LaunchContext.cpp index 9626b5d75..d16ea24d3 100644 --- a/libnd4j/include/execution/cpu/LaunchContext.cpp +++ b/libnd4j/include/execution/cpu/LaunchContext.cpp @@ -53,4 +53,8 @@ namespace nd4j { // return context for current device return LaunchContext::_contexts[0].get(); } + + void LaunchContext::swapContextBuffers(ContextBuffers &buffers) { + // + } } \ No newline at end of file diff --git a/libnd4j/include/execution/cuda/AffinityManager.cu b/libnd4j/include/execution/cuda/AffinityManager.cu index 811dc267a..522698c98 100644 --- a/libnd4j/include/execution/cuda/AffinityManager.cu +++ b/libnd4j/include/execution/cuda/AffinityManager.cu @@ -21,6 +21,7 @@ #include #include #include +#include thread_local int globalThreadToDevice = -1; @@ -98,10 +99,13 @@ namespace nd4j { if (res != 0) throw cuda_exception::build("cudaSetDevice failed", res); + auto previousDeviceId = globalThreadToDevice; + // update thread-device affinity globalThreadToDevice = deviceId; - // TODO: update context buffers? + ContextBuffers newBuffers; + LaunchContext::swapContextBuffers(newBuffers); } std::atomic AffinityManager::_lastDevice;// = std::atomic(initialV); diff --git a/libnd4j/include/execution/cuda/ContextBuffers.cu b/libnd4j/include/execution/cuda/ContextBuffers.cu index f82747b91..c8a149a5f 100644 --- a/libnd4j/include/execution/cuda/ContextBuffers.cu +++ b/libnd4j/include/execution/cuda/ContextBuffers.cu @@ -19,6 +19,7 @@ // #include +#include #include #include @@ -45,6 +46,18 @@ namespace nd4j { if (_allocationPointer != nullptr) cudaFree(_reductionPointer); + + auto _cudaStream = reinterpret_cast(_execStream); + auto _cudaSpecialStream = reinterpret_cast(_specialStream); + + cudaStreamSynchronize(*_cudaStream); + cudaStreamSynchronize(*_cudaSpecialStream); + + cudaStreamDestroy(*_cudaStream); + cudaStreamDestroy(*_cudaSpecialStream); + + delete _cudaStream; + delete _cudaSpecialStream; } } @@ -70,6 +83,19 @@ namespace nd4j { if (res != 0) throw std::runtime_error("_allocationPointer allocation failed"); + _execStream = new cudaStream_t(); + _specialStream = new cudaStream_t(); + if (nullptr == _execStream || nullptr == _specialStream) + throw std::runtime_error("Failed to allocate memory for new CUDA stream"); + + res = cudaStreamCreate(reinterpret_cast(_execStream)); + if (res != 0) + throw cuda_exception::build("Failed to create default CUDA stream with launch context", res); + + res = cudaStreamCreate(reinterpret_cast(_specialStream)); + if (res != 0) + throw cuda_exception::build("Failed to create special CUDA stream with launch context", res); + _allocated = true; } @@ -113,4 +139,18 @@ namespace nd4j { int ContextBuffers::deviceId() { return _deviceId; } + + void* ContextBuffers::execStream() { + if (_execStream == nullptr) + initialize(); + + return _execStream; + } + + void* ContextBuffers::specialStream() { + if (_specialStream == nullptr) + initialize(); + + return _specialStream; + } } diff --git a/libnd4j/include/execution/cuda/LaunchContext.cu b/libnd4j/include/execution/cuda/LaunchContext.cu index 004ed2cac..a881633a0 100644 --- a/libnd4j/include/execution/cuda/LaunchContext.cu +++ b/libnd4j/include/execution/cuda/LaunchContext.cu @@ -35,8 +35,8 @@ namespace nd4j { //////////////////////////////////////////////////////////////////////// LaunchContext::LaunchContext(cudaStream_t *cudaStream, cudaStream_t& specialCudaStream, void* reductionPointer, void* scalarPointer, int* allocationPointer) { - _cudaStream = cudaStream; - _cudaSpecialStream = &specialCudaStream; // ideal is = new cudaStream_t; *_cudaSpecialStream = specialCudaStream; + //_cudaStream = cudaStream; + //_cudaSpecialStream = &specialCudaStream; // ideal is = new cudaStream_t; *_cudaSpecialStream = specialCudaStream; //_reductionPointer = reductionPointer; //_scalarPointer = scalarPointer; //_allocationPointer = allocationPointer; @@ -46,14 +46,7 @@ LaunchContext::LaunchContext(cudaStream_t *cudaStream, cudaStream_t& specialCuda LaunchContext::~LaunchContext() { if (_isAllocated) { - cudaStreamSynchronize(*_cudaStream); - cudaStreamSynchronize(*_cudaSpecialStream); - cudaStreamDestroy(*_cudaStream); - cudaStreamDestroy(*_cudaSpecialStream); - - delete _cudaStream; - delete _cudaSpecialStream; } } @@ -64,32 +57,16 @@ LaunchContext::LaunchContext() { _deviceID = 0; _isAllocated = true; - _cudaStream = new cudaStream_t(); - _cudaSpecialStream = new cudaStream_t(); - if (nullptr == _cudaStream || nullptr == _cudaSpecialStream) - throw std::runtime_error("Failed to allocate memory for new CUDA stream"); - - cudaError_t err = cudaStreamCreate(_cudaStream); - if (err != 0) - throw cuda_exception::build("Failed to create default CUDA stream with launch context", err); - - err = cudaStreamCreate(_cudaSpecialStream); - if (err != 0) - throw cuda_exception::build("Failed to create special CUDA stream with launch context", err); _cublasHandle = CublasHelper::getInstance()->handle(); _cusolverHandle = CublasHelper::getInstance()->solver(); - - auto res = cudaStreamSynchronize(*_cudaStream); - if (res != 0) - throw cuda_exception::build("Initial sync failed", res); } LaunchContext::LaunchContext(Nd4jPointer cudaStream, Nd4jPointer reductionPointer, Nd4jPointer scalarPointer, Nd4jPointer allocationPointer) { _isAllocated = false; - _cudaStream = reinterpret_cast(cudaStream); - _cudaSpecialStream = reinterpret_cast(cudaStream); + //_cudaStream = reinterpret_cast(cudaStream); + // _cudaSpecialStream = reinterpret_cast(cudaStream); //_reductionPointer = reductionPointer; //_scalarPointer = scalarPointer; //_allocationPointer = reinterpret_cast(allocationPointer); @@ -148,11 +125,11 @@ LaunchContext::LaunchContext() { }; cudaStream_t* LaunchContext::getCudaStream() const { - return _cudaStream; + return reinterpret_cast(contextBuffers.execStream()); }; cudaStream_t* LaunchContext::getCudaSpecialStream() const { - return _cudaSpecialStream; + return reinterpret_cast(contextBuffers.specialStream());; }; @@ -169,14 +146,18 @@ LaunchContext::LaunchContext() { }; void LaunchContext::setCudaStream(cudaStream_t* cudaStream) { - _cudaStream = cudaStream; + //_cudaStream = cudaStream; }; void LaunchContext::setCudaSpecialStream(cudaStream_t* cudaStream) { - _cudaSpecialStream = cudaStream; + //_cudaSpecialStream = cudaStream; }; void LaunchContext::setCublasHandle(void *handle) { _cublasHandle = handle; }; + + void LaunchContext::swapContextBuffers(ContextBuffers &buffers) { + contextBuffers = buffers; + }; } \ No newline at end of file diff --git a/libnd4j/include/helpers/cuda/ConstantShapeHelper.cu b/libnd4j/include/helpers/cuda/ConstantShapeHelper.cu index bb7ce4f86..a1217e0e3 100644 --- a/libnd4j/include/helpers/cuda/ConstantShapeHelper.cu +++ b/libnd4j/include/helpers/cuda/ConstantShapeHelper.cu @@ -22,12 +22,13 @@ #include #include #include +#include #include namespace nd4j { ConstantShapeHelper::ConstantShapeHelper() { - auto numDevices = ConstantHelper::getNumberOfDevices(); + auto numDevices = AffinityManager::numberOfDevices(); _cache.resize(numDevices); for (int e = 0; e < numDevices; e++) { @@ -54,7 +55,7 @@ namespace nd4j { } ConstantDataBuffer& ConstantShapeHelper::bufferForShapeInfo(const ShapeDescriptor &descriptor) { - int deviceId = ConstantHelper::getCurrentDevice(); + int deviceId = AffinityManager::currentDeviceId(); _mutex.lock(); @@ -83,7 +84,7 @@ namespace nd4j { bool ConstantShapeHelper::checkBufferExistenceForShapeInfo(ShapeDescriptor &descriptor) { bool result; - auto deviceId = ConstantHelper::getCurrentDevice(); + auto deviceId = AffinityManager::currentDeviceId(); _mutex.lock(); if (_cache[deviceId].count(descriptor) == 0) diff --git a/libnd4j/include/helpers/cuda/ConstantTadHelper.cu b/libnd4j/include/helpers/cuda/ConstantTadHelper.cu index b9ff8887f..da66975c3 100644 --- a/libnd4j/include/helpers/cuda/ConstantTadHelper.cu +++ b/libnd4j/include/helpers/cuda/ConstantTadHelper.cu @@ -21,13 +21,14 @@ #include "../ConstantTadHelper.h" #include #include +#include #include #include #include namespace nd4j { ConstantTadHelper::ConstantTadHelper() { - auto numDevices = ConstantHelper::getNumberOfDevices(); + auto numDevices = AffinityManager::numberOfDevices(); for (int e = 0; e < numDevices; e++) { std::map pack; @@ -61,7 +62,7 @@ namespace nd4j { } TadPack& ConstantTadHelper::tadForDimensions(TadDescriptor &descriptor) { - const int deviceId = ConstantHelper::getCurrentDevice(); + const int deviceId = AffinityManager::currentDeviceId(); _mutex.lock(); diff --git a/libnd4j/include/ops/declarable/helpers/cuda/col2im.cu b/libnd4j/include/ops/declarable/helpers/cuda/col2im.cu index 8a21f03a5..a1725ff2d 100644 --- a/libnd4j/include/ops/declarable/helpers/cuda/col2im.cu +++ b/libnd4j/include/ops/declarable/helpers/cuda/col2im.cu @@ -184,8 +184,8 @@ static void col2imCudaLauncher(const int blocksPerGrid, const int threadsPerBloc void* image, const Nd4jLong* imShapeInfo, const int sH, const int sW, const int pH, const int pW, const int dH, const int dW) { - // col2imCuda2<<<512, 512, 1024, *stream>>>(columns, image, colShapeInfo, imShapeInfo, sH, sW, pH, pW, dH, dW); - col2imCuda<<>>(columns, colShapeInfo, image, imShapeInfo, sH, sW, pH, pW, dH, dW); + col2imCuda2<<<512, 512, 1024, *stream>>>(columns, image, colShapeInfo, imShapeInfo, sH, sW, pH, pW, dH, dW); + //col2imCuda<<>>(columns, colShapeInfo, image, imShapeInfo, sH, sW, pH, pW, dH, dW); } ////////////////////////////////////////////////////////////////////////// diff --git a/libnd4j/include/ops/declarable/helpers/cuda/dropout.cu b/libnd4j/include/ops/declarable/helpers/cuda/dropout.cu index e31349e31..40251fb82 100644 --- a/libnd4j/include/ops/declarable/helpers/cuda/dropout.cu +++ b/libnd4j/include/ops/declarable/helpers/cuda/dropout.cu @@ -135,8 +135,9 @@ namespace helpers { auto step = blockDim.x * gridDim.x; for (int e = tid; e < len; e += step) { - if (output[shape::getIndexOffset(e, outputShape, len)] != T(0.)) - output[shape::getIndexOffset(e, outputShape, len)] = T(input[shape::getIndexOffset(e, gradOutShape, len)] / probValue); + const auto zOffset = shape::getIndexOffset(e, outputShape, len); + if (output[zOffset] != T(0.)) + output[zOffset] = T(input[shape::getIndexOffset(e, gradOutShape, len)] / probValue); } } diff --git a/libnd4j/tests_cpu/layers_tests/ConvolutionTests1.cpp b/libnd4j/tests_cpu/layers_tests/ConvolutionTests1.cpp index 4af21ce20..987206a14 100644 --- a/libnd4j/tests_cpu/layers_tests/ConvolutionTests1.cpp +++ b/libnd4j/tests_cpu/layers_tests/ConvolutionTests1.cpp @@ -1050,14 +1050,14 @@ TYPED_TEST(TypedConvolutionTests1, conv2d_bp_test1) { auto bias = NDArrayFactory::create('c', {oC}, {1,2,3}); auto gradO = NDArrayFactory::create('c', {bS, oH, oW, oC}); - auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{ 0.226, 0.343, 0.46 , 0.577, 1.172, 1.46 , 1.748, 2.036, 1.892, 2.288, 2.684, 3.08 , 1.284, 1.581, 1.878, 2.175, 4.458, 5.133, 5.808, 6.483, 6.186, 7.023, 7.86 , 8.697, - 3.39 , 3.93 , 4.47 , 5.01 , 9.642, 10.803, 11.964, 13.125,11.37 , 12.693, 14.016, 15.339, 5.266, 5.707, 6.148, 6.589,12.98 , 13.916, 14.852, 15.788,14.564, 15.608, 16.652, 17.696, - 3.25 , 4.015, 4.78 , 5.545, 9.812, 11.396, 12.98 , 14.564,10.532, 12.224, 13.916, 15.608, 9.708, 10.977, 12.246, 13.515,25.194, 27.813, 30.432, 33.051,26.922, 29.703, 32.484, 35.265, - 11.814, 13.326, 14.838, 16.35 ,30.378, 33.483, 36.588, 39.693,32.106, 35.373, 38.64 , 41.907,13.474, 14.563, 15.652, 16.741,31.988, 34.22 , 36.452, 38.684,33.572, 35.912, 38.252, 40.592}); + auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{ 0.226f, 0.343f, 0.46f, 0.577f, 1.172f, 1.46f, 1.748f, 2.036f, 1.892f, 2.288f, 2.684f, 3.08f, 1.284f, 1.581f, 1.878f, 2.175f, 4.458f, 5.133f, 5.808f, 6.483f, 6.186f, 7.023f, 7.86f, 8.697f, + 3.39f, 3.93f, 4.47f, 5.01f, 9.642f, 10.803f, 11.964f, 13.125f,11.37f, 12.693f, 14.016f, 15.339f, 5.266f, 5.707f, 6.148f, 6.589f,12.98f, 13.916f, 14.852f, 15.788f,14.564f, 15.608f, 16.652f, 17.696f, + 3.25f, 4.015f, 4.78f, 5.545f, 9.812f, 11.396f, 12.98f, 14.564f,10.532f, 12.224f, 13.916f, 15.608f, 9.708f, 10.977f, 12.246f, 13.515f,25.194f, 27.813f, 30.432f, 33.051f,26.922f, 29.703f, 32.484f, 35.265f, + 11.814f, 13.326f, 14.838f, 16.35f,30.378f, 33.483f, 36.588f, 39.693f,32.106f, 35.373f, 38.64f, 41.907f,13.474f, 14.563f, 15.652f, 16.741f,31.988f, 34.22f, 36.452f, 38.684f,33.572f, 35.912f, 38.252f, 40.592f}); - auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, oC},{14.4 , 14.76, 15.12,14.4 , 14.76, 15.12,14.4 , 14.76, 15.12,14.4 , 14.76, 15.12, 9.24, 9.48, 9.72, 9.24, 9.48, 9.72, 9.24, 9.48, 9.72, 9.24, 9.48, 9.72, - 17.04, 17.52, 18. ,17.04, 17.52, 18. ,17.04, 17.52, 18. ,17.04, 17.52, 18. ,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52, - 11.16, 11.52, 11.88,11.16, 11.52, 11.88,11.16, 11.52, 11.88,11.16, 11.52, 11.88, 7.08, 7.32, 7.56, 7.08, 7.32, 7.56, 7.08, 7.32, 7.56, 7.08, 7.32, 7.56}); + auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, oC},{14.4f, 14.76f, 15.12f,14.4f, 14.76f, 15.12f,14.4f, 14.76f, 15.12f,14.4f, 14.76f, 15.12f, 9.24f, 9.48f, 9.72f, 9.24f, 9.48f, 9.72f, 9.24f, 9.48f, 9.72f, 9.24f, 9.48f, 9.72f, + 17.04f, 17.52f, 18.f,17.04f, 17.52f, 18.f, 17.04f, 17.52f, 18.f, 17.04f, 17.52f, 18.f,10.88f, 11.2f, 11.52f,10.88f, 11.2f, 11.52f,10.88f, 11.2f, 11.52f,10.88f, 11.2f, 11.52f, + 11.16f, 11.52f, 11.88f,11.16f, 11.52f, 11.88f,11.16f, 11.52f, 11.88f,11.16f, 11.52f, 11.88f, 7.08f, 7.32f, 7.56f, 7.08f, 7.32f, 7.56f, 7.08f, 7.32f, 7.56f, 7.08f, 7.32f, 7.56f}); // auto expGradB('c', {oC},{}); input = 2.; @@ -1093,14 +1093,14 @@ TYPED_TEST(TypedConvolutionTests1, conv2d_bp_test2) { auto bias = NDArrayFactory::create('c', {oC}, {1,2,3}); auto gradO = NDArrayFactory::create('c', {bS, oH, oW, oC}); - auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{ 0.014,0.032, 0.05 , 0.068,0.118,0.181, 0.244, 0.307,0.212,0.257, 0.302, 0.347,0.208,0.298, 0.388, 0.478,1.028,1.262, 1.496, 1.73 ,1.036,1.18 , 1.324, 1.468, - 0.928,1.018, 1.108, 1.198,2.9 ,3.134, 3.368, 3.602,2.188,2.332, 2.476, 2.62 ,1.202,1.274, 1.346, 1.418,3.142,3.313, 3.484, 3.655,2.048,2.147, 2.246, 2.345, - 0.086,0.212, 0.338, 0.464,0.694,0.973, 1.252, 1.531,0.716,0.869, 1.022, 1.175,1.216,1.522, 1.828, 2.134,3.908,4.574, 5.24 , 5.906,2.908,3.268, 3.628, 3.988, - 3.664,3.97 , 4.276, 4.582,9.236,9.902,10.568,11.234,5.788,6.148, 6.508, 6.868,3.002,3.182, 3.362, 3.542,7.174,7.561, 7.948, 8.335,4.28 ,4.487, 4.694, 4.901}); + auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{ 0.014f, 0.032f, 0.05f, 0.068f,0.118f,0.181f, 0.244f, 0.307f,0.212f,0.257f, 0.302f, 0.347f,0.208f,0.298f, 0.388f, 0.478f,1.028f,1.262f, 1.496f, 1.73f,1.036f,1.18f, 1.324f, 1.468f, + 0.928f,1.018f, 1.108f, 1.198f,2.9f,3.134f, 3.368f, 3.602f,2.188f,2.332f, 2.476f, 2.62f, 1.202f,1.274f, 1.346f, 1.418f,3.142f,3.313f, 3.484f, 3.655f,2.048f,2.147f, 2.246f, 2.345f, + 0.086f,0.212f, 0.338f, 0.464f,0.694f,0.973f, 1.252f, 1.531f,0.716f,0.869f, 1.022f, 1.175f,1.216f,1.522f, 1.828f, 2.134f,3.908f,4.574f, 5.24f, 5.906f,2.908f,3.268f, 3.628f, 3.988f, + 3.664f,3.97f, 4.276f, 4.582f,9.236f,9.902f,10.568f,11.234f,5.788f,6.148f, 6.508f, 6.868f,3.002f,3.182f, 3.362f, 3.542f,7.174f,7.561f, 7.948f, 8.335f,4.28f,4.487f, 4.694f, 4.901f}); - auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, oC},{1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16, - 1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16, - 1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16}); + auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, oC},{1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f, + 1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f, + 1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f,1.84f, 2.f, 2.16f}); // auto expGradB('c', {oC},{}); input = 2.; diff --git a/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp b/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp index dc29b1b14..c1b34d779 100644 --- a/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp +++ b/libnd4j/tests_cpu/layers_tests/ConvolutionTests2.cpp @@ -166,8 +166,8 @@ TYPED_TEST(TypedConvolutionTests2, Test_DeConv2D_TF_2) { -1.09233856f, 1.12430644f, 0.52273953f, -0.68507338f, -0.69913578f, 0.88440478f, -0.76959240f, 1.07093310f, -0.34802195f, 0.35683727f, -0.76079178f, -1.92807376f, 0.84499562f, 1.39131641f, 0.44825050f, 0.34567752f, 0.44607711f, -1.00986362f, -0.50038189f, -0.09060892f, -2.55645394f, 0.56416476f, -0.83058155f, -0.65931624f, -0.73649710f, 0.59814465f, -0.86736494f, -0.32200798f, -1.28087902f, -0.76818323f, 0.86848933f, -0.98678392f, -1.30813944f, -0.20255326f, 0.26557815f, -0.31090519f, -1.46331608f, -0.62782109f, 0.59034890f, 1.63147473f, -0.17727259f, -0.37636510f, 1.27368402f, 0.19096918f, -0.29936951f, -1.99038267f, 0.54831523f, 0.48849005f, -2.55680346f, -0.63126534f, 1.21715927f, 1.22841084f, -0.67416084f, 0.02927168f, -0.36693662f, 0.63204330f, 0.13721083f, 0.28742912f, 0.19470036f, 0.74873924f, -1.47602463f, 0.86264688f, -0.23730527f, -0.99978864f, -1.17048764f, -0.34996086f, 1.43019187f, 0.26224539f, 0.60689932f, -0.75002515f, -0.79823422f, -1.37300086f, -0.19951135f, -0.12150808f, -0.75272322f, 0.23755015f, 0.31270382f, 1.66539109f, -1.04104745f, 0.79540199f, -0.54042423f, -0.54150617f, 0.43871084f, 0.24163951f, -0.24517761f, -0.66178995f, -1.13064528f, -0.84426326f, 0.56437236f, 0.09088907f, -0.82823074f, 0.81753862f, -1.74096012f, -1.80599844f, -0.60943592f, 1.36094582f, -1.47762752f, 0.15931177f, 1.05569172f, 0.36751524f, 0.06497604f, 0.13536447f, -1.57156146f, 0.22783801f, -0.96910107f, -1.24294984f, -1.47147155f, -1.04790676f, 0.64629447f, -0.32266054f, -0.55675793f, -0.95612079f, -0.23005411f, -0.75229394f, 0.03050950f, -1.72484553f, -2.06055546f, 0.19892083f, -0.13597751f, 0.65180075f, 0.27096850f, 0.08977254f, 0.57564765f, -0.43227410f, 0.09541437f, -0.00358280f, 0.65680492f, 0.04006556f, 0.57160908f, 0.43821687f, 1.96118212f, 0.42602235f, -0.36731303f, 0.67200917f, -0.56667900f, 0.44014785f, 0.06970236f, -1.34415269f, -1.13301528f, -0.08848868f, 0.35615012f, -0.06426942f, -0.81406075f, 0.94097465f, -0.54560357f, -0.65877116f, -1.29646838f, -1.13109028f, -1.64186084f, -2.12723470f, 1.86027610f, 1.22621441f, 0.26098135f, -0.05608099f, 0.21143445f, -0.87244326f, 0.79408187f, 1.24279130f, 0.14458629f, 0.25532281f, -1.24023473f, 2.42278886f, 0.00405578f, -1.00119174f, 1.19856644f, -1.37395728f, -0.16656208f, 0.46858498f, -0.00678801f, -0.34960639f, 0.16614936f, 2.41560221f, -0.53880709f, 0.91618651f, -1.77009308f, 0.32911557f, 0.30216452f, 0.02881077f, 0.77705866f, 0.27061903f, -0.07440855f, -1.14010465f, 1.25383139f, -1.58615100f, 1.04185510f, 0.15140508f, -0.88059032f, -0.33872122f, -0.42526904f, 2.17365575f, 0.29308075f, -2.24234557f, -1.03164542f, -0.09263755f, 0.08050421f, -0.74946511f, -0.64589006f, -1.13416314f, -0.64989561f, 0.16502371f, -0.33831969f, 0.22832428f, -0.08389475f, -0.28009200f, 1.34536922f, -0.19075738f, 0.36238208f, 0.83690089f, 0.26144615f, 0.04457319f, -2.55585861f, -0.01807522f, 1.68334866f, -0.05795629f, -0.21315987f, -1.84039557f, 0.06512877f, -1.77318645f, -0.27637982f, 0.20439345f, 0.67558700f, -0.77179354f, -0.17902173f, 0.70381826f, -0.40395790f, -0.96492916f, 0.84138173f, 2.43879008f, -0.32297835f, -1.74370265f, -0.10330839f, -1.07465363f, 1.85030377f, -0.59153467f, 0.99667048f, -0.56753993f, 0.57383025f, -1.90630126f, 1.24299097f, 0.22797665f, 0.30468231f, -0.07360230f, 1.64654350f, 0.57195550f, 0.03227921f, 1.11005175f, 0.00088721f, 1.19266295f, 0.61323351f, 0.13754399f, 0.59900171f, -0.75831634f, 1.11500823f, 0.99747783f, -1.36923385f, 1.26563418f, 0.01253266f, 0.35483193f, 1.95143735f, -2.02703261f, -1.38265920f, -0.02404256f, 2.02788448f, -0.75144875f, -0.58445263f, 0.26129767f, 0.60691077f, -1.84661067f, 0.65872228f, -0.58298993f, 0.33067298f, -0.09431327f, 0.43333948f, -1.52616286f, -0.25961858f, -1.65459549f, -0.72950101f, -0.89906919f, -0.80081612f, -1.32189929f, -1.36574399f, -0.35809481f, 0.36385000f, 0.31480747f, -0.35797358f, -1.04066050f, 0.07971872f, -0.21176252f, -0.76559299f, -0.10352154f, 0.29248312f, -1.75030553f, 0.68219930f, 0.56189102f, -1.11212170f, 0.06501702f, -0.07131009f, 1.23410738f, 0.29311740f, -1.02052307f, 1.40220940f, -1.00995779f, 0.57955760f, 0.22640309f, 0.74853230f, -0.02586563f, -0.33427954f, 1.70311153f, -0.53405988f, 0.90975094f, -0.46450076f, 0.19904344f, 0.28559047f, 0.23167793f, -0.69065529f, -0.17176504f, -0.29301846f, -0.85477978f, -0.00267053f, -0.28529504f, -0.64201307f, 1.03479636f, 1.03805065f, 0.83270210f, -0.09405448f, 2.50615931f, 0.62019676f, 0.31354564f, -1.51599669f, 0.42848015f, 0.66263914f, 0.74651009f, -1.13042867f, -0.58933645f, -0.35146511f, 0.06223279f, 0.28065836f, 0.66506970f, 0.16942430f, -0.23316263f, -0.87481076f, 1.21992230f, 1.48536301f, -0.79667616f, -0.75519305f, 1.40999961f, -0.42802793f, -0.20252463f, 0.30573779f, -0.23319976f, 1.77525878f, -1.80704832f, 2.71519923f, -0.67500192f, 0.12268137f, -0.13014549f, -0.07479453f, -1.51065743f, 1.04198146f, 0.96205556f, -2.00525570f, -0.37911776f, 0.89329720f, -0.39495832f, -0.03683375f, -0.90928614f, -1.56263304f, 0.45038295f, -2.62184358f, -0.45686841f, -0.52536523f, 1.05351484f, 0.89982438f, -0.63724512f, 3.21004057f, -0.08608918f, 1.55209303f, 0.62688643f, -0.59702635f, 1.85774517f, 0.38172096f, -1.25640929f, -2.59278178f, 0.85050315f, -1.10080361f, -1.26422560f, -1.80045366f, -0.34494889f, 0.68448657f, 1.25671864f, -1.26594126f, 0.32244179f, -0.51956522f, -0.56212711f, -0.95574015f, 0.71973872f, 0.46736258f, -0.11772985f, -1.52736545f, 0.19571695f, 0.73147154f, 0.87724912f, -0.26265728f, -2.60267401f, 0.19263546f, 0.18320183f, 0.11485019f, -0.82999659f, 0.13582672f, -0.08040185f, 0.28152901f, -0.51421624f, -2.32467175f, 0.19923948f, 0.64616692f, 0.29718629f, 0.32785949f, -0.62266952f, -0.98174316f, 1.23276305f, 0.58563638f, 1.28528512f, -2.13718534f, 0.28842899f, 0.12676710f, -1.72105229f, 0.15053287f, 2.19496536f, 1.28683448f, -0.96318281f, 0.17043279f, -0.05245409f, -0.38710704f, -0.30441490f, -0.08249986f, 0.28423953f, 0.72963721f, -1.49658203f, 0.99077344f, -0.78913772f, -1.12661564f, -1.26294816f, 0.16517465f, 0.10124251f, -0.77198768f, -0.16342169f, 0.08615876f, 0.49711797f, -0.66083062f, 0.76648003f, 1.04756033f, 1.46122825f, -0.42798752f, -2.29203916f, 0.30444992f, 0.58697921f, 1.22166932f, 0.09022947f, -0.03920181f, 0.10444995f, 0.10361757f, 1.18224072f, -0.76641631f, 0.90802073f, 1.41639423f, 1.55682337f, 1.28101575f, -0.35396016f, 1.11443567f, 1.18218529f, -0.06048089f, 0.85024464f, -1.01789165f, -0.69154263f, 0.06663221f, 0.68429029f, 0.12560424f, 0.37915874f, -0.66829866f, -0.64524972f, -0.05568011f, 0.12230454f, -0.35041061f, 0.62027830f, -0.16739209f, -0.72145337f, 0.46263054f, -1.67837834f, 0.69413221f, -0.57243419f, 0.37638462f, -0.21446526f, -0.89821470f, 0.60078722f, -1.06706369f, -1.26132309f, 0.35714921f, 2.39221811f, -0.09376130f, 0.30760849f, 0.59180892f, 0.55815399f, -0.32628775f, 1.28890121f, -2.53237987f, -0.98241091f, 1.10520673f, -1.74751687f, -0.90837651f, -0.25220659f, -0.56625104f, -0.30691949f, 0.16058689f, 0.44309673f, -1.09874964f, -0.76747823f, -0.33679363f, -0.02535496f, 0.00990100f, 1.35318136f, -0.70140815f, 0.50937581f, 0.55386209f, -1.21721983f, 0.71376961f, -0.18079315f, -0.11077732f, 0.09292522f, -0.57235324f, 0.62748206f, 0.42587611f, 0.64860481f, -1.10635614f, 1.66414368f, 0.47505483f, 1.48602211f, -0.59611166f, -0.41932896f, -0.96542233f, -0.41756630f, -1.02963889f, -0.70070386f, 1.65803933f, 0.20138647f, 0.05895034f, -1.46152759f, -0.37278318f, 1.05535650f, 0.34437978f, -1.13257408f, 0.17635690f, 0.09386671f, 0.37079874f, 1.47695887f, -1.58420062f, -0.26100200f, 0.44847637f, 0.88847303f, -0.13877590f, -0.64620668f, -0.38019657f, 1.01608157f, 0.13357787f, 0.05137976f, 0.93498152f, -0.62226880f, 0.80461699f, -0.71682596f, -0.88756353f, 0.40933055f, -1.52167451f, 0.79756850f, -0.17307425f, 0.62368619f, -0.22466940f, -1.72802913f, 0.59047443f, -0.58020931f, 0.09096476f, -0.07317388f, 0.44522321f, -0.64880705f, 0.15684015f, 0.08708375f, -0.41556796f, 1.11579072f, -0.81733495f, 0.11643656f, -0.73995101f, 0.93685871f, 1.57971406f, 0.67606360f, 0.70509088f, -0.25283816f, -0.00010609f, -0.61884147f, -0.86409342f, 0.95383751f, -0.05895388f, -1.45261180f, 0.45166013f, -1.01434863f, 0.18496066f, 1.06517637f, 1.81127059f, 0.89470667f, -0.13232610f, 0.46958798f, 0.13884509f, 0.57117194f, 0.29575035f, -0.97884250f, 0.83291447f, -0.59255791f, -0.04354135f, -0.19431923f, 0.30071029f, -0.95421529f, 0.76359886f, -0.47799742f, 0.68254346f, 1.19368529f, -0.48935115f, 0.30357337f, -0.50225669f, -0.23370270f, 1.96702433f, 1.46558523f, 2.68482018f, 0.41622332f, 0.73697484f, 1.43430734f, 0.15387188f, 0.20875402f, -2.49335337f, -1.39674246f, -0.22125854f, -0.00424605f, 0.91416460f, 0.33384630f, 0.44703746f, 0.25610185f, 0.38966551f, -0.01784045f, 1.66148460f, 0.36005461f, 0.95716912f, -0.18246566f, -0.15480693f, 0.38775176f, -0.56969136f, -0.29644895f, -1.04565966f, -1.00455630f, 0.30897698f, -1.46885884f, 0.03657720f, -0.49302089f, 1.34134722f, 0.01673754f, 1.22725964f, 0.55256772f, 0.63803208f, -0.29041430f, 1.11455286f, 0.76329172f, 0.27073982f, 0.77173829f, -1.79884446f, -0.11889492f, -1.92040312f, -0.46382675f, 0.20078070f, -0.98889589f, 1.46711135f, -1.68280172f, -0.52852470f, 0.66245162f, 0.29575166f, 1.34826505f, -0.22362417f, -0.14345661f, -2.34815073f, 1.26572001f, 0.66505629f, 1.01141500f, 1.08030057f, 0.17036134f, 0.00168786f, -0.37282917f, 0.69206375f, 1.07367527f, -0.49708191f, 1.49504781f, 0.58224988f, 0.96593714f, -1.07661915f, 0.25202179f, 0.25531644f, 0.42357162f, -0.31236249f, 0.48383278f, -0.06361829f, 0.24131298f, -0.95695931f, -0.12589653f, 0.36134180f, 3.20266032f, -0.40879184f, -0.66985190f, 1.51674330f, 0.34072638f, 1.15076303f, -0.40199137f, 0.46223637f, -0.48608047f, 0.99119538f, -0.22506073f, 0.30968750f, 0.64210880f, 0.54640514f, 0.18607031f, 1.26293361f, -0.77960914f, 0.79572529f, 1.01936150f, 2.27160740f, -1.48034489f, 0.74466604f, 0.14863680f, 0.31102443f, -1.15673816f, -0.38609681f, -2.65026069f, -0.45524642f, -0.74022961f, 2.74991131f, 0.00103815f, -3.03303242f, -0.41556966f, -0.87103498f, 0.78306234f, -0.88195556f, -0.77297026f, 1.21203196f, -1.09754920f, -0.03556008f, -0.31546223f, 0.72954375f, 0.25251788f, 0.11378583f, 0.50921023f, 0.30301905f, -1.60631680f, 0.27152416f, 1.17342317f, -0.70891970f, -0.08392961f, 0.92137378f, -0.10568139f, -0.31653777f, -0.28878728f, 1.22166574f, 1.12693942f, -0.21325994f, 0.94010323f, 1.21796405f, -0.68866694f, 2.30724216f, 0.28141466f, 0.83481526f, -0.04885862f, 0.01675143f, 1.04355800f, -0.81050140f, 1.51300573f, 0.53429186f, -0.56439877f, 0.38572624f, -0.05620475f, 0.67644542f, 0.72528905f, 0.05937041f, -1.06315899f, -0.51393986f, 0.46937627f, -0.34699562f, -0.64765716f, -1.45512629f, 0.47739139f, -0.88228017f, -2.00791359f, 1.29929042f, 0.05482405f, -0.66725296f, -0.54735124f, 0.09972951f, 0.76675093f, 0.98748523f, 0.08900899f, -0.78854066f, 1.47970486f, -0.61667502f, 0.45625573f, -0.21766303f, -0.46250847f, -0.07130960f, 0.64414692f, 0.12784545f, 0.26393634f, 1.07720757f, -1.23938286f, 0.62483376f, -0.55001754f, -0.05358591f, 0.07322436f, 1.12003291f, -1.00830650f, -0.20486419f, 0.76664752f, 0.28850746f, -0.04464776f, -0.40146068f, 0.73262817f, -1.12827921f, -0.19989438f, -1.15999687f, 1.37973154f, 0.78881019f, -0.34762639f, 1.22088552f, -1.64088547f, 0.63218033f, 0.45736769f, 0.05502866f, 2.22683382f, -1.78935897f, -1.49635041f, 0.83450896f, 1.67770112f, 1.33909333f, 1.51158953f, 0.28595078f, -0.08593627f, 0.45812801f, -0.15193029f, 1.14770603f, -0.88920450f, -1.96352005f, -1.49894583f, 0.49629962f, 1.59872091f, 0.00903497f, 2.15563583f, 2.25149560f, -2.01200557f, 2.56229877f, -1.38850498f, 0.73552012f, -0.39378855f, 0.52616280f, -0.03685786f, 0.87403935f, 0.12163408f, 0.74297994f, -0.30697080f, 0.38139752f, 0.49113834f, -0.95485127f, -0.99908817f, 0.71716321f, 0.04000283f, -2.09645271f, 1.38789880f, 1.37198520f, 0.82493287f, 0.17114936f, 0.53696346f, -0.19516060f, -0.50377476f, -0.91730285f, -0.70113552f, -0.02406530f, 0.84943396f, -0.17428185f, -1.09140801f, -0.68156958f, 1.70756388f, -1.00399911f, 0.03023832f, -0.39023280f, -1.89737976f, 1.14469039f, -0.58337289f, -0.60037899f, -1.17490256f, -1.56342828f, 0.48714057f, 0.62266618f, -0.15967095f, 1.32789338f, -1.25700688f, -0.55633998f, -0.83128709f, -0.49346271f, 1.59561753f, -0.24675299f, 0.38012561f, 0.91796309f, -0.38522810f, -0.65509188f, 0.94100451f, -0.57324487f, 2.19070768f, 1.24058700f, -0.75978851f, -0.40460554f, 0.79189235f, 0.70192885f, 1.93569362f, -0.03070199f, 0.77010989f, 0.58794290f, 0.51087004f, 0.22892070f, 0.35007235f, 1.56023848f, -0.67453802f, -0.18485607f, 0.64349502f, -0.31489357f, -1.95834625f, 0.06560058f, 2.30394220f, 1.18194163f, -0.88034087f, -1.05000436f, -1.05471325f, -0.98481798f, 0.49904808f, 0.16438948f, -1.10297823f, -1.39736509f, 0.01306054f, -1.85160267f, -0.87292641f, -0.15418227f, 0.43412164f, 1.16518164f, 0.06273691f, 0.24659210f, -0.08267246f, 1.28885782f, 0.73575675f, -0.01019809f, -0.08753663f, -0.61827368f, -0.40863234f, 2.12599611f, -0.53620332f, 0.53789747f, -0.66386080f, -1.70461988f, 0.86608189f, -1.11151052f, 0.14120635f, 1.18858743f, -0.31760478f, -0.73533046f, 0.20978074f, -0.84074509f, 0.16523147f, -1.03362834f, 0.59721231f, 0.21318658f, 0.23671274f, 1.75115061f, 0.25363782f, -1.32541454f, 1.13056135f, 0.24652456f, 0.60381413f, 0.21478581f, 0.75044096f, -0.63125616f, -1.69889998f, -0.02116571f, 1.46165359f, 1.03068244f, 0.63693464f, 0.67795700f, 1.20033514f, -1.39205134f, -0.61743122f, 0.56549704f, 0.65182322f, -0.74250507f, -1.61939359f, 1.14054918f, -0.45725963f, 1.74519682f, -0.66251940f, -0.94811529f, -1.60865819f, -0.59968346f, 0.86309159f, -1.91936195f, -1.02646923f, -1.50352538f, 0.58292735f, 0.05320299f, 1.53582895f, 0.01069612f, 0.15226212f, -0.71840125f, -1.36896348f, 2.14600968f, 0.96626586f, -0.52014917f, 0.41001406f, 0.59478027f, 0.15282436f, 0.27790198f, 0.76614654f, -0.38971323f, -0.01839927f, -1.57882118f, 0.61391610f, -0.62133092f, -0.03968323f, -0.88467252f, -1.24041140f, 2.07306671f, -0.41776338f, 0.14537935f, -0.91069067f, 1.67362070f, 4.72630215f, -0.07395106f, 0.46280116f, -0.40843824f, 0.70683080f, -0.27510864f, -0.63465804f, -0.83630908f, -0.44419941f, 0.60405648f, -0.65039170f, -1.02413189f, 1.05983019f, 1.73366308f, 0.73343736f, -0.00895882f, -1.00826013f, 0.17323074f, 0.73995626f, 0.24128854f, 0.94510227f, 0.25557515f, 0.02244723f, -0.95197725f, -0.16297856f, -0.38497585f, 1.17993331f, 1.20282137f, -1.31491220f, 0.44229278f, -0.24349044f, -0.01230415f, 1.37944865f, 0.48554277f, -0.54510897f, -0.10793537f, 0.41121426f, -0.12889031f, 0.26434359f, 1.27966082f, 0.64518744f, -0.15577169f, -0.99864733f, -0.61746484f, 2.01614976f, 1.56254935f, 1.86473298f, -0.54662132f, -0.22047071f, -0.06118120f, 0.84799510f, 0.17009684f, -1.30523121f, 0.64000309f, 0.36299205f, -0.59620583f, 1.36372304f, -0.05389515f, -0.93849313f, 0.98043185f, -0.39373067f, -0.84898937f, 1.32077873f, 1.05988657f, -1.35339200f, 0.23259017f, 0.63816410f, -0.80297333f, 0.60017115f, 1.25715804f, 1.18894124f, -0.62473553f, 1.05611980f, 0.02335166f, 1.07509828f, 0.25873449f, -1.68341100f, 0.54547334f, 0.79288185f, -0.93678916f, 0.19202201f, -1.48575914f, 1.08649087f, 0.50851744f, -0.45758674f, -0.39734635f, 0.35637981f, -1.63079453f, -0.75910008f, 0.92640859f, -0.55599529f, -0.40276715f, 0.31307653f, 0.39907026f, -1.18830419f, 0.71051043f, 0.14157933f, -0.39581308f, -1.64361024f, -0.06161860f, -0.25312796f, 1.10018682f, 0.56500763f, 0.80385065f, 0.35395023f, 0.81813669f, 0.27644628f, 0.65563256f, 1.73197234f, 0.68178749f, 0.76769936f, 0.44597456f, 0.67761195f, 0.67635447f, -0.32315412f, 0.19330767f, -0.25557944f, 1.91693723f, 0.38335562f, 0.07107610f, -0.57384586f, 0.79184365f, 1.87835479f, 0.60902315f, -0.94220877f, 0.79479855f, -0.25656971f, 0.08739131f, 0.53384244f, 1.22159266f, -0.39152125f, -1.46373534f, -0.02458516f, 1.62825716f, -1.26112676f, 0.19967082f, -0.71114451f, 0.27929229f, 0.65001321f, -0.11868202f, -0.55587751f, 0.78069001f, 0.57969242f, -0.60274386f, 0.31650013f, 0.90339553f, 0.09453616f, -0.37119162f, -1.00320566f, 0.33299938f, -0.48636708f, 0.26342997f, -0.91914523f, 0.28682709f, -1.24780893f, -1.59254742f, 0.97176319f, 0.14744301f, -0.53056234f, -1.73221612f, -0.67645556f, 0.98705006f, 0.79895812f, -2.04333115f, -0.60132772f, -0.91653955f, -0.28094748f, 0.47943443f, 0.38157779f, -0.67648011f, 1.09093642f, 1.66012859f, -0.29358891f, -1.26773024f, 0.36747769f, -1.10141146f, 0.82383633f, -0.89772314f, -0.47145563f, 0.63939518f, -0.64430422f, -0.48889321f, -0.37680882f, -1.06962025f, -1.28689516f, 1.28365147f, 0.61859220f, -0.84676331f, 1.38404000f, 1.21053445f, -0.14871351f, 1.06349385f, 1.45878971f, -0.47362664f, 1.40707004f, 1.25224137f, 0.87364739f, 0.92858213f, 0.00157326f, 1.45661485f, -0.27318576f, 0.15482858f, -1.07058907f, -0.06903186f, -0.74147576f, -1.64111829f, -0.67226541f, -1.13458407f, 1.28511488f, -0.41041154f, 2.09085560f, 0.45243183f, -0.67437285f, 0.84960121f, -1.49300814f, -0.42961186f, -2.35021853f, 0.57255560f, -0.73903763f, 1.37607956f, -2.44575167f, 1.25105727f, 1.38575912f, -1.16299784f, -0.13719854f, -1.11507034f, 0.35796806f, -0.64511567f, -0.87903833f, 0.32833642f, -0.87696886f, 0.02714214f, 0.30224666f, -0.69118696f, -1.23500824f, 0.76678628f, -3.20508122f, -0.24704689f, 0.49019828f, -1.20862615f, -0.03778638f, -0.07273687f, -0.11517122f, -1.75857520f, -1.64188445f, 1.21574795f, 0.57325113f, 1.14370298f, -1.07824504f, 1.70653832f, -0.03700557f, -0.47645858f, 0.11065386f, -1.03143036f, -2.18094873f, -0.94403434f, -0.09335683f, -0.44817665f, 1.39707148f, -1.21947956f, 0.56575936f, -0.69612634f, -1.12361753f, -0.17105591f, 1.15422392f, 0.02840637f, 0.09469353f, -0.52859986f, -2.08487725f, 1.28789508f, -0.03740775f, 0.61196613f, 1.23405397f, 1.56595814f, -0.65800631f, 2.02985072f, -0.69446486f, -0.88443804f, -0.23448054f, -0.43628734f, -0.45888957f, -0.21943338f, 1.78258693f, 1.75214970f, 0.71804136f, 0.49782532f, 0.37886053f, -1.59176385f, -1.74758542f, -0.02820176f, 0.75398153f, 1.00119829f, 0.80881971f, -0.53365272f, -0.22720885f, 0.37476870f, 0.01005529f, -1.23421800f, -0.13431595f, -1.01843679f, 1.87386346f, -1.68539488f, -1.04942071f, -0.77322137f, 0.53964764f, 0.29278332f, -0.58299130f, -1.56022692f, -0.79441273f, 0.49289709f, 0.44112054f, 1.07305002f, 0.54899335f, 1.13781393f, 0.77809113f, 0.81795985f, 0.16576190f, 0.32552773f, -0.20250474f, 1.46543837f, 0.12731771f, 0.21013761f, -1.34241438f, 0.44267517f, 0.93246883f, 0.08808212f, 0.92653406f, -1.21083558f, 0.17247954f, -0.70557106f, 0.04630012f, 0.48834828f, 0.89634645f, 0.46683592f, -0.29553145f, 0.46363977f, -0.48971879f, -0.88603491f, -0.12333342f, 0.37073737f, 0.92061806f, 0.54675460f, -0.14716248f, 0.75578392f, -0.98173791f, -1.15983224f, -0.58713156f, 0.07950903f, -0.59016788f, 0.41622928f, -0.32474482f, 0.42086437f, 0.23061797f, 0.62596649f, -0.22615278f, -2.14721417f, 1.01685894f, -0.25976995f, 0.00739352f, -1.31597066f, 0.39005190f, -1.09549701f, 1.68375242f, 0.43331525f, -0.37124026f, 0.22255214f, 0.59654880f, -0.73840386f, -1.20048976f, 0.12226126f, 0.12997478f, 1.04826224f, 0.03894836f, -0.36289826f, 1.14466560f, -1.18198848f, -0.03713558f, 0.67677927f, -0.42329931f, -0.89409167f, -0.77874780f, 0.58438253f, -0.35176343f, -1.53329861f, -0.02995299f, -0.40145162f, -1.51052392f, 0.09194464f, -1.13275242f, -0.61983156f, -0.40004560f, -0.19893464f, 0.22134103f, -0.03903082f, 1.14894116f, -0.03476744f, 0.22520730f, -0.55851930f, 0.76650429f, -0.57863152f, -1.34161711f, -0.31498179f, -1.19411755f, 1.70044947f, -0.17428267f, -0.35983825f, -0.42613637f, 0.58165723f, -0.77866900f, -1.59727287f, -0.61723864f, 1.51078022f, 0.32971445f, -0.86441469f, 0.60552609f, 0.00208178f, -0.47096625f, -1.10479307f, -1.21652532f, -0.08211990f, -1.43739200f, -1.31684434f, 0.43312529f, -0.76822090f, 1.88128507f, -0.02179282f, 1.04971325f, -1.55004108f, 1.25337446f, 0.11203052f, -1.16048300f, 1.59467411f, -1.29469275f, 1.14019871f, 1.20021439f, 1.84098923f, 0.05004879f, 0.73529941f, 2.05272865f, -0.13080600f, -0.08436690f, -1.17919350f, -0.66256678f, -0.36727047f, 0.73840511f, 1.22293818f, -0.00206342f, -0.29839504f, -0.00618613f, 1.04213119f, 1.21176076f, -0.62886089f, -0.02589060f, 0.96009409f, -0.64478731f, -1.16516542f, 0.57528079f, 1.04294407f, -0.09774588f, 0.45935291f, 1.03263175f, 1.00633478f, -1.82209253f, -0.18035053f, -0.28302726f, -0.83813244f, 0.57593471f, -0.03807700f, 1.60498738f, 0.16530658f, -1.43083501f, 2.10824299f, 0.30279446f, -0.03961089f, -0.38900724f, 1.31272805f, -0.56575215f, 0.57970244f, -0.48305038f, 1.34114623f, 0.21859215f, 0.66399640f, -1.52087069f, -1.30717897f, 0.14394683f, 0.97648209f, -0.71372712f, -1.22574198f, -0.27702177f, 0.04041927f, 0.02442212f, 2.19617033f, -0.48566443f, 0.81463927f, 0.20383844f, 1.17562282f, -0.33829874f, -0.42141283f, -0.96415234f, -2.39141965f, -1.04285860f, -0.23004992f, 0.41186509f, 0.03811268f, 0.36818987f, -0.71099734f, -0.56749570f, 0.18486284f, -0.44530040f, 2.14008284f, -0.27467576f, 1.70690107f, -1.40462613f, 0.24697532f, -1.31629777f, -2.20674944f, -0.67868507f, -1.15767133f, -0.64391804f, -1.79037917f, 0.58749497f, -1.58303332f, -0.69021022f, 1.64376318f, -0.95393223f, 1.98415601f, -0.10991055f, 0.02474386f, 0.23683345f, -0.63420391f, -0.57991928f, 0.83028817f, -0.40033704f, 0.19212338f, 0.74640590f, 1.10264432f, -1.65286255f, 0.92683482f, -1.42252541f, -0.74605089f, 2.14535880f, 0.12971123f, -0.47971717f, 1.67546797f, 0.42268261f, 0.22648531f, -0.42369929f, 0.77403021f, -1.31818616f, -0.67143595f, -0.04311426f, 1.64128351f, 0.34776631f, -0.39353722f, -0.42765084f, 0.16170517f, -0.54488391f, -0.38428506f, 0.42097485f, -0.55982012f, -1.74543798f, 1.53704774f, 0.43562424f, -0.30395737f, 0.31846946f, 0.39205357f, 0.57386035f, -1.11912560f, -1.39164317f, -1.04337609f, 0.31629622f, 1.51927638f, 0.88745505f, -0.40445471f, 0.25783861f, 1.88646257f, 0.36509129f, -1.13266826f, -0.45394278f, -0.48400903f, -1.22332740f, 0.38626808f, -1.10049105f, 0.84138852f, 1.27863181f, 0.53942156f, -0.67743856f, -0.03896645f, 1.70393491f, 0.60997570f, 0.43368068f, -0.13338457f, -0.18920666f, -0.29583672f, -1.40738738f, 1.03876019f, 1.71253765f, 2.12821221f, -0.96092403f, 0.93841934f, -0.79030478f, 1.36427641f, -1.39196694f, 0.08514920f, 0.16223004f, 0.71259701f, 0.20150672f, 0.25068361f, -0.99952722f, 1.80129099f, -1.28586197f, -0.64957166f, -0.94813949f, -0.40161121f, 0.31977695f, 0.54932386f, -0.67757767f, 1.88086259f, 0.92337233f, -1.64887333f, 0.44333732f, -0.19468001f, 0.12977587f, 0.21171951f, 0.27679422f, 0.49134475f, -1.44429457f, 1.25617445f, 0.39978400f, 0.99869555f, -1.61617446f, 1.61177349f, 0.70243025f, -0.95748568f, -0.61795151f, -0.77302909f, 0.72967088f, 0.81964350f, -0.71813750f, 0.90140164f, -1.45950246f, -0.79972702f, 0.40875742f, 0.00152073f, -1.74491429f, 1.53776145f, 0.75769204f, -0.22075878f, -0.58385569f, 2.18884754f, 0.33597681f, -1.66265559f, 1.03805876f, -1.55245185f, -0.03582226f, -1.94542754f, -0.76081425f, -0.50471377f, 1.35763168f, -0.39631784f, -0.17134467f, -0.82220149f, -0.41021580f, -0.00940776f, -0.80176353f, -0.19816744f, 1.22061026f, -0.14486519f, -0.71727395f, -0.65721530f, 0.47020102f, -0.70403302f, -0.94795334f, 1.79884899f, 0.07779162f, -1.50615680f, 0.04140327f, -0.22001404f, 0.63735324f, 0.79237640f, -2.25412822f, -0.52519119f, -0.87280381f, -0.07100742f, -0.94734806f, -0.12286110f, -0.13623615f, -0.42595413f, 0.17547913f, -0.81707209f, 0.36855817f, -1.68186557f, 0.19312963f, -0.66249490f, -0.98283452f, -0.33314428f, 0.40918943f, 0.88268638f, -0.05390308f, -0.22440539f, -0.15879378f, -0.34859571f, -0.01013108f, -0.30005428f, -1.19408464f, 0.21789688f, -1.07769871f, 0.81475031f, -0.69555300f, 2.35201311f, -0.40362412f, 0.93497628f, 1.13343573f, 0.92343372f, 0.26987928f, 0.46123627f, 0.22577702f, 1.26289701f, -0.45956740f, 0.55994868f, -0.58410591f, 0.13304594f, -0.25806463f, 0.49044946f, -0.82065403f, -3.06672239f, -0.27774641f, 0.68504512f, -0.21386372f, 1.11427057f, -0.73201770f, 0.51655543f, 1.77261138f, 0.72081727f, 0.11116749f, 0.16637769f, -0.74987584f, 0.66579849f, -0.75808716f, 0.20678560f, -0.67698354f, -0.82141948f, 0.61008269f, 0.66520184f, 0.44894725f, 0.73015076f, -1.52517414f, 0.11714164f, 1.90452611f, -1.30355322f, 0.12144456f, 1.18547559f, -0.07349755f, -2.28061509f, 0.83522540f, 0.78438890f, 2.19334102f, 0.90305614f, -0.59345531f, 0.77925014f, 1.32338643f, 0.14068902f, 1.19032264f, 0.20666829f, -0.76595837f, 0.74967057f, 2.86965609f, 0.55690205f, -1.72530472f, -0.83317834f, -0.85842621f, -0.29678273f, 1.80955839f, -0.70496303f, 1.19106734f, -0.92985237f, -1.00617313f, -0.56049556f, -0.29382578f, -2.04022193f, -1.95356870f, -0.42553005f, -0.33369407f, 1.02115977f, -1.45769477f, -0.67720300f, 0.53819913f, 1.57643425f, -0.47015440f, -1.47861958f, -0.00545934f, -0.97836047f, 0.42680529f, 1.56110144f, -1.49487829f, -0.65198445f, 0.22720462f, 1.83036661f, -0.47099793f, -0.09915133f, 0.14923312f, -1.16313052f, 0.67798084f, -1.63665557f, -0.38220280f, 0.01719763f, 0.30041245f, 0.43148938f, -0.44021657f, -1.25734651f, 0.02465564f, -1.00845659f, -0.28574651f, 0.01367745f, 0.77253437f, -0.99399441f, 0.61445391f, 0.18343423f, -0.50997210f, 0.41359940f, 0.77279282f, 0.83511519f, 0.27929801f, 0.70800692f, -0.20278299f, 1.57884383f, 0.22650529f, 0.43347472f, 0.74003208f, -0.71401161f, -0.69829476f, -1.56766701f, -0.99254119f, 1.27301061f, 2.73726511f, 0.66089469f, -1.95778012f, -1.24642098f, -0.63579029f, -1.63168180f, -0.66980726f, 0.81933254f, 0.61866677f, 1.40594471f, 0.05158535f, 0.00196500f, -0.24592508f, -0.50780547f, -0.83905292f, -0.10748957f, 0.04490763f, 0.27769178f, -0.23227681f, 0.82108080f, 0.03562285f, 0.95483875f, -1.49897683f, 0.67809856f, 0.35497451f, -0.44021592f, -1.67361462f, -0.88895375f, 1.44293678f, -0.85046643f, -0.46437624f, -1.87252641f, 0.26775804f, -0.24535774f, 0.73365933f, 0.52253938f, 0.27947086f, -0.58796054f, 0.59045380f, 1.93476331f, -0.46775359f, 0.25238225f, -1.26601815f, -0.13324316f, -0.71454948f, -0.21610366f, -1.49586582f, 1.04903507f, 0.22208478f, 0.25512528f, -0.46157327f, -0.41319233f, -0.63846964f, -0.25100923f, 0.81277549f, -0.26959971f, 0.88737756f, 1.24578953f, -0.91121447f, -1.05756927f, 0.44390878f, 0.16672316f, -1.22941923f, 0.89547867f, -1.50212002f, -1.69620168f, 0.53339505f, -0.23656729f, -1.69879091f, 0.01510374f, 0.08315694f, -0.73196459f, -1.60263407f, -1.07601058f, -0.76389569f, -1.65307498f, -0.61484390f, -0.43546933f, 0.71318507f, -0.16273083f, 0.64122051f, -0.15406294f, 1.17673671f, -0.91240519f, 0.71091145f, 2.40497613f, 1.26343656f, 0.71469337f, 0.20705548f, 0.81776261f, 0.36253929f, -1.92106628f, -0.09300470f, -0.36648872f, 1.27732766f, -0.39180157f, -0.61186749f, -1.03455031f, -0.25079829f, -0.61479062f, -1.07094336f, 0.82218504f, 0.89934880f, 0.41308978f, -0.59968555f, 0.37682834f, -1.77388155f, 0.00294951f, -0.66145372f, -0.50789726f, -0.85123241f, -0.89909405f, -1.89454281f, -0.56692821f, 1.52272677f, -0.11961794f, 0.27843913f, -0.60582250f, 1.01871169f, -0.36098275f, -0.12242325f, -0.67375034f, -0.11204147f, -2.62773919f, -0.95901299f, 0.14040214f, 1.32364666f, -1.35099924f, -0.11077739f, -0.79319423f, 0.75949597f, -0.25485823f, -0.90959758f, -0.42373934f, -1.29850340f, 0.85699379f, -1.11882365f, 0.63470817f, 0.49696380f, -0.07983235f, -0.23903450f, -0.22618714f, -0.12117998f, -0.09442677f, 1.55589819f, -0.11996678f, -1.72700179f, 0.54683149f, -0.40804827f, -0.50099218f, 0.34596699f, -1.81841791f, 0.06385052f, 0.84428120f, 0.69901514f, 1.94559097f, 0.43251973f, 0.16794942f, 1.82829034f, 1.70959795f, 0.36130908f, -0.94608402f, -0.53498030f, 0.47781768f, -0.24203247f, 1.25065851f, 0.51788396f, -2.09381890f, 0.72973937f, 0.03281829f, 0.58632666f, 1.85737121f, -0.49569523f, 0.45921183f, 1.87173629f, 0.22803484f, 1.66433418f, -1.05872321f, -1.13663685f, 0.12397861f, -0.65112090f, 0.98152941f, 0.83739656f, -0.18783289f, 1.84249437f, -0.90706986f, -0.80824369f, -1.23854923f, -0.86488134f, -1.02627063f, 0.10976455f, -0.61403006f, 1.27554715f, 0.14653525f, -0.03953953f, -0.08512071f, -1.30043304f, -0.02566035f, 0.12054887f, 0.00282162f, 0.48921332f, -1.74398839f, 1.44554436f, -1.35854721f, 0.69256759f, 0.34101671f, 2.50045252f, 0.49121150f, -0.27115449f, 0.93974596f, 0.26258010f, 0.27151433f, -0.87214381f, -0.92580765f, -1.03269923f, 0.20615758f, -0.37822601f, 0.58983004f, 0.16426525f, 0.68218285f, 1.98158526f, 0.47492698f, 0.54224718f, 1.28722692f, -1.76915324f, -1.11240053f, 0.77428484f, 0.27184650f, 2.22473478f, -0.05574624f, 0.39976570f, -0.43911108f, 0.52805597f, 0.17340177f, 1.36057591f, -0.35004014f, 1.72787797f, 0.68357420f, 1.25532615f, -0.56752264f, 0.51840127f, -0.21237844f, -0.58821255f, -0.85278064f, 1.90179110f, -0.67447448f, -0.36831430f, -0.22930753f, 0.98231596f, -0.07011599f, -0.08560387f, 0.05998110f, -0.02481356f, -0.57335132f, -0.44288307f, -0.24468307f, 0.53321087f, 1.19609559f, 0.10664973f, 0.24379487f, 0.93687552f, 0.93615580f, 1.74319768f, -0.68310338f, 1.32163060f, 0.61918712f, -0.76501870f, -0.54549301f, 1.74077415f, -0.69977754f, -0.66880983f, -1.15981388f, 0.81571609f, 0.53788543f, 0.47898352f, -0.02484704f, -1.64646924f, -0.69822907f, 0.27020717f, 0.05027051f, 1.75149667f, 0.01548872f, 0.32615909f, 2.55151844f, -1.29172051f, -0.36133784f, 0.98637396f, 0.14009331f, -0.50038946f, -0.92230296f, 0.17307127f, 1.05361068f, -1.46784890f, 2.38960409f, 1.19413340f, -1.33349669f, 1.59141159f, -0.71811068f, 1.22429430f, 1.26947939f, 1.08177102f, -1.18138707f, -0.72775704f, 0.17282635f, -0.40554270f, -0.40341887f, 0.46564049f, -1.02069795f, -0.07653128f, -0.13979210f, -0.31195050f, -1.72042310f, 1.37131393f, 0.63849634f, 0.75561279f, 1.81152904f, 0.26686314f, 1.32796574f, 0.56100166f, 0.70058894f, -0.88962644f, -0.04360984f, -0.88249093f, 0.24311203f, 0.50410056f, -2.22567797f, 0.94520348f, -2.12467694f, 0.47282359f, -0.71379906f, -0.09857135f, 0.62374717f, 1.37182784f, 0.73380554f, 0.59745449f, 2.80427694f, 0.67253572f, 1.65335357f, 1.69891667f, 1.34585941f, -0.79989213f, 1.44980943f, -0.52013642f, -0.46971673f, -1.50070012f, -0.25687039f, -0.56916732f, 0.71065760f, -1.31996286f, 0.96031237f, 0.13929774f, 1.49679291f, -0.05966444f, -0.58674580f, -0.08278833f, -0.93390942f, 0.42415768f, -1.77889526f, 0.75336021f, -0.72699982f, -0.82880586f, 0.63955617f, 0.42771208f, -0.42366457f, -0.91581815f, 0.94750947f, 0.43123913f, -0.99053741f, 0.70470595f, -1.16662264f, 1.14847183f, -0.83885664f, 0.46714026f, -2.27748466f, -1.23656678f, 0.14695056f, -0.33159894f, -0.52553117f, -0.04391259f, -0.29630372f, 0.25949728f, 0.96991086f, -0.37714824f, -0.28251833f, 0.16106486f, 1.38844633f, -0.18713553f, -1.30708838f, 0.48490265f, 0.29553881f, -0.45505449f, 0.83341682f, 0.87346369f, -0.63516861f, 0.66063565f, 0.93892503f, -2.73996735f, -0.81515318f, -0.91458052f, 0.00978268f, 0.43472794f, -0.08090764f, 1.37249672f, 0.76722521f, -1.19154143f, 0.22046764f, 0.34916410f, 0.51383299f, -0.56379753f, -2.49949312f, -0.74207872f, -0.68400806f, -0.09663232f, -0.07199454f, -1.05562651f, -0.75028551f, -0.87253797f, 0.69039482f, 0.45923674f, -1.27515161f, -0.04555376f, -1.41501272f, -0.83773375f, -0.74807298f, 1.36646152f, 0.06317432f, -1.32559633f, 1.89092779f, 1.24883330f, -1.03608561f, 1.08677161f, -0.99629849f, -0.69947034f, -0.85716367f, -0.07947286f, -0.25485426f, -0.19732477f, 1.64581251f, 1.04618108f, 1.87186897f, -0.18198362f, -0.83807969f, 0.70462501f, -3.18930101f, 0.74610996f, -0.60935193f, -0.49383929f, -2.88986492f, 0.51707613f, 1.04620326f, 1.09837818f, -1.19840038f, -0.10391295f, -0.20789115f, -1.51052022f, -0.31087330f, 0.22411564f, -1.30506921f, -1.52000105f, -1.51593041f, 1.04321992f, 0.97611690f, 0.90424490f, 1.83324766f, -0.08682299f, 0.47035542f, 1.70865905f, -0.31108001f, 0.04115159f, -1.36352801f, -0.90797836f, 0.32128647f, 0.66191489f, 0.08681208f, 0.14993365f, 0.47110486f, -0.31522670f, -0.38906571f, -0.08876022f, -0.13106902f, 2.25685239f, -0.62211353f, -1.68553007f, -0.23707703f, 0.69236159f, -0.46686995f, -0.27520603f, 0.26619941f, 1.48525345f, 1.61278927f, 0.49452963f, 1.20846486f, -1.11853909f, -0.30010033f, -0.75471467f, -1.69959772f, -0.52042168f, -0.43881389f, -1.45240712f, 1.02122891f, 1.73639011f, -0.03813924f, -0.22239220f, 0.15797073f, -0.64418089f, -0.60228932f, -0.83248150f, -0.02042520f, 0.38137484f, 0.86056453f, 0.06410559f, -0.62785137f, -0.49916875f, -2.53796315f, -0.79168582f, -0.69197005f, -0.77175534f, -0.28669405f, -0.79764080f, 0.97218460f, -0.10351621f, -0.52759898f, 1.02840185f, 1.16363287f, 0.08351815f, -0.61088538f, 0.59944046f, 1.54409397f, -1.39842033f, 0.27917057f, -0.27146137f, 1.46310735f, 0.03626106f, 0.15038440f, -0.07894899f, -1.42527366f, 1.69641745f, 1.48384345f, -0.43328866f, -0.54252565f, -0.94416499f, 1.54436302f, -0.81367069f, -1.67925239f, -0.17525831f, 0.27891046f, -0.69066733f, 0.89911050f, 0.11606655f, 0.67450327f, 0.41538724f, 0.90886223f, 1.19786549f, 0.85810721f, 1.32862210f, -0.83469814f, -1.09682298f, 0.88092703f, -0.97478902f, -0.11664717f, -0.07929394f, -0.69581884f, -0.16928329f, -0.70731819f, -0.40485084f, -0.28954300f, 0.52882415f, 0.38769314f, -1.38704026f, 1.15099049f, -0.43566978f, 0.34459323f, 0.49520254f, 1.11130333f, 0.28783718f, -0.53783375f, -1.63577271f, 1.02222812f, 0.86302060f, 0.48346213f, 0.46627176f, -1.30133855f, -1.48477137f, 0.31219670f, -1.21498191f, 0.89838904f, 0.87186617f, -0.39968935f, 0.34930915f, -0.32909471f, -1.39364409f, 2.13006306f, 0.33270469f, 0.00215986f, 0.97776711f, 0.24908836f, 1.56164885f, 0.45157790f, -1.55970144f, 0.27677536f, 0.07662498f, -0.08262251f, -0.17658773f, 0.65820259f, 2.01052690f, -1.71946216f, 0.84686053f, -1.23594892f, 1.40792072f, -1.47772563f, -0.36132276f, -0.50405115f, 0.09009213f, 0.81659186f, 1.85574234f, -0.64974433f, 0.63352364f, 1.01766217f, -1.54804432f, -0.42570522f, -0.24763709f, 0.72822112f, -0.93733686f, 0.68087620f, -1.40644944f, 0.48672482f, 0.09725539f, -0.64416331f, -0.95747960f, 0.36771363f, 0.39155054f, -0.71790671f, -2.17222738f, -0.08655047f, -0.97842115f, -0.22991380f, 0.52029115f, -1.42072022f, 0.29576331f, 0.32391560f, -1.00823236f, 1.67909145f, 1.16841447f, -0.32307062f, 0.15756166f, -0.97590631f, -0.39429301f, -0.03583352f, 0.17554663f, 0.57961231f, -0.46873134f, -0.23343173f, -0.85060924f, 1.71745574f, -0.04658702f, 0.63088381f, -0.67581934f, -1.53171062f, -1.58800113f, -1.17987096f, -1.16737640f, -0.87544650f, -1.17138922f, 0.38979119f, -2.39369726f, -1.34747124f, 0.58450359f, 0.87791806f, -0.04459394f, 0.97995293f, -0.10354915f, 0.65324986f, -0.17833626f, -0.85849386f, -0.42063358f, 0.19708554f, 0.10255250f, -0.59539181f, 0.86194044f, 1.68610668f, 0.55275291f, -0.43127069f, -0.04218780f, -0.08466262f, 0.31236625f, -0.92824298f, -0.09879152f, 0.32358822f, 1.04045570f, 0.35617545f, 0.09059231f, 1.19069445f, 1.96978688f, 0.63561743f, 0.15030998f, -0.29879019f, 0.22774190f, -1.01608860f, 1.03605175f, 0.47804731f, -0.30450734f, -0.61382371f, 0.45390254f, -1.93547988f, 2.01267338f, 0.52447683f, 0.18379784f, 1.11913633f, -1.24273467f, 0.15803322f, 1.72184098f, -0.79349059f, 0.10258614f, -1.53445125f, 0.02630571f, 0.81649125f, 0.91089755f, -1.12968338f, 1.04016411f, 0.28999722f, 0.74863863f, -0.61388236f, 0.01665530f, 1.43592548f, 0.68138391f, 0.11963340f, -1.26123953f, 1.36340797f, 0.25696915f, -0.58877039f, 1.42209792f, 0.55563360f, -1.33329606f, 1.84695840f, 0.88433737f, 1.04359078f, 0.18906727f, -0.03448994f, 1.17944050f, 0.86783957f, 0.44934425f, -0.77892244f, -1.76232874f, -1.01689589f, 0.78943914f, 0.92141974f, -1.00187087f, -0.13809921f, -0.90222073f, 1.10094714f, -0.13657950f, -0.44349849f, -1.61441302f, 1.05724919f, 1.50337231f, -0.05785890f, -0.76958144f, -0.51498759f, 0.69227600f, -0.37975949f, 1.31949317f, 0.82049531f, 0.32868597f, -0.31557772f, -0.75534385f, 1.27303052f, 0.43453619f, 0.11296938f, 1.18182182f, 2.23387384f, -0.86412978f, -0.01599468f, -0.70869064f, -0.09221385f, -1.23729551f, 0.79490280f, 0.03522846f, -0.95069039f, -1.73461652f, 0.72329187f, 1.40385795f, -0.11585230f, -0.78033113f, 0.07491048f, -1.12873089f, 0.18476245f, 0.57568848f, -0.28792691f, 1.35411644f, -0.76956165f, 0.29571572f, 1.03178787f, -0.38780826f, 0.31680650f, 0.69368076f, -1.23856580f, -0.49848995f, 0.14766994f, 1.02625990f, 3.03858209f, -0.51030380f, 0.96796870f, 1.35078156f, -1.07729447f, 0.84322494f, 0.54886484f, 1.31453705f, -0.45792100f, 0.31196272f, -0.15701357f, 0.83586836f, -0.74952888f, -1.17432022f, -0.31002575f, -1.02149463f, -0.36117774f, -1.22079086f, 0.03532525f, 0.00555908f, -0.45891216f, 0.29636297f, -0.68272704f, 0.41257843f, 0.37988129f, 0.01747893f, 0.82739186f, 1.52292180f, -0.79456621f, 2.20275712f, 2.13212132f, -0.81393015f, -1.15712392f, 0.22488308f, 0.62776327f, -0.85444915f, 0.44017896f, 0.05863331f, -0.83198178f, 0.93063420f, -0.16121253f, 0.12382501f, -0.37826315f, 0.93118382f, 0.19507533f, -0.58595538f, 1.46994352f, 0.13170272f, -0.70031989f, -0.12820166f, 0.30487457f, 0.84148771f, -0.68807501f, 0.21187615f, -0.67030680f, -1.79136002f, 0.70810199f, -1.20959783f, -0.08468831f, -0.06317700f, 1.35527098f, -0.47018668f, -0.91693246f, 0.14818805f, -0.05405350f, 1.16875637f, -0.17363262f, -1.61833882f, -0.32934523f, -0.38346377f, -0.62702698f, 0.34135151f, 0.48015586f, -0.65263331f, -0.04689486f, 0.01156854f, 0.37580970f, -0.16174591f, 0.59627324f, 0.24351901f, -0.87983090f, 1.57049024f, 1.25836349f, -0.41464049f, -0.62279183f, 0.09693756f, -0.23850618f, -0.49007827f, 0.22298151f, 0.10914832f, -0.35192192f, -1.27221346f, 1.10203624f, -0.86399704f, -0.47319838f, -0.77105570f, -1.68624854f, 0.81198281f, 0.82534081f, 0.75654501f, 1.47631240f, -0.61000234f, -0.58933264f, 0.54822850f, -1.22829592f, 0.11107657f, 0.56449169f, 1.50693524f, -0.59280968f, -0.64286685f, -0.20120731f, 0.27184448f, 1.55500400f, -0.48919386f, 1.04044867f, -0.87048137f, -0.40569979f, 0.21908638f, -0.51829034f, -1.48748124f, 0.02990401f, 1.83462536f, 0.29885170f, 1.32370698f, -1.30129600f, 2.43271399f, 0.22967771f, -1.13014007f, 0.95529765f, -0.83325785f, 0.43633386f, 0.85774118f, 0.78160155f, 0.58583075f, 1.18906367f, -1.54354560f, -0.68320692f, 0.01900371f, -0.79777133f, 0.12851712f, 1.10176420f, 0.79418170f, -1.41154039f, 0.36929929f, 1.12176800f, 1.23849642f, -0.89377707f, 1.01390159f, -0.50889206f, -1.12554002f, 0.17932732f, 0.48949540f, -0.54235244f, -0.28146735f, -1.39125514f, 0.13309635f, -1.12864995f, -1.29901242f, -0.04266220f, -1.98028529f, -1.34869373f, 0.00038156f, -0.92473024f, 1.48010647f, -0.02754467f, -0.26030368f, 0.93083733f, 0.27946711f, 0.64052200f, -0.04220961f, 1.25002527f, -1.07923257f, 0.19048618f, 0.08900311f, -0.40813437f, -0.73068553f, 0.52122378f, 0.68990833f, -0.38749605f, -1.09269309f, -1.63480806f, 1.01789618f, -0.61596102f, 0.81049860f, 1.30838764f, -1.49213874f, -0.77916288f, -0.72660202f, -0.92013240f, -1.61726642f, -0.11527207f, 0.35143322f, -1.11646879f, -1.45525432f, -0.82892823f, 0.15512508f, 1.01891017f, 1.40162635f, 1.02494884f, 0.33882582f, -0.78747398f, -0.26009330f, -0.38519114f, 0.79247451f, 0.02065756f, -0.48030257f, 1.01167107f, -1.74057114f, -0.84549171f, -0.15337363f, -1.92544484f, 1.01270044f, 0.00762185f, -0.16405612f, 1.61778915f, 0.93316060f, -0.68960994f, -1.13214970f, -0.94695878f, -0.28418848f, 0.17102109f, -0.08787476f, -1.83799696f, -0.13761258f, -0.18652774f, 1.46456254f, 0.34169790f, -0.40697145f, 1.49663997f, -0.99555492f, -0.67775637f, -0.51951116f, 1.35157657f, -0.27099034f, -0.46987835f, 2.28101230f, 0.59104478f, 0.75010139f, 1.01472175f, 0.25741309f, -0.56074983f, 1.12267506f, 0.35336846f, 0.61733276f, -1.63976014f, -0.17700450f, -0.25093642f, -0.75599891f, 2.10956192f, 0.95155340f, 0.72049862f, 0.50492924f, 0.62067389f, 2.08688402f, -0.73604703f, 0.63383341f, -0.53528428f, -2.11538506f, -0.98173052f, 0.59560484f, -0.26205051f, -0.91948050f, 0.00593397f, -0.11734286f, -1.41261208f, -0.83611172f, -0.27682739f, -0.20619918f, -0.36557615f, 0.77194935f, 1.67695415f, -1.39265156f, 0.04892010f, -0.37773246f, 0.16124558f, -0.18348448f, -1.38248885f, 0.58459854f, 0.65064198f, 1.11349559f, 0.36708066f, -0.15471332f, 0.14208725f, -2.06860566f, 0.29629150f, 0.93084633f, -0.47215626f, 0.60208917f, 0.95415461f, 1.03390312f, -0.03639749f, -0.23988228f, 1.27037442f, 0.95133096f, 0.33187470f, -0.34527761f, 0.22134073f, 1.01799667f, -0.81475645f, -1.18869019f, 0.23314142f, 0.25180560f, -1.23762786f, 1.25283313f, 0.16980635f, 0.40740708f, 0.59256923f, 0.16274920f, -0.69713289f, -0.16444311f, -2.41602516f, 0.37952334f, -0.05604568f, -0.23772651f, 0.20581599f, -0.54303211f, 1.71877348f, 0.83602583f, -0.32586128f, 0.73609394f, -1.73640239f, 0.07249248f, 0.31248692f, 1.77627432f, 0.97660398f, -0.42095289f, -0.18750280f, -0.84246057f, 0.29762223f, 1.87054563f, -1.46980762f, -0.45306337f, 1.52366042f, 1.39061129f, -0.04980387f, -0.55382830f, -0.96987218f, -0.06910808f, -0.41276473f, -0.83891344f, -0.92597574f, 0.60252470f, 0.21938549f, -0.04451685f, -1.00330937f, -0.36955237f, -1.52876902f, 0.27296364f, -1.96721256f, 0.05291027f, -0.91540521f, 0.48990685f, -1.99560380f, -0.68551093f, -0.14532298f, -1.56881595f, -0.08319287f, 0.31003201f, -1.42829597f, -0.61810297f, -0.03581250f, 0.77747720f, 1.25297558f, -1.36239243f, -1.13274276f, -0.35045877f, -2.34157228f, 0.04515179f, -0.83044821f, 1.81353962f, -1.36855912f, 0.39704823f, 0.16665934f, -0.16654585f, 1.17806077f, 1.00086153f, -1.25474250f, -1.46876431f, 1.18021631f, -0.32257929f, 2.12062597f, 0.86819613f, -1.18048275f, -1.69747460f, -0.74092305f, 0.05086798f, 1.15339577f, 1.32972670f, 0.27247882f, 0.98499072f, 2.35597157f, 0.30179837f, -0.66633248f, 0.13794266f, -0.22753908f, -0.22868259f, -1.81792033f, 0.50151759f, -0.79408127f, -1.05343878f, 0.45727381f, 0.84800923f, -1.73605800f, -0.02032863f, 1.82778001f, 1.41025102f, -0.81715560f, 0.25888795f, -0.25075480f, 0.66256499f, 0.11993053f, 1.81336939f, -0.06345166f, -1.49658346f, 0.07531686f, 0.96972889f, 0.87405980f, 0.75830793f, -0.13497087f, -2.45855975f, -0.65984958f, 0.93919373f, -0.97305542f, 0.73477978f, 1.04337513f, -1.22712576f, -0.46385625f, -1.20876372f, -0.82760453f, 0.01455977f, -1.05089867f, -0.02801843f, 0.60899758f, -0.82052249f, -1.48932517f, -0.98073828f, -0.19311285f, -0.25602359f, 0.50351876f, -1.24557400f, -0.82138073f, -1.45966852f, 0.44991320f, -0.75550151f, -0.98550314f, -1.21418869f, -1.15771639f, -1.72192061f, -0.39616469f, -0.55566746f, -1.31880891f, -0.08843257f, 1.00422776f, 0.35846478f, 0.46060917f, 0.77326930f, 1.60129988f, -1.85124147f, -0.30582917f, 1.30227256f, 1.81890345f, -0.44084981f, 0.25315762f, 0.70259613f, -0.94882858f, 1.97040296f, 0.71473581f, -0.68193883f, -0.36290962f, 1.16348684f, 0.15418798f, 1.07806778f, 0.40554729f, 0.10280909f, -1.06474805f, 0.64398485f, -0.63568884f, -0.06108581f, -1.03290677f, 1.02834034f, 1.15284693f, 0.14046004f, 1.86630619f, 0.46804786f, -0.68397558f, 1.60733378f, -1.64890087f, -1.03819239f, -1.19212389f, -0.78382361f, 0.03925850f, 1.52259934f, 0.09540676f, -0.21220762f, 0.55955195f, -0.39845437f, -2.14541650f, 0.49337825f, -0.68574250f, 0.74040270f, 0.50783634f, -1.60461199f, -1.26806450f, -0.12652303f, -0.83992827f, -0.15524681f, 0.40098447f, 0.23392735f, -0.23262636f, 0.06525709f, -0.35994548f, -1.08432877f, -0.21395946f, -0.78357452f, -0.57157278f, 0.71407390f, 0.86596155f, -1.13723528f, 0.13460183f, -1.20881450f, 0.71018457f, 0.68943661f, -0.70428050f, 0.64600736f, 0.01990297f, -0.10575775f, -0.80263519f, 0.10618331f, 0.08865548f, 1.51651669f, 0.60851854f, 1.15161908f, 1.04919207f, 1.18359745f, -0.04352076f, -0.83643389f, -0.07922365f, 0.10597949f, -1.34984851f, -1.91319740f, 0.71585363f, -2.10845160f, 0.64385056f, -0.54551518f, -1.02039802f, -1.62510490f, 1.65401149f, -0.42711899f, 0.07970079f, -0.21404363f, 0.30498922f, 1.07942021f, 0.63995659f, -1.82114816f, 0.56396323f, 1.07084870f, -2.00350380f, 0.53339815f, 0.18500003f, 1.15034151f, -0.21436051f, -0.99986565f, -0.58812016f, -0.07247020f, 0.78910017f, 0.48839527f, 0.98795873f, 0.10357288f, -0.05604928f, 0.38977858f, 0.73745090f, 1.40838420f, 0.25967824f, 0.23588051f, -0.03451392f, 1.04897523f, -1.77121758f, 2.35625434f, -0.67086869f, -0.84005541f, -0.85940343f, -1.04449213f, -0.65917015f, -0.78713167f, -0.95910054f, 0.38597879f, -0.31879017f, -0.86260867f, -1.08593106f, 0.02802678f, 0.99484950f, -0.55113328f, 2.60936737f, -0.03388772f, -0.47583574f, -0.14021793f, 0.99019170f, -1.22431207f, 0.78734446f, -1.77037835f, 0.15018673f, 0.36423206f, 1.36447549f, -1.61007094f, 0.51875496f, -1.60788095f, -1.73557448f, -0.41414359f, -0.93710536f, 0.38715765f, 0.04243837f, -1.59682858f, -1.10728157f, 1.88292623f, -1.01428258f, 0.01074958f, -1.88169158f, -0.31616244f, 0.45334938f, 1.12449574f, -1.16699445f, -1.59505820f, 0.04126552f, -0.89016622f, 0.45838884f, 0.71463561f, 0.14563711f, 0.30694655f, 0.67193079f, 0.61429602f, 1.00201404f, -0.49295208f, 0.05997690f, 0.99491668f, -0.73801446f, -1.17185295f, 0.94778723f, 0.36106884f, -0.43561545f, 0.04102699f, 0.52626407f, 0.08442099f, -1.57626402f, 1.56855237f, -1.65396678f, 1.74014664f, -0.38219589f, 0.39305371f, -0.31705827f, -1.15742850f, 0.11669596f, 0.54043210f, -0.52270615f, -0.13375773f, 0.68094701f, -1.84134769f, -1.49383473f, 0.14632171f, -0.54607725f, -1.20867658f, -1.28439069f, -1.81734920f, 1.54257309f, 0.78347659f, -0.24049839f, 1.69973648f, 0.99825776f, 0.99971974f, -0.26055810f, 0.34143049f, -0.44862366f, 0.11253342f, -0.60932243f, 0.70383030f, -1.87318194f, 0.21953633f, 0.82791799f, 1.64545465f, -0.42693698f, -0.64897031f, -0.97996652f, -1.06616282f, 0.52939081f, -0.12541170f, -0.57480675f, 0.73600835f, 0.35711968f, -0.03528263f, 0.79997194f, 0.55742902f, -0.28909785f, 0.64331138f, -1.79893720f, 1.01572442f, 0.27111965f, -0.51778597f, 0.12906317f, 0.76148927f, 1.51315522f, 0.41101140f, 0.38008851f, 0.66759896f, -0.13804778f, 0.64854795f, 1.73474562f, 0.75999504f, -0.73411214f, -0.05406699f, 1.35664344f, -0.25298578f, -0.12696666f, -0.42628938f, 0.61129904f, 1.55259824f, -0.05820796f, -0.38598019f, -0.87325627f, -0.55066222f, -1.24557889f, -0.26509118f, -0.32103062f, 1.14031804f, -0.75985742f, 0.70659167f, -1.15016067f, 1.24906838f, 0.90396994f, -0.16241251f, 0.43682271f, -1.42695689f, 0.47134697f, -1.66143429f, 0.08698819f, -1.00775325f, -2.24129725f, -1.04226267f, -0.98537570f, -0.89938259f, -1.80710697f, -1.22866321f, 0.78125423f, 1.55150509f, 0.46235040f, 0.18444096f, 0.19313288f, -2.20686269f, -0.40341458f, 0.50321484f, 0.47339424f, -0.81383848f, -0.21972439f, 0.66612029f, 0.60239881f, 1.20443010f, 0.70015103f, 0.30632916f, 0.01489905f, 0.68129027f, -0.89645082f, -2.68969011f, -0.96684915f, 1.66421318f, 0.74333072f, -0.78321886f, 1.60063362f, -1.27524030f, -1.95856726f, 0.47504124f, 0.15398432f, -0.20796098f, -0.13449343f, 0.93458968f, 1.60390890f, 0.21798505f, -0.27035928f, -1.23248971f, -1.25361061f, 1.34666133f, 1.07233441f, 0.88799530f, -1.23687923f, -0.40781614f, -0.11916534f, -0.88050151f, -0.66422415f, -2.61471510f, 0.78276747f, 2.42323995f, -1.70715427f, 0.71550035f, -0.60298312f, 0.70491880f, 0.46175584f, 0.80827898f, -0.45108104f, -0.98219043f, -1.72823501f, 1.73190725f, 0.53906441f, -1.50445580f, -0.59250867f, -0.07239901f, 0.44743437f, -0.13740127f, 1.69935930f, -1.00480616f, -0.58191377f, 0.39853972f, -0.60960841f, -0.45473522f, -0.76396072f, -0.31872150f, 1.74509728f, -0.59950751f, 0.89810580f, -0.81400329f, 1.14280319f, 1.11165059f, -1.31295311f, -1.60784578f, -0.87506992f, -1.13461006f, -2.09486437f, -0.16449419f, -0.37728927f, 0.47595578f, -0.55342919f, -0.17574213f, 2.21499181f, 1.14331865f, -0.14938518f, 0.18935619f, -0.33802557f, 0.52538890f, 0.82673949f, 1.16562462f, 1.24713838f, 0.98890215f, -0.64991701f, 1.49886703f, 1.97769642f, 0.08059916f, -1.60925281f, -1.23822486f, -1.40829837f, 0.51331180f, -0.29928651f, -1.04348791f, -0.39911583f, 0.69380492f, 1.54516888f, 1.22791195f, 2.25008130f, 1.33348894f, -0.21775827f, -0.71937007f, 0.54982573f, 1.70691478f, 0.32459491f, -0.57187974f, -0.21614684f, 1.08274269f, 0.41384646f, 0.24497485f, -1.43703413f, 0.89616930f, 0.82032162f, -0.24598582f, 0.84271127f, -0.81894702f, -0.01828136f, 1.70397091f, 0.39505738f, -0.51221430f, -0.87979966f, 0.10795479f, 0.45194778f, -0.76008922f, 1.23394477f, -0.56798172f, 1.06459570f, -0.44333413f, -2.40399075f, -0.37267187f, 1.42946172f, 0.95734519f, 1.86127949f, -0.15217264f, 1.68742633f, 1.97638428f, -0.44211119f, -0.98393327f, -0.54173928f, -1.72017395f, 0.74697793f, -1.77827263f, -1.92299354f, -0.17189410f, -0.48633271f, -2.21230388f, -0.45906609f, -0.53493047f, 0.37253976f, -0.56951141f, 0.07728028f, 0.03530006f, -1.18123293f, 1.94158125f, -1.55930352f, 0.69334733f, -1.95163214f, -0.95800400f, -0.01804711f, -0.56747472f, -0.99099451f, -1.52853060f, -0.98279524f, -1.67307866f, 0.96121490f, 0.35654056f, 1.74034202f, -1.44633865f, -0.27781928f, 1.79457986f, -0.41029963f, -0.76871634f, 0.36555341f, -0.77664107f, 0.19535238f, -0.76185411f, -0.19828433f, -0.88820636f, 0.63885397f, 0.11346363f, -2.50265074f, 0.16319332f, -1.01288569f, 1.86605489f, 0.89761645f, 1.11795115f, -0.00714116f, -0.89034635f, -0.76447034f, -0.18822117f, -0.48340848f, -0.99788517f, 1.02172959f, -0.39395007f, 0.72566581f, -0.81438208f, -0.71715081f, 0.96243578f, -1.36424279f, -1.13870537f, 1.17602491f, 0.16320205f, 0.71959788f, 1.66669416f, 0.55690295f, -0.28912008f, -1.19219172f, 0.23308393f, -0.37963116f, 0.45347008f, -0.42606446f, 1.30938649f, 1.25128853f, 0.57649273f, 0.34440875f, -0.23893952f, -1.06604803f, 0.31336102f, 0.75727910f, 0.46772480f, -0.37650385f, -0.06036821f, 1.03686309f, 0.46158856f, -1.81028461f, 1.43393028f, 0.85494965f, -2.34685564f, -0.17571987f, -0.45592231f, -1.31190526f, 1.73194158f, -0.11856517f, 0.07041293f, 0.25689471f, -0.56000596f, 2.06649089f, 0.38954756f, 1.36627376f, 0.13905638f, 0.77370811f, 0.43944249f, -0.08798827f, 0.07245751f, -1.30234015f, 0.29710820f, 0.74389762f, 0.11971968f, -0.07381748f, 1.32652700f, 1.34079397f}); - auto input2 = NDArrayFactory::create('c', {3, 4, 4, 5}, {0.98114507,0.96400015,0.58669623,0.60073098,0.75425418,0.44258752,0.76373084,0.96593234,0.34067846,0.57962620,0.77517051,0.97472977,0.79237527,0.68690428,0.21719366,0.79959206,0.84814187,0.22496814,0.08646965,0.31110474,0.79813162,0.19661444,0.57760099,0.72138960,0.15244268,0.87687051,0.11130344,0.01087698,0.34817841,0.54992017,0.23443850,0.31725614,0.59755220,0.20364695,0.00531392,0.23403114,0.07442912,0.83707647,0.89291743,0.09044587,0.69041462,0.29904183,0.61904680,0.85306847,0.34467042,0.95839152,0.54517124,0.29640937,0.94855959,0.95970016,0.94045145,0.95510301,0.34666505,0.34717010,0.69245678,0.71669175,0.59043738,0.64924132,0.06033522,0.60185199,0.04690073,0.59241154,0.40229547,0.23002481,0.45161195,0.73743778,0.93209113,0.37294358,0.50177744,0.15072501,0.26146917,0.05252146,0.04758931,0.76448288,0.85149045,0.08840467,0.07692576,0.33180160,0.27241259,0.74834620,0.56453640,0.23057286,0.68429752,0.11961551,0.39045977,0.44356094,0.77018807,0.07984410,0.47926806,0.26165759,0.18606064,0.89972877,0.17962874,0.47273120,0.64641705,0.61890443,0.58730015,0.25937832,0.35231561,0.10243882,0.17459193,0.95906995,0.09227025,0.30003223,0.41601210,0.38269713,0.84799751,0.59295173,0.76277990,0.68910424,0.37672606,0.40675461,0.94346058,0.91438505,0.84728183,0.64367667,0.74899979,0.60570691,0.16417363,0.68852426,0.85486889,0.22585792,0.86953176,0.07465519,0.93096301,0.38008822,0.38752587,0.44004038,0.13170612,0.94541045,0.89349973,0.69245307,0.94978877,0.98776658,0.79445884,0.30607409,0.58264961,0.37980538,0.41810784,0.48903038,0.51615888,0.57682794,0.82481897,0.78341080,0.48446465,0.17447931,0.71125424,0.30263851,0.70675352,0.03215584,0.92381065,0.22343694,0.08851149,0.91402490,0.70074717,0.30912192,0.37723206,0.97579397,0.23554587,0.95939133,0.41565709,0.01741416,0.58362787,0.22106662,0.89065537,0.31900249,0.41280911,0.67947610,0.04545590,0.15352812,0.85412524,0.84933222,0.80000225,0.93147073,0.70094105,0.69269875,0.95282194,0.65913582,0.79186874,0.59855248,0.39707430,0.95126239,0.15618217,0.33446689,0.98123758,0.84770758,0.98081012,0.54427413,0.18728519,0.89792955,0.53360126,0.72812986,0.13307744,0.51217443,0.66708084,0.29416915,0.31298995,0.39155037,0.29288291,0.87063305,0.61759154,0.73723332,0.37167635,0.82122716,0.22937430,0.76570536,0.47911792,0.02826214,0.94277323,0.59945469,0.19042060,0.68173155,0.82771295,0.95649538,0.40833101,0.90838542,0.55245881,0.49011012,0.36773444,0.34513527,0.42050683,0.16113964,0.30969388,0.27174174,0.12117655,0.35270175,0.81967867,0.63723136,0.84309389,0.71822576,0.84883484,0.32306117,0.08176457,0.56175486,0.34892198,0.09306929,0.85437582,0.13925577,0.48629188,0.29923539}); - auto exp = NDArrayFactory::create('c', {3, 8, 8, 16}, {5.98743296,-2.83037376,-0.87943113,1.41339970,1.32433391,-1.20299149,-0.02893090,2.05326009,1.19417048,5.58212376,3.28139353,1.19237995,-1.09431255,-2.55264497,3.11014652,6.81296825,-2.09029293,-4.32068443,-0.52808392,-1.97968531,-0.18673831,0.84605980,4.55825520,2.71503139,0.15210046,0.85310984,-3.82062817,2.76470995,3.69004202,-1.45017099,-2.59361267,-1.35094655,7.24145126,-5.25432396,0.19920218,-4.30596399,1.35318923,-3.88142037,3.67493343,2.25931478,2.87630725,1.66349852,6.21347952,0.94105923,-1.61742055,-2.35699606,0.12850338,1.79141688,-2.09535933,-6.35418081,-0.06303531,-4.38615131,0.48237842,0.26528549,3.38231516,3.76315165,-0.40254810,-0.23716694,-6.13381910,-0.41950428,-0.89680839,-1.46491277,-1.98541689,-0.99357355,5.58237648,-2.38937521,-0.00872564,-2.37138414,4.91117287,-4.51916361,0.97943687,2.91052818,-2.50362611,1.70252812,5.04137802,3.57108784,-1.87532270,-3.66677809,-2.38861251,5.55765152,-7.27571774,-1.68887305,-0.72266489,-4.42809057,-0.92118186,1.02381468,4.44284725,5.17150497,-0.42438728,2.02693963,-1.36484981,-1.47912180,0.26649538,-0.02091765,-2.86906910,-3.03046989,1.35122132,-3.21707630,2.21112418,0.24121630,3.96940088,-7.66105747,2.76352382,-0.99061489,-2.16720009,-1.63170409,1.12701774,-1.02415371,-0.90435314,-1.51372027,-0.76884907,0.39066136,-0.89562428,-2.03204703,1.28074932,-2.14551091,-2.36843777,0.46580017,0.75451565,-0.00336730,-1.06597757,3.27195978,-0.41307712,-0.10376054,-1.34102952,-2.22901654,2.31929803,1.40851438,-2.23774385,0.20417206,-1.12153268,-0.13188094,-3.96649432,2.10269976,0.49845099,6.18937683,-0.51783508,-0.48048639,-1.92970264,3.16670656,1.13355756,-0.07890664,1.31536257,-0.43924797,-0.04562932,-0.87974954,0.75411212,-2.39745235,-3.97132111,0.37202546,-2.40399146,-1.50796390,-3.08302689,0.23075986,-0.94316757,1.34948587,0.58591264,2.18529797,7.97652435,2.32798409,-4.09404373,0.89634895,0.77697754,-0.65091681,-7.05506849,5.86194515,2.51394033,4.69959354,0.20835471,3.18049693,-1.29682434,3.70832396,-0.48123091,-1.67904007,-1.35418940,1.58435583,-1.13851106,-1.19225955,0.59713769,-5.80462933,-7.45143986,-1.08658695,1.03244078,-1.75307107,-7.07100582,3.85825157,1.62127817,2.32572675,0.56171900,-0.80591971,3.98835945,0.15742642,-2.97832179,0.13821673,-0.72556758,-0.84936106,-7.28444147,3.94134307,0.80779338,7.47784615,8.23335075,4.80595016,-4.89574575,4.03362942,-6.67522192,-4.55204487,2.12511182,-2.70781207,-1.57226098,-3.08408356,-0.30812448,-5.32870674,-5.13238287,0.49605465,-0.55042171,0.46324944,-3.83545256,-0.12562510,-0.20978995,-0.13068712,-1.92144060,-1.68787408,5.45581436,-0.79583496,-2.38866687,-3.90546346,-0.47028148,-0.14319679,-3.37016582,2.00905991,-1.21345615,1.81376505,7.73004007,0.74310112,-4.64536428,3.78111577,-9.05182457,-0.10674095,1.53476238,0.63345337,-0.40907967,-1.44729769,-1.87145400,-2.46623540,1.07472968,0.77390999,-3.93438888,4.49174690,-0.96686655,1.92278123,0.30049133,-0.02388665,-1.99777114,-3.23885751,5.87784004,2.13776040,3.56758308,-3.37774134,-3.67526293,1.63700044,-1.69959962,-0.99112594,6.03103638,1.67399430,-1.28699589,7.16759014,12.63490295,3.62937450,-4.75982571,2.17861104,-2.03065681,4.30207729,-0.46797156,-2.96022511,-6.02702332,3.09229851,-1.39771092,-0.03471333,3.22175527,5.63565636,1.78195477,-0.63545251,-3.99497652,1.46043062,4.60050488,-2.96651959,-2.03159475,-1.52386189,-0.15129802,-3.90390921,-0.63852370,0.79210538,2.35288715,-5.55609035,5.36427498,-0.60248077,-0.26181316,5.04884720,8.53192806,5.05080223,-6.56371737,1.52260923,-7.13623667,6.49414349,2.33445597,-4.11490965,-6.44347477,-0.47079402,-0.63467920,2.60399365,1.05958164,3.66901422,-1.05657935,1.88611507,-6.37475634,2.01480770,3.36020517,-5.11001921,-0.46132171,2.16525555,4.21938848,-2.08346295,2.86168146,1.26987600,6.76066971,-7.84916353,4.11700916,0.47985530,-4.60113716,7.42062473,6.37472820,4.37820530,-7.12197018,0.01357239,-7.90392113,8.32131577,-0.87593079,-0.16994858,-5.86345863,-0.20697471,-1.37845206,1.63819647,1.59720242,-0.74357712,-1.88725603,-1.98357940,-8.57950306,-4.10104513,3.57231879,-2.89855957,-0.11263305,2.78033924,1.53078973,-2.93089223,0.73189604,3.20563078,3.92601013,-5.21916151,0.89163935,-0.42978728,-6.70888853,4.56477976,1.20105875,3.83393812,-6.27205181,4.05993128,-7.35513067,1.60660768,-1.21052051,1.58191252,-1.37899971,-1.20117283,2.93301678,1.06302834,1.38993621,-1.66884089,-3.34452581,1.04498529,-4.10412455,-4.03310585,1.61513603,-1.09388447,2.11451387,-0.94192362,-0.23287666,5.88265705,-0.83010495,-2.15317154,-0.60276151,-1.49265075,3.93397975,5.45194483,1.45161700,-2.57401872,-5.59288931,4.29170895,1.87151814,0.08362055,-0.28767288,1.17675185,0.85266006,1.30549634,-5.60830832,0.19398519,-0.83982587,1.75940764,-5.46077394,1.64495635,0.17102760,-0.54459631,-2.21975255,-0.37443402,-2.08474159,1.85959935,11.19680309,-0.18611598,-2.59765387,3.06330776,-1.52183700,-4.88415241,-0.75097847,2.58201051,7.40885210,3.58994508,1.62457407,3.12514591,-4.36833286,1.39830995,3.61003447,-0.63837433,-3.62661815,3.78898096,2.92802262,5.87374496,-4.38554621,-2.53411579,-2.87311554,-1.31391978,-4.26736879,3.45099425,1.58769250,1.73341393,-1.08842182,2.27120280,-1.78938174,-2.29940319,7.07046986,0.51426595,-6.22928905,5.28968811,2.31827855,-4.20915890,-1.27249205,5.92120600,3.19458675,7.09252501,3.96577907,6.41484213,-4.66009521,10.00181389,0.51108456,-4.62243366,-5.18351841,2.12961674,5.10694027,7.29412317,0.15912467,-3.38902974,-4.01918602,-2.17383957,0.13118666,0.27872476,-0.92317247,3.51440644,1.84171486,1.03378081,1.30569839,-2.09583759,9.03952980,-0.55187917,-2.04549074,1.08294606,-2.65263700,-2.93977118,1.88909876,0.96043622,1.76579499,3.14314699,5.86394691,7.36944389,-7.04524136,6.68673229,-5.52591467,-2.19745898,-4.32036924,0.52971321,2.26268244,6.91575766,-0.94590527,-3.98923349,-0.12266219,0.24294075,-1.07783222,1.87989080,-3.57109427,1.61553633,0.42486978,0.75852054,-6.19481468,-3.80570698,2.39946675,-1.93851781,-5.42234039,-6.34092760,-2.52374983,-1.85044456,3.92693520,0.40042299,4.69742584,5.40483189,-1.02398944,8.89605045,0.64680403,0.89943957,0.76993859,-1.88244629,1.90714884,3.10836840,-0.17064989,0.84892416,-6.94988108,1.92141032,-1.36458397,6.39284658,0.45201308,2.58823442,6.33375788,-4.76916075,-8.45738983,-0.48962492,2.40652561,4.56602001,-3.34420681,1.86862195,-7.01420689,-6.94657421,-2.47419310,-4.61693668,-0.18822384,-0.36949772,2.01374269,4.11018658,-5.11564064,8.04294395,2.88567662,-2.87645102,-1.23238611,-5.91409397,-0.62205851,1.38689423,-0.01120412,5.25955677,-1.98474956,-3.72012186,3.00445986,4.99141550,2.97457719,2.70827627,6.04544449,-0.20756161,-10.87035751,0.80454814,0.33568168,-2.48132324,-2.84452009,2.63126230,-3.99351716,-7.39294338,3.62798953,-8.65815926,2.65992808,-6.98126554,3.09881067,0.67735767,-1.15946686,5.63180256,-0.17694545,-8.59651184,3.75297594,-2.35913754,-0.20330384,5.49958467,1.00861740,1.42849684,0.00062013,-0.11073381,2.15207863,4.07368469,1.14344299,-1.27953362,6.64699316,-0.73672432,-8.55606937,-0.19439441,-4.14319754,-4.69964647,-5.86446047,2.87106085,-3.42714882,-5.00668287,6.22464132,-7.72335291,4.05667686,-5.72637177,6.35073948,-1.29593158,0.00813985,3.63368607,-1.05764008,-7.88486052,3.73919106,1.41835213,-1.04935634,0.65119827,0.03547254,1.88996327,1.58701086,-0.56215239,-0.80187100,4.55604362,-0.67249978,1.41084409,7.86281586,-2.38301182,-8.50535774,-3.82098866,-2.40856767,-5.33439016,-3.34747362,2.69389009,-1.64118791,4.52447939,0.04468334,-1.48768258,-0.69848812,-0.71123981,3.66259432,6.10314512,1.37305343,-0.62758982,-2.99383426,4.20510864,1.48497128,-0.08954811,2.43872309,-0.59880185,0.37431365,2.45458341,-3.28401661,-1.94629693,-1.93975246,-0.26385683,-0.45814323,-0.18108580,-3.74811840,-0.29739976,-2.24116230,-0.28150487,-2.24421668,3.46930790,8.35415077,0.05562943,-2.81079793,1.10388446,-2.82245207,-2.98102283,-1.08132946,1.19089699,8.00183105,6.35385323,3.72591257,4.59467506,-5.74890900,4.42238331,-3.36533451,0.18350232,3.05606651,1.18788099,2.87450886,0.27472210,-2.80111074,-0.66314960,-1.96376896,0.75167024,-4.72056293,1.10629988,-5.00775242,1.48246133,-3.91681528,-1.86573625,-6.17714882,-0.67820001,5.69730282,1.04399037,-4.93794823,3.09619617,2.18692017,-5.54232264,-3.10046840,-0.68972743,2.81824327,3.04334164,6.13203907,4.14081764,1.02573645,5.71970081,-6.01574707,-2.07346702,0.99554527,1.69641590,0.66776669,-0.80132431,-2.03513098,-3.42513680,-0.06704485,-1.87195873,-5.42428589,-0.20748445,-1.52408111,0.97084987,-0.48799962,-0.45379883,-0.26652339,-1.20720732,3.94169855,-3.18480229,-1.87440264,-1.18028760,0.52011997,-2.13437462,-4.52583313,1.69722807,-0.89371562,3.37972403,6.38838720,6.98663378,-4.05421400,6.89512825,-5.09085655,-2.16257906,-3.33272719,-3.01246452,0.37613097,1.80455804,-0.36456174,-5.32273912,-1.29978943,-0.53685790,-2.12896323,2.55506587,-2.57999182,3.40891910,1.36033249,0.83864629,-2.88629293,-7.36048365,5.61314154,1.32668555,-2.58041072,-3.71943092,1.60647738,-2.74816346,2.47269106,0.85507953,8.39183426,3.42624784,-0.01519036,5.68412066,2.51771593,1.03045523,-2.08733034,-2.44337177,0.81668580,1.30275154,2.99679208,-2.91957355,-1.71337795,3.34979844,1.51825011,5.20375061,2.27888370,1.38787699,4.23474550,-4.05878592,-4.85074377,-0.22794735,4.64402294,1.24391258,-2.04935098,1.26285601,-7.51862240,0.62138438,-1.95792389,-0.96587181,0.85141110,0.79354531,7.93766356,6.07677746,2.05947518,6.55480623,1.44032848,-0.70615625,-0.07896036,-5.08359432,-0.01047915,-1.89632201,2.57555676,3.83779287,0.42850614,1.80754125,-0.06942326,6.35997963,6.06101418,-0.97032297,5.71477222,-6.06671238,-3.46607208,-4.98306370,2.84659123,-2.11025190,-0.04609144,5.26831341,-9.56940651,-3.67193556,-1.71143103,-1.35221267,-4.26226807,-6.89146233,8.21761799,5.69823503,2.28137946,1.88911343,-1.44562483,-1.60295713,-0.52568185,-3.31892347,-2.81997776,0.35287106,2.98202395,-1.39432132,-2.70001364,-4.14169264,3.50194883,4.12610435,5.52755260,2.65859175,3.61353087,-0.83027136,-5.10652542,-4.48625374,2.06585884,-2.76383352,-0.64300913,8.19686604,0.96106279,2.45952058,2.47275925,-1.03288829,-0.64897656,-3.77937531,4.27940083,2.58320260,-0.57665241,1.87247813,-3.81604433,-0.24543774,-1.62118483,-0.73075479,-0.48533297,2.05016756,0.45561486,0.03316188,0.77791005,-1.56283605,2.36616826,5.58082104,-1.30925488,-1.06329608,2.17189479,-3.43008828,-4.71520567,-2.56184673,0.17508316,-3.25817418,-0.41749167,0.18119079,-0.73181152,3.99792433,-3.08002281,-0.99143314,-1.83520067,1.18565679,2.98040128,5.67814350,2.35128760,1.41600966,4.02718067,-0.08193968,0.64636409,1.35931289,2.37125754,1.75978124,3.90977740,1.50662971,-2.84089065,1.29824126,-3.38730979,-1.61005294,0.58292413,-0.03019404,-1.57986510,-0.56102908,-3.03128719,0.51644313,-2.01147819,0.98400700,3.00028515,0.74579155,-3.37098312,0.93339360,-1.29018497,-2.14695001,1.30411184,0.71501279,7.47793055,4.06516457,3.50772929,3.52762985,0.55643129,0.32272506,-4.30955982,2.49414706,2.07820845,-0.34377906,4.39805031,2.77561307,-3.91292810,2.43981409,0.18861845,-2.76658440,-4.97148752,3.25273705,-0.08929539,0.19818619,-5.83767605,-0.97381884,-5.68745661,-5.42433214,3.98769903,-0.40394354,-1.83387578,-0.80109525,1.47454357,-3.14899540,0.80130816,-2.26348829,4.06121159,6.13077354,5.31226397,2.94966197,-3.65217376,-1.08136678,-7.14119816,-0.85269439,-0.70365787,-0.81598872,3.62807679,3.08123684,-7.82739496,4.07951784,-0.14204243,-0.66969109,-5.07225513,2.88492823,0.47202343,0.72683257,-6.84280777,0.41807127,-5.09785986,-3.74514675,2.03936672,-1.06096244,-1.52409148,-0.97046643,2.27491093,-1.55597985,-1.29215479,-0.79737484,-0.01979581,7.65407991,5.54527044,4.04147148,-2.64274883,-1.89246953,-3.89547634,-1.06029689,-2.85982800,-1.41247237,1.55836034,3.38194537,-2.97655582,0.87510300,1.26282072,-1.77029657,-3.57144690,-4.19456863,0.53179169,-1.42221975,-3.09144497,-0.84294832,-5.02758694,-2.68011904,0.89156240,-0.34783912,4.64484835,-2.34453487,-1.28573155,0.09990287,0.01828218,-1.79960847,-1.06579173,1.08763921,0.43687880,3.24747229,3.83097172,1.07253766,-1.33810723,0.76530832,1.58660865,5.60743904,-3.54124737,-0.89264417,-3.83942485,-1.03707337,-1.61659896,1.65349591,1.72698796,4.96013832,0.78927267,-0.35563886,-3.48121166,3.79677629,2.59023166,2.74940348,-2.17589283,-5.91757107,2.43766379,-4.15906048,-1.74731481,-2.49113035,-0.57349741,-4.04455185,-1.46939647,2.21418452,0.09153593,2.23016739,7.91880608,4.04464149,0.07706618,-2.41892862,-2.19280314,7.61760712,-5.89153862,0.33551922,-1.70855618,-0.30561331,-0.14341974,-2.48878574,1.31269515,3.45388412,-0.02453184,-0.12132037,-4.27916241,1.25179088,4.09455204,-1.83801770,-1.86743176,-4.02864933,3.44515228,-4.39244986,-0.56988084,-1.69426417,2.18254852,-4.78135824,1.73193693,-2.27968478,-1.49523509,2.51696730,4.03677559,-2.03679037,1.32167840,-2.22570705,-2.74843621,6.29655170,-3.67230225,-1.86765468,-0.14842367,-1.21552539,-0.92038238,-0.51692355,1.08433771,-0.01929832,0.15660909,2.31432915,-3.86507082,-0.69797570,0.13505173,-1.50951028,-0.69980979,-1.51297045,3.63725281,0.13388813,2.73131752,-0.96528149,4.92000961,-5.92699385,1.69444644,-1.17121375,-2.33710480,1.35302818,1.39608085,1.68293881,0.94960749,1.89011908,-4.08865070,0.13722643,-1.62849212,-0.19044125,1.37906075,-3.92504406,-1.45033538,-0.42085981,3.38237071,-3.06508875,-1.39420545,1.13067436,0.92206454,0.49917889,-2.74508023,-2.19221997,1.77914095,0.10854459,-2.62178278,2.35042715,-0.15322030,-0.67014873,-1.75627899,2.64074945,2.76339936,2.67275214,-0.62736398,0.58251178,-4.64895678,5.50419283,2.53566456,-2.44196153,-0.07845879,-2.80389643,-0.64810950,-0.05813205,1.67155504,-2.69673729,-1.72486305,-0.53888649,1.86805439,-1.37128329,-5.37923479,-2.08133769,0.58187997,-1.39498150,0.21874082,4.33726025,6.29673958,0.72312093,-3.32683516,1.73482585,-0.00766110,-2.63785434,-0.13511759,4.07195950,0.94139838,3.15717316,1.53720927,1.87664819,-2.33655119,6.18176556,-2.73912525,-2.45279956,2.20392370,-0.56854641,0.98915887,-2.64472580,2.40633702,-4.93327999,-1.28942823,0.98247659,1.31774998,0.07669818,-5.91169453,-0.43135011,1.27404964,-0.59787154,-0.22716975,0.74409103,10.27316475,-2.29192710,-2.19403267,3.78925133,3.19553399,-4.42490482,-0.80781460,2.16568565,-2.54165983,2.54885101,4.18779039,1.73079813,-1.48891807,11.60153770,-0.98686743,-2.88813901,2.32898521,-0.36101711,2.34522438,0.29057693,1.39800644,-4.31848240,-3.21217132,0.11740226,-1.21613467,0.57248503,-4.44853830,1.54665899,3.14459944,1.76809108,0.26693153,0.86913753,9.47121620,-2.07677889,2.08578467,1.30181742,1.58683562,-3.52757788,-1.32763624,0.79821301,-2.19358301,1.17707348,6.01983643,4.11209440,-2.04209709,7.00413418,-1.84904683,-1.32542288,-0.01298118,0.70377320,0.27815005,2.07879829,-0.71606725,-4.94399881,-2.11898828,-0.39051518,-2.21034360,3.05337906,-1.56889665,1.97065282,2.61320901,-0.34063196,-0.57001418,-2.13183641,3.48879004,-0.12067288,0.48568326,-1.81424558,2.28868723,1.44802380,1.25918829,-1.76415455,5.35742331,3.50682044,4.71371317,5.89110756,8.51241302,4.07391453,-0.05887252,-0.18202400,2.27119660,6.78274727,-2.87470293,-5.14336634,0.76443815,2.04625130,-0.43199503,-1.01353514,2.42951298,2.35641170,0.32345510,-4.04195738,-4.77967072,0.26564783,6.11455107,-2.53868008,-3.11839914,-1.04203856,5.17195654,-4.15338612,-3.84149241,0.48130888,3.09706950,-4.18423653,5.26233864,3.55831861,3.75122595,8.14969349,6.80038738,4.68907356,-1.40135396,-3.19287133,-3.15895939,8.77363205,-4.48793411,-3.80537176,-2.40145254,-2.74341679,-2.02862644,5.33402443,9.25365734,2.50246119,0.32847846,-1.50564361,-4.26163197,-1.40994716,2.50708485,0.44500345,-0.62516934,4.09846306,5.29355669,-4.02224922,0.73442125,0.46648952,0.67028689,-6.30715466,6.56297970,3.80854273,-5.19078207,4.98839283,7.59161472,0.46010983,-2.10227895,0.29324162,-2.67019558,4.57838106,-3.02338457,-3.08647728,-2.00112700,-3.81710315,-0.08346784,1.69288683,5.68807268,3.29351830,0.54618967,1.83540761,-5.38810253,0.51326782,4.40081882,-4.03805828,0.49482727,-1.36024392,2.91845679,-2.00959015,2.47489738,-1.43354976,1.92024410,-6.55897284,1.79488957,-0.89570928,-6.13094234,-0.45504010,2.35239482,1.29039919,-4.78849840,-1.52545333,-6.50420475,2.99257326,-0.55620033,0.26807702,-2.52090979,-4.59419632,0.57965040,2.19423151,2.04760551,-0.57048106,-2.20812702,-0.04777686,1.38053393,-2.71448946,-1.06219673,-3.62008905,1.85719645,1.28355026,-2.76315832,1.65295160,-4.01645803,-3.10454416,-0.65713316,1.22384977,-0.70416176,4.45064926,1.31602776,2.06907344,2.48872757,4.25775290,3.50504255,-0.68262041,1.29799378,-1.01969171,2.98593879,0.12607655,0.37219539,-0.84196299,-3.80019331,-1.82315290,-0.38489276,-1.45200360,-4.00882292,0.61042011,-0.16738498,1.33787775,-2.26938057,1.03656030,8.89089870,-1.60370600,-5.38691807,5.72182989,2.72854710,-6.18535757,-3.13408709,2.79175353,5.18425512,9.46434212,2.40110517,1.11330092,-3.57366538,4.80967665,0.40691876,-3.65484858,0.92398167,2.53852940,3.17747331,2.14199781,-1.69107199,-1.91864693,-3.18452644,-2.42408276,-2.14332366,-1.35526609,-4.50732136,0.58234072,-1.81547785,0.57311213,1.10584176,-0.97226644,11.73174381,-2.00559855,-1.81175601,2.33131361,0.49264961,-0.42245382,-1.37528467,1.55768061,0.21152198,13.08896351,10.33674145,5.77929306,-6.19886398,5.67007637,-6.61288071,-2.58029866,-4.05192375,1.77221894,0.29821560,5.23508501,-5.09560966,-0.97536200,-5.17957878,1.02876794,-4.52072096,2.22126532,-4.81708670,0.44538212,-2.30738068,3.15900373,-4.99227905,0.82632786,9.65415478,-0.63819492,-3.25479436,-0.13276935,0.21337092,-2.22116399,-3.04922724,0.65568435,-0.10706246,4.58047390,7.80782652,5.49080181,-3.97114491,6.43327618,-6.54772758,-2.10962629,-0.79831678,-0.08316499,2.48658133,4.14070511,-0.59806836,-4.58636141,-0.31166920,0.31757897,-3.92562199,0.65357721,0.55871534,1.71843934,1.62395024,0.00695819,-4.56716251,-3.76420808,4.24979544,-0.86128616,0.23126510,-6.32968998,1.83346081,3.81335950,2.98407745,-1.80454743,6.61764765,-1.39372075,-0.86780751,7.24317265,2.24205112,1.05702817,0.55431479,-1.54557061,3.36389136,4.70898724,1.11327887,-3.78462076,-3.63381767,2.86510396,0.74203897,0.81488025,3.54250598,3.24824381,3.19000244,-0.58995843,-7.05670738,3.18306041,3.95191574,0.81820154,-1.91068232,-2.05426741,-1.05589008,-3.18377590,-1.86278260,-8.80374908,0.93416154,-4.60517359,8.38999462,5.26356745,-8.89992714,8.95298958,4.22590351,1.00351548,-6.90151119,-8.07641125,-4.82450199,8.02293015,4.11661243,0.95457208,-7.07843113,-4.30524826,5.02697992,5.21011686,0.80132771,3.23420191,3.82452774,-2.13171721,-7.88879967,1.31062031,1.90848613,-3.51572514,-3.75684500,3.62577081,-5.76075602,-2.79389215,0.32598805,-4.28981733,4.21048594,-3.84532523,3.19815183,-0.40756655,-2.19974327,6.25655174,3.42396951,-1.88986623,-1.92803884,-2.97344875,-0.09756154,5.24342251,-0.72513700,1.06113195,-1.30720282,4.69107103,0.58984971,2.33985567,1.46385121,3.16576266,6.77769995,-5.92685127,-12.61141014,-2.83663774,4.90253258,-6.32688522,-3.00096869,2.38634992,-7.21459866,-5.89208746,2.84085894,-1.21792030,6.70161343,-4.00450230,5.29881001,-1.45574808,0.77542424,1.38336325,-0.21572059,-3.38088870,2.33249640,0.68824625,-3.68440270,0.33481622,-0.39239681,0.14560902,1.61039007,-3.11967754,2.49372435,2.68783092,-1.17559779,0.95257235,4.35451412,-0.56818569,-7.32110357,-7.58534050,-2.10573673,-3.34446383,-0.32183546,-0.78525496,-1.76974547,5.19060802,-2.11319876,-3.41755080,-0.36864156,1.32680905,0.45004874,6.17223930,-1.60707474,0.46096295,-3.88852644,1.84729624,-0.03412050,0.99224162,-2.05553341,3.47793245,-0.06305170,0.51314175,-2.91650558,-1.78121483,-2.85465693,0.24649808,-2.70376635,0.42334458,-1.13862336,-0.98409218,-0.96593523,2.22128963,0.53402066,3.33979344,8.57430458,2.34217858,-2.40062976,5.81624222,1.13290989,-5.06850052,-4.72865725,1.82859278,6.78569555,8.56885242,2.76462936,0.33891773,-2.81092787,0.79498398,-2.27208567,1.55182552,2.17166376,6.12517643,3.56859684,0.27685475,-1.38408327,-1.03533340,-3.46618199,0.79240030,-3.89390516,-0.55852515,-1.16367757,-0.07008934,-2.20105195,3.81210446,-0.66834474,0.43603873,10.92334938,2.48571420,-6.34997845,4.23135757,0.45045292,-4.13489866,-3.92324209,1.88537407,2.57159734,9.90973091,4.37453461,7.34546280,-2.51120615,11.12575245,-3.23452854,-2.49947500,1.39819741,-3.78950691,2.40617585,5.10036278,-3.55743456,-6.42888737,-2.51929998,-1.90880990,-1.81618094,1.60946512,-4.09737110,1.96408439,-1.90115595,2.44444203,-2.31254292,-4.01332951,8.65541840,-0.58626485,-4.02226830,0.43893200,-3.78272748,-5.46277428,0.01306701,0.61185312,0.24469066,1.30214953,5.87789631,8.75197792,-5.31634712,3.43556309,-5.90755081,0.54375106,-2.48162293,-3.51843548,2.55853295,5.06387186,-2.09662485,-3.00377345,-3.21781397,-0.14537808,-4.65453672,1.92747557,0.41553855,4.09379959,0.83387995,1.50868511,-6.54959488,-8.38881016,5.50689125,-2.88616610,-1.21597648,-0.23817590,1.50816703,-2.26873541,2.29862142,-1.61143053,5.97371244,4.71440220,-0.20635787,8.85926723,0.56064367,-1.04103339,-4.47060108,-2.63824081,3.06782055,-2.07702565,3.38269401,-1.59988797,-3.80122590,2.35341501,2.69095278,3.87612104,1.89984226,0.95496917,3.14841127,-5.84543085,-7.24945450,-2.65708590,2.87417006,0.97556210,-3.75203967,1.55287778,-7.43401051,-1.29005826,-3.40252638,-4.01049423,2.82721639,-1.21479535,8.54563904,7.39749908,-0.61361837,7.60177565,1.65812778,-0.83008504,-3.60961151,-7.69062138,-1.26275063,-4.17071676,5.28448200,4.04685593,-1.18231702,1.15276611,1.58620787,6.75060844,3.29332161,-0.67640316,5.78984785,-3.14913464,-6.41867924,-2.58316016,-2.04366302,2.01089478,-3.81723452,3.63843751,-5.13238430,-3.79432917,4.86581373,-1.06922054,3.95978498,-0.78166616,8.35650539,5.35834265,0.35594034,9.41657066,-0.84108615,-6.54425859,-3.44328952,-6.55536795,-0.08963367,-1.53906262,0.17658240,-0.13108420,-0.44371247,-0.78411150,2.64754868,9.66306782,1.70506203,-0.31588936,4.31715870,-6.16665173,-10.43371868,-3.72962189,4.35245228,-1.75867891,-4.20046234,8.62637043,1.45946813,-3.30153608,0.85179043,-2.66643381,3.01863337,-2.52916121,8.35405540,-0.37298933,-0.89473486,6.88681793,-4.46370125,-7.50776386,3.80255938,-3.55003357,1.43528831,-2.20383263,2.34999895,2.03803205,1.94830751,-1.85976326,0.97718471,5.53710842,-0.80560827,0.23925614,5.98795223,-2.03578377,-7.77835321,-2.79955530,-1.88185954,-2.49112058,-0.76095992,2.71161270,-0.55918610,0.83789903,-1.42063200,-0.61528748,-4.18273115,1.76384258,4.21265936,5.50964785,-0.93324339,3.83215356,1.52210593,-0.91594946,1.31148386,3.20160103,1.24493563,-0.72693497,1.84716725,3.09897518,-1.34605026,-1.17511916,-1.05526352,-1.08590937,-1.41319299,-3.75052118,-2.67095542,-0.76179552,-3.32081509,-1.04692316,-1.30194843,-1.98795474,5.01223469,0.21895903,-1.85535169,3.12362719,0.16198632,-3.86784005,-2.03062248,-0.15415624,8.22020721,4.83055592,4.50315666,4.19443417,0.42727345,-4.67786789,-5.18739986,2.53988838,3.19683266,1.80313504,1.94664574,0.59795094,-4.21626759,0.50492239,-0.41232634,-0.99224532,-3.94929314,1.74060190,-0.92474866,-1.00664830,-6.17397356,-1.33146775,-3.78111315,-4.91876888,2.50303864,-0.34890354,-1.25013232,0.38168997,-1.84135628,-4.46107960,-4.05920792,-2.61709857,0.71046209,9.80566883,6.34086990,2.73394704,-2.03342366,-2.21424174,-5.56514263,-4.74755144,-2.20672894,0.09010231,1.70423889,3.19200158,-6.99027634,1.14216340,0.05824995,-0.76996505,-6.51575899,-0.41109252,0.78229940,1.36170781,-5.65170193,1.12221193,-4.60430050,-4.40174437,4.01805925,0.10774946,-2.77991009,-0.18023163,0.02151692,-1.77023101,-1.86639869,-0.69443607,4.92290831,6.83520412,4.27372265,6.54272366,-7.59249687,-1.40776849,-3.52368808,1.01398587,-3.58802676,-0.35658866,1.14716864,3.75847244,-2.30159235,-0.72130895,-0.24564353,-1.77531350,-3.08677864,-0.73486501,-1.20357263,0.60789430,-3.46990204,-0.20668676,-5.46096087,-5.22016764,0.98259866,1.81012678,3.92534304,-2.94997001,1.65154219,2.27040243,0.99095678,0.09144652,-0.99103236,-1.11210847,0.78181303,2.38706732,2.96695375,-0.17279971,0.31143007,1.35465562,2.03586054,6.19515753,-3.14652419,-2.89027119,-3.26665854,-1.93043876,-0.46601450,1.07655203,1.74946189,4.02148342,0.69275337,0.50094581,-4.07613230,2.98369169,4.24537849,0.49480581,-2.02408123,-2.02068973,6.54505825,-5.19377470,-0.12596917,-0.70204186,-0.98308045,-3.19708824,1.63609934,1.35475993,0.16313422,4.13918924,7.69187021,3.72601676,-1.97790039,-1.16739464,-3.31835508,8.14553452,-1.78718984,1.21505618,-3.84255409,-3.21992350,0.07376552,-0.81223297,3.57002878,1.48521733,-0.45995998,0.30551746,-3.33944130,1.39538884,1.84758544,-0.21494150,-2.27316713,-4.37771225,6.48841667,-5.00251961,-0.45162797,-5.01056004,0.70199943,-4.60057783,-2.22394514,0.07777429,-1.49820781,3.47308421,6.13231564,1.18605387,-4.78924608,-3.49548388,-2.73382568,6.24617863,-2.74291611,-1.03833354,-2.20752788,-2.33219409,1.48633552,1.65796840,4.95045471,2.58479190,-0.90922785,0.71312457,-4.44465590,1.37020862,2.37683725,0.18805164,-3.28422308,-1.64939332,3.64181972,-3.75277281,3.67203593,-0.11204052,2.24140930,-3.90657187,2.56883717,-1.44016707,-2.83842611,-0.29104578,2.17757058,-0.71431804,1.36911654,0.85083604,-1.60110259,-1.97247636,-1.61163378,-0.81236130,-0.38993555,-3.03631902,-0.38213277,0.06394482,3.19348621,0.36771113,1.36763072,2.49159527,-0.39599860,-2.69996762,-0.97561121,-2.97563028,-0.49662948,-0.17564940,-2.79042959,0.72395414,2.07260203,-0.99439794,-2.20248008,-0.07389921,0.65536159,4.73054695,-0.63917702,0.58788192,-3.60156059,6.59609890,3.88419437,-3.38469863,-3.56237841,-2.03295064,0.07279694,3.71804547,0.79928309,-2.13411403,-1.13909864,-0.34193408,-1.00338125,-1.44231665,-5.39835978,-0.45086145,1.16064668,2.58335257,2.10072684,4.64244223,7.10090065,1.01974952,-4.44687223,2.99792576,1.10303724,-1.22736573,-3.91514421,3.07458854,2.18765211,3.34481716,2.46166849,2.99648619,-0.94046807,5.55028200,0.92199719,-0.83934361,-0.72042274,0.84869325,1.46914721,0.85937387,4.77306223,-4.06436539,-2.59847593,2.44828081,0.50484699,-2.71092367,-6.39010477,0.91778028,3.25469685,1.30310678,1.35258150,3.56171441,7.82435083,-2.51527429,-4.24328852,2.36876059,1.94595242,-2.59290171,-6.62389565,3.32567835,2.13659120,4.09299326,3.48293996,2.64965177,-3.19157362,13.37204266,-0.50297594,-4.57448196,3.95582604,-0.69038916,0.10098404,1.18737555,3.65761185,-5.69623756,-2.03357077,1.02868807,-1.38448596,-0.05690211,-8.48874187,0.56755424,1.45485961,0.66273880,0.06495565,1.79539490,8.46864319,-1.22696662,-1.87585378,-0.99768794,2.72801924,-0.66980243,-2.31924677,0.33271110,0.11666083,1.86980045,5.95332909,7.38583708,-2.80956483,6.79227638,-6.78070831,1.21884382,-1.40695429,0.90236962,-1.13695288,0.50760663,1.00955284,-5.39029121,0.24987072,2.24283314,-4.02145576,2.18057394,-3.35627747,1.26061773,1.30342579,0.11311233,-1.11199212,-4.06509686,5.82649660,-1.24059582,5.51652861,-1.90937877,1.10658336,-0.47065550,-2.39167786,-1.95931304,4.12717247,1.15396059,1.26015663,7.97836876,7.33633423,2.27785325,-2.83802366,-2.74850106,0.86126029,6.18781090,-1.43707538,-6.97134876,-3.25486469,-1.95214593,0.91066706,0.89637989,1.06481194,6.25791073,0.81779671,-1.08384395,-3.21191931,2.04216075,4.76030350,-2.37217665,-1.42571259,-6.35876131,4.62536526,-5.40060568,-3.14868999,-1.00587153,1.80662942,-7.03201485,6.08373499,0.99862772,2.21717811,4.06814623,6.02428913,5.33422756,-0.87013257,-2.22477579,-2.51505303,5.82925224,-0.82854009,-4.30698347,-1.75007713,2.08352375,-2.25235629,1.17517352,5.77717733,2.27472878,2.72778273,-1.95411634,-4.52602863,1.13983536,1.16340065,-2.02740526,-3.11290503,-1.94906235,1.54855204,-4.52984142,1.97465122,-1.79415476,4.03510094,-8.45349979,10.87430096,2.19863629,-5.39083815,5.86213875,6.25744534,6.52600002,-4.72149038,-1.75254321,-5.51459169,7.03155518,-2.01889277,-4.58441257,-3.61226106,0.42395937,-0.93263882,2.28703761,2.80611467,2.59498215,0.65989012,-1.51268566,-4.49465561,-4.70453882,5.44696808,-4.37603617,0.46670085,2.82488608,2.18854523,-2.04817152,1.19557285,1.53618634,4.44758606,-7.31593513,7.43966007,-3.55480957,-5.29834652,2.14622784,1.65194583,2.71262598,-4.86145496,0.79726243,-8.88541985,1.19627261,0.79660845,-1.98016644,1.03741014,-3.93128228,1.05535269,2.01378822,-0.46086323,-0.77754641,-1.43942690,0.49809402,-2.27861357,-3.29815221,0.38201320,-3.98481083,4.88261318,-0.44555628,-2.57224536,2.35001850,-2.65835261,-2.43422794,-2.97889376,1.07349825,1.88157082,4.74075413,0.60376728,-0.48894715,-1.15800071,4.68110943,-0.86976886,1.49192941,0.62665290,0.20652676,0.53916287,-1.45706177,0.66133004,1.34405875,-4.27689552,-0.20838106,-5.14266443,-1.29718637,-1.74506426,-0.86022055,-3.57553625,0.46880072,-1.25287139,3.28596354,11.33191013,1.23942876,-3.87616491,7.57880497,-0.22940339,-5.68512678,-1.94969654,5.85449600,3.75705457,4.24395847,1.60086083,2.62553668,-0.93964291,5.84753895,-0.79931092,0.48274064,2.07170033,3.02243996,2.63509989,-0.76043403,-1.64048159,-6.17683458,-3.09974527,-2.12773156,-0.89379883,2.82242465,-1.99981332,-0.08763933,0.01921120,-1.94142103,2.48067307,0.41083777,8.24922180,-1.84516132,-1.39224625,5.03956223,0.49562740,-5.28296328,-0.20005548,3.13672113,0.51187158,7.11563921,6.43059587,3.48430967,-5.37095928,8.03863049,-5.53923941,-2.16421175,-3.77641368,3.29633045,5.04030085,2.25945377,-3.04169011,-2.16198015,-2.49559617,-0.26252726,-6.99201345,2.87374353,-0.12568980,0.23314142,-1.32087135,4.39030552,-0.24638844,-4.37242651,14.09276772,1.23987353,-1.72249663,0.31124914,-2.13725138,-3.74915648,-1.87147236,0.47318631,1.13337576,3.00416899,8.82548523,4.80538750,-5.28486395,5.51870108,-5.15801477,0.95712411,-1.50416136,2.34657240,4.20726633,5.56757259,-3.30645251,-3.39945269,-2.68488026,-2.53525281,-3.15145874,2.74529529,-0.96283442,2.87778258,0.22186530,1.24905694,-7.07941198,-5.45916176,3.46988297,0.92430985,-0.98330998,-2.23672342,-3.03262734,0.73941302,0.98004431,0.83219361,7.17411804,4.27849865,0.14765590,8.61269569,9.04497051,1.53991723,-2.08305025,-4.34939337,0.63786775,2.60098696,0.02432060,-1.48516297,-4.06825686,5.12420368,-0.75312757,1.96927559,4.91575956,3.41533065,3.62557888,-4.35002136,-5.91343403,0.45026422,4.93286371,3.45830250,-4.39032364,-0.51697755,-7.41543341,-3.06703568,1.01196158,2.47106576,5.54014874,-4.65312243,8.61000633,8.25905323,-1.41497111,8.69221878,0.40090930,1.11325574,-1.67089832,-4.01080132,1.07925677,2.68086481,-0.73093414,-1.35081220,-7.85765076,-5.98989439,-0.04651213,4.63693142,2.07757711,-0.22652936,3.45525455,-0.69198442,-10.39761639,-2.02106953,4.77755499,-2.67665577,-1.72481167,4.49634743,-2.55717134,-4.55044937,0.46377492,-3.08933020,3.86891365,-2.79104614,8.36974335,0.86471701,-5.39342690,12.54906940,-0.41536295,-5.29502535,-3.94430566,-5.67391300,-4.65079165,2.22505951,-0.30000746,2.27855444,-4.81604433,-1.73440599,4.68784523,5.00208044,0.18863934,-1.74989462,3.17923450,-1.59773099,-12.59962940,-1.54495025,-0.00576371,1.79913878,-2.43449807,1.49516344,-3.90507102,1.68647158,4.50177765,-5.32286358,3.47539330,-2.90529680,1.61576962,0.83679676,-5.55615807,3.78939056,-4.46644831,-5.95550919,0.37808037,0.51334500,1.74658906,-0.82085419,-0.65387219,3.67790437,0.03758264,-2.42622781,1.83335185,4.73835945,-0.83536482,-0.03993917,3.78230667,-4.81265640,-8.26869011,-1.30363441,-2.09106350,-3.96769738,-1.89037073,0.38682747,0.05434489,5.72213697,0.55685395,-3.47729349,-1.11535001,2.09416127,5.08877802,5.72183466,1.29632664,0.16822398,-2.43180108,3.49967623,2.15753818,-0.26548505,3.24446392,-0.00599277,1.08215356,-0.23225522,-2.40723038,0.18496060,-3.70608735,-0.19918591,-1.64028871,0.80792952,-0.85334057,-2.52314138,-3.12099195,0.17949918,-0.82650864,2.32224989,9.56476116,-0.20134282,-0.48428559,2.86784410,0.07289505,-3.92880869,-2.11887884,0.59164631,6.31267452,7.49149418,2.88749456,2.40504885,-3.57608175,-1.48019314,-0.69410253,0.90275228,-0.34111357,2.19190216,3.39090061,3.39631820,-5.19105434,2.67546582,-2.56549048,-0.59797800,-4.21802664,0.63918972,-0.69969130,0.47496963,-4.30976725,0.16531238,-3.59595251,-0.76877379,11.79971790,-0.93276632,-1.48630571,8.04754066,2.09168458,-3.77018499,-4.19337654,0.26171905,1.99359691,8.96759701,8.39609814,6.19231987,-5.36037970,4.69818354,-4.22453928,-4.61665344,-2.52073431,1.34026706,2.80182385,2.56681514,-4.04676390,-3.01466990,-4.10480118,0.38737059,-0.37146521,-2.26529670,-1.72867084,0.93472683,-2.47562981,0.89871657,-1.67618203,-0.28950238,5.30124855,-0.14731219,-0.81319761,-1.11265934,0.11356127,-2.52802444,-1.93826056,1.06187987,1.48062325,4.28070498,5.69893932,9.26904392,-4.23773003,5.78582096,-6.18445301,-2.85200453,-5.30461454,-4.16009140,-0.07239690,4.11531162,-1.12266588,-1.50265646,0.47661865,-1.90043914,-6.48978710,1.71005368,0.18256521,-0.88272136,-0.51324779,-0.78045660,-5.21036625,-4.11805344,3.99454761,-1.04999924,-6.99629354,-5.02737141,0.94748145,-2.35882139,4.13982439,-1.41835535,7.56763077,3.97024012,-4.08156776,6.90305424,0.53571963,-2.22625160,-2.09144926,-4.98530245,-0.15102190,0.59995949,3.28562784,0.77991986,-3.08389306,3.34046674,0.41394949,5.10031366,2.99692893,0.17706826,2.85998058,-6.68330860,-6.72653008,-0.04071128,3.71085787,3.17834806,-4.88019037,6.74075413,-7.41782188,-5.22026348,-1.94595623,-3.61318684,1.85610664,1.08613706,6.41580677,1.46376514,-4.11524010,9.59146214,-2.92772651,-1.70753336,-1.51594138,-4.88185692,1.47331417,-2.23893595,4.98459148,1.29359996,-2.29221845,-0.99594390,3.05759239,6.86030054,2.40487719,3.28339863,7.72739315,-3.60563445,-9.73502827,-1.51672328,-0.08473521,-2.43673515,-3.26616001,3.63767886,-11.25394535,-5.17597103,-1.27523947,-7.82669783,0.67929745,-4.50530529,5.49323797,6.78993320,-2.28033876,4.61412525,2.55109429,-12.38607693,-0.63024014,-3.45992327,-0.84092742,-0.03252453,4.58635283,5.28213978,-1.28417206,-1.71185923,-0.26850975,8.28257561,4.47432184,2.72818279,8.42217731,-4.22216320,-8.95128918,-1.57179546,1.34253705,-5.47035217,-5.50866985,4.64156532,-6.11207914,-5.46734476,3.54298997,-2.79237103,-0.70766860,-3.62739944,3.22660995,-2.02262759,0.11224222,2.63832402,-0.91955596,-4.65958309,-0.29729855,-1.78957534,-0.40749407,0.51688713,0.83725226,0.30945438,1.20769620,-1.75219965,2.59689760,5.01501608,-1.59034789,0.58155286,3.75831509,-5.26110506,-8.65382767,-6.19066620,-0.61932850,-2.71863723,-0.87443137,3.40582991,-1.27868056,3.51236677,-2.07806540,-0.85076392,-1.14599180,1.16361260,1.86411846,5.86179352,0.69029891,-0.06060839,1.54649436,-0.60351688,1.51970077,0.04187265,1.64540339,2.75502157,2.46308279,1.69071770,-3.23827076,0.92096543,-3.09458661,-1.23823690,0.24035048,-0.74456501,-1.85476089,-0.32914662,-2.10325241,1.19795251,-2.05372071,1.02114081,2.56286955,0.42165697,-1.65826249,4.00724554,-2.18727994,-1.05848944,-0.52338278,-0.28714985,8.08780861,5.04444599,3.51866961,3.37445784,-1.96067202,-1.21509445,-3.96595931,-0.80801201,0.76944816,1.80147493,4.14419460,-0.12201095,-2.77788162,1.13284469,-2.05441403,-0.61129224,-2.69690657,1.91634214,-2.17146754,-0.22308528,-6.02561045,0.49161875,-6.74280357,-4.62689781,2.47910833,1.86534905,-3.24152899,-1.39898300,0.29427958,-2.16338181,0.90073711,1.75551236,4.42651892,8.34437466,5.50070190,5.68162251,1.65345454,-2.72315669,-5.43411493,-0.29380533,1.07508349,-1.73533511,2.56912184,3.62010550,-6.30422783,1.74158525,-1.22070909,-0.80982518,-4.14757967,4.29217434,0.70600843,-2.09282112,-5.09018898,-0.11623126,-5.99775553,-4.66743088,1.61512172,-1.30276895,-3.17103505,-0.26310229,-1.00843918,-0.77664804,-2.05240250,0.04728425,1.15720487,4.01001406,7.24615860,2.55452180,-5.76347876,0.34683830,-6.05540276,-4.70677900,-0.93182588,-4.37759733,2.93209839,1.63947964,-2.43563962,1.35213876,0.00670356,-0.02742785,-2.16460943,1.39449501,0.23929763,2.37476778,-4.17733765,-0.81475425,-6.15027046,-5.74441719,3.53978682,0.66798484}); + auto input2 = NDArrayFactory::create('c', {3, 4, 4, 5}, {0.98114507f, 0.96400015f, 0.58669623f, 0.60073098f, 0.75425418f, 0.44258752f, 0.76373084f, 0.96593234f, 0.34067846f, 0.57962620f, 0.77517051f, 0.97472977f, 0.79237527f, 0.68690428f, 0.21719366f, 0.79959206f, 0.84814187f, 0.22496814f, 0.08646965f, 0.31110474f, 0.79813162f, 0.19661444f, 0.57760099f, 0.72138960f, 0.15244268f, 0.87687051f, 0.11130344f, 0.01087698f, 0.34817841f, 0.54992017f, 0.23443850f, 0.31725614f, 0.59755220f, 0.20364695f, 0.00531392f, 0.23403114f, 0.07442912f, 0.83707647f, 0.89291743f, 0.09044587f, 0.69041462f, 0.29904183f, 0.61904680f, 0.85306847f, 0.34467042f, 0.95839152f, 0.54517124f, 0.29640937f, 0.94855959f, 0.95970016f, 0.94045145f, 0.95510301f, 0.34666505f, 0.34717010f, 0.69245678f, 0.71669175f, 0.59043738f, 0.64924132f, 0.06033522f, 0.60185199f, 0.04690073f, 0.59241154f, 0.40229547f, 0.23002481f, 0.45161195f, 0.73743778f, 0.93209113f, 0.37294358f, 0.50177744f, 0.15072501f, 0.26146917f, 0.05252146f, 0.04758931f, 0.76448288f, 0.85149045f, 0.08840467f, 0.07692576f, 0.33180160f, 0.27241259f, 0.74834620f, 0.56453640f, 0.23057286f, 0.68429752f, 0.11961551f, 0.39045977f, 0.44356094f, 0.77018807f, 0.07984410f, 0.47926806f, 0.26165759f, 0.18606064f, 0.89972877f, 0.17962874f, 0.47273120f, 0.64641705f, 0.61890443f, 0.58730015f, 0.25937832f, 0.35231561f, 0.10243882f, 0.17459193f, 0.95906995f, 0.09227025f, 0.30003223f, 0.41601210f, 0.38269713f, 0.84799751f, 0.59295173f, 0.76277990f, 0.68910424f, 0.37672606f, 0.40675461f, 0.94346058f, 0.91438505f, 0.84728183f, 0.64367667f, 0.74899979f, 0.60570691f, 0.16417363f, 0.68852426f, 0.85486889f, 0.22585792f, 0.86953176f, 0.07465519f, 0.93096301f, 0.38008822f, 0.38752587f, 0.44004038f, 0.13170612f, 0.94541045f, 0.89349973f, 0.69245307f, 0.94978877f, 0.98776658f, 0.79445884f, 0.30607409f, 0.58264961f, 0.37980538f, 0.41810784f, 0.48903038f, 0.51615888f, 0.57682794f, 0.82481897f, 0.78341080f, 0.48446465f, 0.17447931f, 0.71125424f, 0.30263851f, 0.70675352f, 0.03215584f, 0.92381065f, 0.22343694f, 0.08851149f, 0.91402490f, 0.70074717f, 0.30912192f, 0.37723206f, 0.97579397f, 0.23554587f, 0.95939133f, 0.41565709f, 0.01741416f, 0.58362787f, 0.22106662f, 0.89065537f, 0.31900249f, 0.41280911f, 0.67947610f, 0.04545590f, 0.15352812f, 0.85412524f, 0.84933222f, 0.80000225f, 0.93147073f, 0.70094105f, 0.69269875f, 0.95282194f, 0.65913582f, 0.79186874f, 0.59855248f, 0.39707430f, 0.95126239f, 0.15618217f, 0.33446689f, 0.98123758f, 0.84770758f, 0.98081012f, 0.54427413f, 0.18728519f, 0.89792955f, 0.53360126f, 0.72812986f, 0.13307744f, 0.51217443f, 0.66708084f, 0.29416915f, 0.31298995f, 0.39155037f, 0.29288291f, 0.87063305f, 0.61759154f, 0.73723332f, 0.37167635f, 0.82122716f, 0.22937430f, 0.76570536f, 0.47911792f, 0.02826214f, 0.94277323f, 0.59945469f, 0.19042060f, 0.68173155f, 0.82771295f, 0.95649538f, 0.40833101f, 0.90838542f, 0.55245881f, 0.49011012f, 0.36773444f, 0.34513527f, 0.42050683f, 0.16113964f, 0.30969388f, 0.27174174f, 0.12117655f, 0.35270175f, 0.81967867f, 0.63723136f, 0.84309389f, 0.71822576f, 0.84883484f, 0.32306117f, 0.08176457f, 0.56175486f, 0.34892198f, 0.09306929f, 0.85437582f, 0.13925577f, 0.48629188f, 0.29923539f}); + auto exp = NDArrayFactory::create('c', {3, 8, 8, 16}, {5.98743296f, -2.83037376f, -0.87943113f, 1.41339970f, 1.32433391f, -1.20299149f, -0.02893090f, 2.05326009f, 1.19417048f, 5.58212376f, 3.28139353f, 1.19237995f, -1.09431255f, -2.55264497f, 3.11014652f, 6.81296825f, -2.09029293f, -4.32068443f, -0.52808392f, -1.97968531f, -0.18673831f, 0.84605980f, 4.55825520f, 2.71503139f, 0.15210046f, 0.85310984f, -3.82062817f, 2.76470995f, 3.69004202f, -1.45017099f, -2.59361267f, -1.35094655f, 7.24145126f, -5.25432396f, 0.19920218f, -4.30596399f, 1.35318923f, -3.88142037f, 3.67493343f, 2.25931478f, 2.87630725f, 1.66349852f, 6.21347952f, 0.94105923f, -1.61742055f, -2.35699606f, 0.12850338f, 1.79141688f, -2.09535933f, -6.35418081f, -0.06303531f, -4.38615131f, 0.48237842f, 0.26528549f, 3.38231516f, 3.76315165f, -0.40254810f, -0.23716694f, -6.13381910f, -0.41950428f, -0.89680839f, -1.46491277f, -1.98541689f, -0.99357355f, 5.58237648f, -2.38937521f, -0.00872564f, -2.37138414f, 4.91117287f, -4.51916361f, 0.97943687f, 2.91052818f, -2.50362611f, 1.70252812f, 5.04137802f, 3.57108784f, -1.87532270f, -3.66677809f, -2.38861251f, 5.55765152f, -7.27571774f, -1.68887305f, -0.72266489f, -4.42809057f, -0.92118186f, 1.02381468f, 4.44284725f, 5.17150497f, -0.42438728f, 2.02693963f, -1.36484981f, -1.47912180f, 0.26649538f, -0.02091765f, -2.86906910f, -3.03046989f, 1.35122132f, -3.21707630f, 2.21112418f, 0.24121630f, 3.96940088f, -7.66105747f, 2.76352382f, -0.99061489f, -2.16720009f, -1.63170409f, 1.12701774f, -1.02415371f, -0.90435314f, -1.51372027f, -0.76884907f, 0.39066136f, -0.89562428f, -2.03204703f, 1.28074932f, -2.14551091f, -2.36843777f, 0.46580017f, 0.75451565f, -0.00336730f, -1.06597757f, 3.27195978f, -0.41307712f, -0.10376054f, -1.34102952f, -2.22901654f, 2.31929803f, 1.40851438f, -2.23774385f, 0.20417206f, -1.12153268f, -0.13188094f, -3.96649432f, 2.10269976f, 0.49845099f, 6.18937683f, -0.51783508f, -0.48048639f, -1.92970264f, 3.16670656f, 1.13355756f, -0.07890664f, 1.31536257f, -0.43924797f, -0.04562932f, -0.87974954f, 0.75411212f, -2.39745235f, -3.97132111f, 0.37202546f, -2.40399146f, -1.50796390f, -3.08302689f, 0.23075986f, -0.94316757f, 1.34948587f, 0.58591264f, 2.18529797f, 7.97652435f, 2.32798409f, -4.09404373f, 0.89634895f, 0.77697754f, -0.65091681f, -7.05506849f, 5.86194515f, 2.51394033f, 4.69959354f, 0.20835471f, 3.18049693f, -1.29682434f, 3.70832396f, -0.48123091f, -1.67904007f, -1.35418940f, 1.58435583f, -1.13851106f, -1.19225955f, 0.59713769f, -5.80462933f, -7.45143986f, -1.08658695f, 1.03244078f, -1.75307107f, -7.07100582f, 3.85825157f, 1.62127817f, 2.32572675f, 0.56171900f, -0.80591971f, 3.98835945f, 0.15742642f, -2.97832179f, 0.13821673f, -0.72556758f, -0.84936106f, -7.28444147f, 3.94134307f, 0.80779338f, 7.47784615f, 8.23335075f, 4.80595016f, -4.89574575f, 4.03362942f, -6.67522192f, -4.55204487f, 2.12511182f, -2.70781207f, -1.57226098f, -3.08408356f, -0.30812448f, -5.32870674f, -5.13238287f, 0.49605465f, -0.55042171f, 0.46324944f, -3.83545256f, -0.12562510f, -0.20978995f, -0.13068712f, -1.92144060f, -1.68787408f, 5.45581436f, -0.79583496f, -2.38866687f, -3.90546346f, -0.47028148f, -0.14319679f, -3.37016582f, 2.00905991f, -1.21345615f, 1.81376505f, 7.73004007f, 0.74310112f, -4.64536428f, 3.78111577f, -9.05182457f, -0.10674095f, 1.53476238f, 0.63345337f, -0.40907967f, -1.44729769f, -1.87145400f, -2.46623540f, 1.07472968f, 0.77390999f, -3.93438888f, 4.49174690f, -0.96686655f, 1.92278123f, 0.30049133f, -0.02388665f, -1.99777114f, -3.23885751f, 5.87784004f, 2.13776040f, 3.56758308f, -3.37774134f, -3.67526293f, 1.63700044f, -1.69959962f, -0.99112594f, 6.03103638f, 1.67399430f, -1.28699589f, 7.16759014f, 12.63490295f, 3.62937450f, -4.75982571f, 2.17861104f, -2.03065681f, 4.30207729f, -0.46797156f, -2.96022511f, -6.02702332f, 3.09229851f, -1.39771092f, -0.03471333f, 3.22175527f, 5.63565636f, 1.78195477f, -0.63545251f, -3.99497652f, 1.46043062f, 4.60050488f, -2.96651959f, -2.03159475f, -1.52386189f, -0.15129802f, -3.90390921f, -0.63852370f, 0.79210538f, 2.35288715f, -5.55609035f, 5.36427498f, -0.60248077f, -0.26181316f, 5.04884720f, 8.53192806f, 5.05080223f, -6.56371737f, 1.52260923f, -7.13623667f, 6.49414349f, 2.33445597f, -4.11490965f, -6.44347477f, -0.47079402f, -0.63467920f, 2.60399365f, 1.05958164f, 3.66901422f, -1.05657935f, 1.88611507f, -6.37475634f, 2.01480770f, 3.36020517f, -5.11001921f, -0.46132171f, 2.16525555f, 4.21938848f, -2.08346295f, 2.86168146f, 1.26987600f, 6.76066971f, -7.84916353f, 4.11700916f, 0.47985530f, -4.60113716f, 7.42062473f, 6.37472820f, 4.37820530f, -7.12197018f, 0.01357239f, -7.90392113f, 8.32131577f, -0.87593079f, -0.16994858f, -5.86345863f, -0.20697471f, -1.37845206f, 1.63819647f, 1.59720242f, -0.74357712f, -1.88725603f, -1.98357940f, -8.57950306f, -4.10104513f, 3.57231879f, -2.89855957f, -0.11263305f, 2.78033924f, 1.53078973f, -2.93089223f, 0.73189604f, 3.20563078f, 3.92601013f, -5.21916151f, 0.89163935f, -0.42978728f, -6.70888853f, 4.56477976f, 1.20105875f, 3.83393812f, -6.27205181f, 4.05993128f, -7.35513067f, 1.60660768f, -1.21052051f, 1.58191252f, -1.37899971f, -1.20117283f, 2.93301678f, 1.06302834f, 1.38993621f, -1.66884089f, -3.34452581f, 1.04498529f, -4.10412455f, -4.03310585f, 1.61513603f, -1.09388447f, 2.11451387f, -0.94192362f, -0.23287666f, 5.88265705f, -0.83010495f, -2.15317154f, -0.60276151f, -1.49265075f, 3.93397975f, 5.45194483f, 1.45161700f, -2.57401872f, -5.59288931f, 4.29170895f, 1.87151814f, 0.08362055f, -0.28767288f, 1.17675185f, 0.85266006f, 1.30549634f, -5.60830832f, 0.19398519f, -0.83982587f, 1.75940764f, -5.46077394f, 1.64495635f, 0.17102760f, -0.54459631f, -2.21975255f, -0.37443402f, -2.08474159f, 1.85959935f, 11.19680309f, -0.18611598f, -2.59765387f, 3.06330776f, -1.52183700f, -4.88415241f, -0.75097847f, 2.58201051f, 7.40885210f, 3.58994508f, 1.62457407f, 3.12514591f, -4.36833286f, 1.39830995f, 3.61003447f, -0.63837433f, -3.62661815f, 3.78898096f, 2.92802262f, 5.87374496f, -4.38554621f, -2.53411579f, -2.87311554f, -1.31391978f, -4.26736879f, 3.45099425f, 1.58769250f, 1.73341393f, -1.08842182f, 2.27120280f, -1.78938174f, -2.29940319f, 7.07046986f, 0.51426595f, -6.22928905f, 5.28968811f, 2.31827855f, -4.20915890f, -1.27249205f, 5.92120600f, 3.19458675f, 7.09252501f, 3.96577907f, 6.41484213f, -4.66009521f, 10.00181389f, 0.51108456f, -4.62243366f, -5.18351841f, 2.12961674f, 5.10694027f, 7.29412317f, 0.15912467f, -3.38902974f, -4.01918602f, -2.17383957f, 0.13118666f, 0.27872476f, -0.92317247f, 3.51440644f, 1.84171486f, 1.03378081f, 1.30569839f, -2.09583759f, 9.03952980f, -0.55187917f, -2.04549074f, 1.08294606f, -2.65263700f, -2.93977118f, 1.88909876f, 0.96043622f, 1.76579499f, 3.14314699f, 5.86394691f, 7.36944389f, -7.04524136f, 6.68673229f, -5.52591467f, -2.19745898f, -4.32036924f, 0.52971321f, 2.26268244f, 6.91575766f, -0.94590527f, -3.98923349f, -0.12266219f, 0.24294075f, -1.07783222f, 1.87989080f, -3.57109427f, 1.61553633f, 0.42486978f, 0.75852054f, -6.19481468f, -3.80570698f, 2.39946675f, -1.93851781f, -5.42234039f, -6.34092760f, -2.52374983f, -1.85044456f, 3.92693520f, 0.40042299f, 4.69742584f, 5.40483189f, -1.02398944f, 8.89605045f, 0.64680403f, 0.89943957f, 0.76993859f, -1.88244629f, 1.90714884f, 3.10836840f, -0.17064989f, 0.84892416f, -6.94988108f, 1.92141032f, -1.36458397f, 6.39284658f, 0.45201308f, 2.58823442f, 6.33375788f, -4.76916075f, -8.45738983f, -0.48962492f, 2.40652561f, 4.56602001f, -3.34420681f, 1.86862195f, -7.01420689f, -6.94657421f, -2.47419310f, -4.61693668f, -0.18822384f, -0.36949772f, 2.01374269f, 4.11018658f, -5.11564064f, 8.04294395f, 2.88567662f, -2.87645102f, -1.23238611f, -5.91409397f, -0.62205851f, 1.38689423f, -0.01120412f, 5.25955677f, -1.98474956f, -3.72012186f, 3.00445986f, 4.99141550f, 2.97457719f, 2.70827627f, 6.04544449f, -0.20756161f, -10.87035751f, 0.80454814f, 0.33568168f, -2.48132324f, -2.84452009f, 2.63126230f, -3.99351716f, -7.39294338f, 3.62798953f, -8.65815926f, 2.65992808f, -6.98126554f, 3.09881067f, 0.67735767f, -1.15946686f, 5.63180256f, -0.17694545f, -8.59651184f, 3.75297594f, -2.35913754f, -0.20330384f, 5.49958467f, 1.00861740f, 1.42849684f, 0.00062013f, -0.11073381f, 2.15207863f, 4.07368469f, 1.14344299f, -1.27953362f, 6.64699316f, -0.73672432f, -8.55606937f, -0.19439441f, -4.14319754f, -4.69964647f, -5.86446047f, 2.87106085f, -3.42714882f, -5.00668287f, 6.22464132f, -7.72335291f, 4.05667686f, -5.72637177f, 6.35073948f, -1.29593158f, 0.00813985f, 3.63368607f, -1.05764008f, -7.88486052f, 3.73919106f, 1.41835213f, -1.04935634f, 0.65119827f, 0.03547254f, 1.88996327f, 1.58701086f, -0.56215239f, -0.80187100f, 4.55604362f, -0.67249978f, 1.41084409f, 7.86281586f, -2.38301182f, -8.50535774f, -3.82098866f, -2.40856767f, -5.33439016f, -3.34747362f, 2.69389009f, -1.64118791f, 4.52447939f, 0.04468334f, -1.48768258f, -0.69848812f, -0.71123981f, 3.66259432f, 6.10314512f, 1.37305343f, -0.62758982f, -2.99383426f, 4.20510864f, 1.48497128f, -0.08954811f, 2.43872309f, -0.59880185f, 0.37431365f, 2.45458341f, -3.28401661f, -1.94629693f, -1.93975246f, -0.26385683f, -0.45814323f, -0.18108580f, -3.74811840f, -0.29739976f, -2.24116230f, -0.28150487f, -2.24421668f, 3.46930790f, 8.35415077f, 0.05562943f, -2.81079793f, 1.10388446f, -2.82245207f, -2.98102283f, -1.08132946f, 1.19089699f, 8.00183105f, 6.35385323f, 3.72591257f, 4.59467506f, -5.74890900f, 4.42238331f, -3.36533451f, 0.18350232f, 3.05606651f, 1.18788099f, 2.87450886f, 0.27472210f, -2.80111074f, -0.66314960f, -1.96376896f, 0.75167024f, -4.72056293f, 1.10629988f, -5.00775242f, 1.48246133f, -3.91681528f, -1.86573625f, -6.17714882f, -0.67820001f, 5.69730282f, 1.04399037f, -4.93794823f, 3.09619617f, 2.18692017f, -5.54232264f, -3.10046840f, -0.68972743f, 2.81824327f, 3.04334164f, 6.13203907f, 4.14081764f, 1.02573645f, 5.71970081f, -6.01574707f, -2.07346702f, 0.99554527f, 1.69641590f, 0.66776669f, -0.80132431f, -2.03513098f, -3.42513680f, -0.06704485f, -1.87195873f, -5.42428589f, -0.20748445f, -1.52408111f, 0.97084987f, -0.48799962f, -0.45379883f, -0.26652339f, -1.20720732f, 3.94169855f, -3.18480229f, -1.87440264f, -1.18028760f, 0.52011997f, -2.13437462f, -4.52583313f, 1.69722807f, -0.89371562f, 3.37972403f, 6.38838720f, 6.98663378f, -4.05421400f, 6.89512825f, -5.09085655f, -2.16257906f, -3.33272719f, -3.01246452f, 0.37613097f, 1.80455804f, -0.36456174f, -5.32273912f, -1.29978943f, -0.53685790f, -2.12896323f, 2.55506587f, -2.57999182f, 3.40891910f, 1.36033249f, 0.83864629f, -2.88629293f, -7.36048365f, 5.61314154f, 1.32668555f, -2.58041072f, -3.71943092f, 1.60647738f, -2.74816346f, 2.47269106f, 0.85507953f, 8.39183426f, 3.42624784f, -0.01519036f, 5.68412066f, 2.51771593f, 1.03045523f, -2.08733034f, -2.44337177f, 0.81668580f, 1.30275154f, 2.99679208f, -2.91957355f, -1.71337795f, 3.34979844f, 1.51825011f, 5.20375061f, 2.27888370f, 1.38787699f, 4.23474550f, -4.05878592f, -4.85074377f, -0.22794735f, 4.64402294f, 1.24391258f, -2.04935098f, 1.26285601f, -7.51862240f, 0.62138438f, -1.95792389f, -0.96587181f, 0.85141110f, 0.79354531f, 7.93766356f, 6.07677746f, 2.05947518f, 6.55480623f, 1.44032848f, -0.70615625f, -0.07896036f, -5.08359432f, -0.01047915f, -1.89632201f, 2.57555676f, 3.83779287f, 0.42850614f, 1.80754125f, -0.06942326f, 6.35997963f, 6.06101418f, -0.97032297f, 5.71477222f, -6.06671238f, -3.46607208f, -4.98306370f, 2.84659123f, -2.11025190f, -0.04609144f, 5.26831341f, -9.56940651f, -3.67193556f, -1.71143103f, -1.35221267f, -4.26226807f, -6.89146233f, 8.21761799f, 5.69823503f, 2.28137946f, 1.88911343f, -1.44562483f, -1.60295713f, -0.52568185f, -3.31892347f, -2.81997776f, 0.35287106f, 2.98202395f, -1.39432132f, -2.70001364f, -4.14169264f, 3.50194883f, 4.12610435f, 5.52755260f, 2.65859175f, 3.61353087f, -0.83027136f, -5.10652542f, -4.48625374f, 2.06585884f, -2.76383352f, -0.64300913f, 8.19686604f, 0.96106279f, 2.45952058f, 2.47275925f, -1.03288829f, -0.64897656f, -3.77937531f, 4.27940083f, 2.58320260f, -0.57665241f, 1.87247813f, -3.81604433f, -0.24543774f, -1.62118483f, -0.73075479f, -0.48533297f, 2.05016756f, 0.45561486f, 0.03316188f, 0.77791005f, -1.56283605f, 2.36616826f, 5.58082104f, -1.30925488f, -1.06329608f, 2.17189479f, -3.43008828f, -4.71520567f, -2.56184673f, 0.17508316f, -3.25817418f, -0.41749167f, 0.18119079f, -0.73181152f, 3.99792433f, -3.08002281f, -0.99143314f, -1.83520067f, 1.18565679f, 2.98040128f, 5.67814350f, 2.35128760f, 1.41600966f, 4.02718067f, -0.08193968f, 0.64636409f, 1.35931289f, 2.37125754f, 1.75978124f, 3.90977740f, 1.50662971f, -2.84089065f, 1.29824126f, -3.38730979f, -1.61005294f, 0.58292413f, -0.03019404f, -1.57986510f, -0.56102908f, -3.03128719f, 0.51644313f, -2.01147819f, 0.98400700f, 3.00028515f, 0.74579155f, -3.37098312f, 0.93339360f, -1.29018497f, -2.14695001f, 1.30411184f, 0.71501279f, 7.47793055f, 4.06516457f, 3.50772929f, 3.52762985f, 0.55643129f, 0.32272506f, -4.30955982f, 2.49414706f, 2.07820845f, -0.34377906f, 4.39805031f, 2.77561307f, -3.91292810f, 2.43981409f, 0.18861845f, -2.76658440f, -4.97148752f, 3.25273705f, -0.08929539f, 0.19818619f, -5.83767605f, -0.97381884f, -5.68745661f, -5.42433214f, 3.98769903f, -0.40394354f, -1.83387578f, -0.80109525f, 1.47454357f, -3.14899540f, 0.80130816f, -2.26348829f, 4.06121159f, 6.13077354f, 5.31226397f, 2.94966197f, -3.65217376f, -1.08136678f, -7.14119816f, -0.85269439f, -0.70365787f, -0.81598872f, 3.62807679f, 3.08123684f, -7.82739496f, 4.07951784f, -0.14204243f, -0.66969109f, -5.07225513f, 2.88492823f, 0.47202343f, 0.72683257f, -6.84280777f, 0.41807127f, -5.09785986f, -3.74514675f, 2.03936672f, -1.06096244f, -1.52409148f, -0.97046643f, 2.27491093f, -1.55597985f, -1.29215479f, -0.79737484f, -0.01979581f, 7.65407991f, 5.54527044f, 4.04147148f, -2.64274883f, -1.89246953f, -3.89547634f, -1.06029689f, -2.85982800f, -1.41247237f, 1.55836034f, 3.38194537f, -2.97655582f, 0.87510300f, 1.26282072f, -1.77029657f, -3.57144690f, -4.19456863f, 0.53179169f, -1.42221975f, -3.09144497f, -0.84294832f, -5.02758694f, -2.68011904f, 0.89156240f, -0.34783912f, 4.64484835f, -2.34453487f, -1.28573155f, 0.09990287f, 0.01828218f, -1.79960847f, -1.06579173f, 1.08763921f, 0.43687880f, 3.24747229f, 3.83097172f, 1.07253766f, -1.33810723f, 0.76530832f, 1.58660865f, 5.60743904f, -3.54124737f, -0.89264417f, -3.83942485f, -1.03707337f, -1.61659896f, 1.65349591f, 1.72698796f, 4.96013832f, 0.78927267f, -0.35563886f, -3.48121166f, 3.79677629f, 2.59023166f, 2.74940348f, -2.17589283f, -5.91757107f, 2.43766379f, -4.15906048f, -1.74731481f, -2.49113035f, -0.57349741f, -4.04455185f, -1.46939647f, 2.21418452f, 0.09153593f, 2.23016739f, 7.91880608f, 4.04464149f, 0.07706618f, -2.41892862f, -2.19280314f, 7.61760712f, -5.89153862f, 0.33551922f, -1.70855618f, -0.30561331f, -0.14341974f, -2.48878574f, 1.31269515f, 3.45388412f, -0.02453184f, -0.12132037f, -4.27916241f, 1.25179088f, 4.09455204f, -1.83801770f, -1.86743176f, -4.02864933f, 3.44515228f, -4.39244986f, -0.56988084f, -1.69426417f, 2.18254852f, -4.78135824f, 1.73193693f, -2.27968478f, -1.49523509f, 2.51696730f, 4.03677559f, -2.03679037f, 1.32167840f, -2.22570705f, -2.74843621f, 6.29655170f, -3.67230225f, -1.86765468f, -0.14842367f, -1.21552539f, -0.92038238f, -0.51692355f, 1.08433771f, -0.01929832f, 0.15660909f, 2.31432915f, -3.86507082f, -0.69797570f, 0.13505173f, -1.50951028f, -0.69980979f, -1.51297045f, 3.63725281f, 0.13388813f, 2.73131752f, -0.96528149f, 4.92000961f, -5.92699385f, 1.69444644f, -1.17121375f, -2.33710480f, 1.35302818f, 1.39608085f, 1.68293881f, 0.94960749f, 1.89011908f, -4.08865070f, 0.13722643f, -1.62849212f, -0.19044125f, 1.37906075f, -3.92504406f, -1.45033538f, -0.42085981f, 3.38237071f, -3.06508875f, -1.39420545f, 1.13067436f, 0.92206454f, 0.49917889f, -2.74508023f, -2.19221997f, 1.77914095f, 0.10854459f, -2.62178278f, 2.35042715f, -0.15322030f, -0.67014873f, -1.75627899f, 2.64074945f, 2.76339936f, 2.67275214f, -0.62736398f, 0.58251178f, -4.64895678f, 5.50419283f, 2.53566456f, -2.44196153f, -0.07845879f, -2.80389643f, -0.64810950f, -0.05813205f, 1.67155504f, -2.69673729f, -1.72486305f, -0.53888649f, 1.86805439f, -1.37128329f, -5.37923479f, -2.08133769f, 0.58187997f, -1.39498150f, 0.21874082f, 4.33726025f, 6.29673958f, 0.72312093f, -3.32683516f, 1.73482585f, -0.00766110f, -2.63785434f, -0.13511759f, 4.07195950f, 0.94139838f, 3.15717316f, 1.53720927f, 1.87664819f, -2.33655119f, 6.18176556f, -2.73912525f, -2.45279956f, 2.20392370f, -0.56854641f, 0.98915887f, -2.64472580f, 2.40633702f, -4.93327999f, -1.28942823f, 0.98247659f, 1.31774998f, 0.07669818f, -5.91169453f, -0.43135011f, 1.27404964f, -0.59787154f, -0.22716975f, 0.74409103f, 10.27316475f, -2.29192710f, -2.19403267f, 3.78925133f, 3.19553399f, -4.42490482f, -0.80781460f, 2.16568565f, -2.54165983f, 2.54885101f, 4.18779039f, 1.73079813f, -1.48891807f, 11.60153770f, -0.98686743f, -2.88813901f, 2.32898521f, -0.36101711f, 2.34522438f, 0.29057693f, 1.39800644f, -4.31848240f, -3.21217132f, 0.11740226f, -1.21613467f, 0.57248503f, -4.44853830f, 1.54665899f, 3.14459944f, 1.76809108f, 0.26693153f, 0.86913753f, 9.47121620f, -2.07677889f, 2.08578467f, 1.30181742f, 1.58683562f, -3.52757788f, -1.32763624f, 0.79821301f, -2.19358301f, 1.17707348f, 6.01983643f, 4.11209440f, -2.04209709f, 7.00413418f, -1.84904683f, -1.32542288f, -0.01298118f, 0.70377320f, 0.27815005f, 2.07879829f, -0.71606725f, -4.94399881f, -2.11898828f, -0.39051518f, -2.21034360f, 3.05337906f, -1.56889665f, 1.97065282f, 2.61320901f, -0.34063196f, -0.57001418f, -2.13183641f, 3.48879004f, -0.12067288f, 0.48568326f, -1.81424558f, 2.28868723f, 1.44802380f, 1.25918829f, -1.76415455f, 5.35742331f, 3.50682044f, 4.71371317f, 5.89110756f, 8.51241302f, 4.07391453f, -0.05887252f, -0.18202400f, 2.27119660f, 6.78274727f, -2.87470293f, -5.14336634f, 0.76443815f, 2.04625130f, -0.43199503f, -1.01353514f, 2.42951298f, 2.35641170f, 0.32345510f, -4.04195738f, -4.77967072f, 0.26564783f, 6.11455107f, -2.53868008f, -3.11839914f, -1.04203856f, 5.17195654f, -4.15338612f, -3.84149241f, 0.48130888f, 3.09706950f, -4.18423653f, 5.26233864f, 3.55831861f, 3.75122595f, 8.14969349f, 6.80038738f, 4.68907356f, -1.40135396f, -3.19287133f, -3.15895939f, 8.77363205f, -4.48793411f, -3.80537176f, -2.40145254f, -2.74341679f, -2.02862644f, 5.33402443f, 9.25365734f, 2.50246119f, 0.32847846f, -1.50564361f, -4.26163197f, -1.40994716f, 2.50708485f, 0.44500345f, -0.62516934f, 4.09846306f, 5.29355669f, -4.02224922f, 0.73442125f, 0.46648952f, 0.67028689f, -6.30715466f, 6.56297970f, 3.80854273f, -5.19078207f, 4.98839283f, 7.59161472f, 0.46010983f, -2.10227895f, 0.29324162f, -2.67019558f, 4.57838106f, -3.02338457f, -3.08647728f, -2.00112700f, -3.81710315f, -0.08346784f, 1.69288683f, 5.68807268f, 3.29351830f, 0.54618967f, 1.83540761f, -5.38810253f, 0.51326782f, 4.40081882f, -4.03805828f, 0.49482727f, -1.36024392f, 2.91845679f, -2.00959015f, 2.47489738f, -1.43354976f, 1.92024410f, -6.55897284f, 1.79488957f, -0.89570928f, -6.13094234f, -0.45504010f, 2.35239482f, 1.29039919f, -4.78849840f, -1.52545333f, -6.50420475f, 2.99257326f, -0.55620033f, 0.26807702f, -2.52090979f, -4.59419632f, 0.57965040f, 2.19423151f, 2.04760551f, -0.57048106f, -2.20812702f, -0.04777686f, 1.38053393f, -2.71448946f, -1.06219673f, -3.62008905f, 1.85719645f, 1.28355026f, -2.76315832f, 1.65295160f, -4.01645803f, -3.10454416f, -0.65713316f, 1.22384977f, -0.70416176f, 4.45064926f, 1.31602776f, 2.06907344f, 2.48872757f, 4.25775290f, 3.50504255f, -0.68262041f, 1.29799378f, -1.01969171f, 2.98593879f, 0.12607655f, 0.37219539f, -0.84196299f, -3.80019331f, -1.82315290f, -0.38489276f, -1.45200360f, -4.00882292f, 0.61042011f, -0.16738498f, 1.33787775f, -2.26938057f, 1.03656030f, 8.89089870f, -1.60370600f, -5.38691807f, 5.72182989f, 2.72854710f, -6.18535757f, -3.13408709f, 2.79175353f, 5.18425512f, 9.46434212f, 2.40110517f, 1.11330092f, -3.57366538f, 4.80967665f, 0.40691876f, -3.65484858f, 0.92398167f, 2.53852940f, 3.17747331f, 2.14199781f, -1.69107199f, -1.91864693f, -3.18452644f, -2.42408276f, -2.14332366f, -1.35526609f, -4.50732136f, 0.58234072f, -1.81547785f, 0.57311213f, 1.10584176f, -0.97226644f, 11.73174381f, -2.00559855f, -1.81175601f, 2.33131361f, 0.49264961f, -0.42245382f, -1.37528467f, 1.55768061f, 0.21152198f, 13.08896351f, 10.33674145f, 5.77929306f, -6.19886398f, 5.67007637f, -6.61288071f, -2.58029866f, -4.05192375f, 1.77221894f, 0.29821560f, 5.23508501f, -5.09560966f, -0.97536200f, -5.17957878f, 1.02876794f, -4.52072096f, 2.22126532f, -4.81708670f, 0.44538212f, -2.30738068f, 3.15900373f, -4.99227905f, 0.82632786f, 9.65415478f, -0.63819492f, -3.25479436f, -0.13276935f, 0.21337092f, -2.22116399f, -3.04922724f, 0.65568435f, -0.10706246f, 4.58047390f, 7.80782652f, 5.49080181f, -3.97114491f, 6.43327618f, -6.54772758f, -2.10962629f, -0.79831678f, -0.08316499f, 2.48658133f, 4.14070511f, -0.59806836f, -4.58636141f, -0.31166920f, 0.31757897f, -3.92562199f, 0.65357721f, 0.55871534f, 1.71843934f, 1.62395024f, 0.00695819f, -4.56716251f, -3.76420808f, 4.24979544f, -0.86128616f, 0.23126510f, -6.32968998f, 1.83346081f, 3.81335950f, 2.98407745f, -1.80454743f, 6.61764765f, -1.39372075f, -0.86780751f, 7.24317265f, 2.24205112f, 1.05702817f, 0.55431479f, -1.54557061f, 3.36389136f, 4.70898724f, 1.11327887f, -3.78462076f, -3.63381767f, 2.86510396f, 0.74203897f, 0.81488025f, 3.54250598f, 3.24824381f, 3.19000244f, -0.58995843f, -7.05670738f, 3.18306041f, 3.95191574f, 0.81820154f, -1.91068232f, -2.05426741f, -1.05589008f, -3.18377590f, -1.86278260f, -8.80374908f, 0.93416154f, -4.60517359f, 8.38999462f, 5.26356745f, -8.89992714f, 8.95298958f, 4.22590351f, 1.00351548f, -6.90151119f, -8.07641125f, -4.82450199f, 8.02293015f, 4.11661243f, 0.95457208f, -7.07843113f, -4.30524826f, 5.02697992f, 5.21011686f, 0.80132771f, 3.23420191f, 3.82452774f, -2.13171721f, -7.88879967f, 1.31062031f, 1.90848613f, -3.51572514f, -3.75684500f, 3.62577081f, -5.76075602f, -2.79389215f, 0.32598805f, -4.28981733f, 4.21048594f, -3.84532523f, 3.19815183f, -0.40756655f, -2.19974327f, 6.25655174f, 3.42396951f, -1.88986623f, -1.92803884f, -2.97344875f, -0.09756154f, 5.24342251f, -0.72513700f, 1.06113195f, -1.30720282f, 4.69107103f, 0.58984971f, 2.33985567f, 1.46385121f, 3.16576266f, 6.77769995f, -5.92685127f, -12.61141014f, -2.83663774f, 4.90253258f, -6.32688522f, -3.00096869f, 2.38634992f, -7.21459866f, -5.89208746f, 2.84085894f, -1.21792030f, 6.70161343f, -4.00450230f, 5.29881001f, -1.45574808f, 0.77542424f, 1.38336325f, -0.21572059f, -3.38088870f, 2.33249640f, 0.68824625f, -3.68440270f, 0.33481622f, -0.39239681f, 0.14560902f, 1.61039007f, -3.11967754f, 2.49372435f, 2.68783092f, -1.17559779f, 0.95257235f, 4.35451412f, -0.56818569f, -7.32110357f, -7.58534050f, -2.10573673f, -3.34446383f, -0.32183546f, -0.78525496f, -1.76974547f, 5.19060802f, -2.11319876f, -3.41755080f, -0.36864156f, 1.32680905f, 0.45004874f, 6.17223930f, -1.60707474f, 0.46096295f, -3.88852644f, 1.84729624f, -0.03412050f, 0.99224162f, -2.05553341f, 3.47793245f, -0.06305170f, 0.51314175f, -2.91650558f, -1.78121483f, -2.85465693f, 0.24649808f, -2.70376635f, 0.42334458f, -1.13862336f, -0.98409218f, -0.96593523f, 2.22128963f, 0.53402066f, 3.33979344f, 8.57430458f, 2.34217858f, -2.40062976f, 5.81624222f, 1.13290989f, -5.06850052f, -4.72865725f, 1.82859278f, 6.78569555f, 8.56885242f, 2.76462936f, 0.33891773f, -2.81092787f, 0.79498398f, -2.27208567f, 1.55182552f, 2.17166376f, 6.12517643f, 3.56859684f, 0.27685475f, -1.38408327f, -1.03533340f, -3.46618199f, 0.79240030f, -3.89390516f, -0.55852515f, -1.16367757f, -0.07008934f, -2.20105195f, 3.81210446f, -0.66834474f, 0.43603873f, 10.92334938f, 2.48571420f, -6.34997845f, 4.23135757f, 0.45045292f, -4.13489866f, -3.92324209f, 1.88537407f, 2.57159734f, 9.90973091f, 4.37453461f, 7.34546280f, -2.51120615f, 11.12575245f, -3.23452854f, -2.49947500f, 1.39819741f, -3.78950691f, 2.40617585f, 5.10036278f, -3.55743456f, -6.42888737f, -2.51929998f, -1.90880990f, -1.81618094f, 1.60946512f, -4.09737110f, 1.96408439f, -1.90115595f, 2.44444203f, -2.31254292f, -4.01332951f, 8.65541840f, -0.58626485f, -4.02226830f, 0.43893200f, -3.78272748f, -5.46277428f, 0.01306701f, 0.61185312f, 0.24469066f, 1.30214953f, 5.87789631f, 8.75197792f, -5.31634712f, 3.43556309f, -5.90755081f, 0.54375106f, -2.48162293f, -3.51843548f, 2.55853295f, 5.06387186f, -2.09662485f, -3.00377345f, -3.21781397f, -0.14537808f, -4.65453672f, 1.92747557f, 0.41553855f, 4.09379959f, 0.83387995f, 1.50868511f, -6.54959488f, -8.38881016f, 5.50689125f, -2.88616610f, -1.21597648f, -0.23817590f, 1.50816703f, -2.26873541f, 2.29862142f, -1.61143053f, 5.97371244f, 4.71440220f, -0.20635787f, 8.85926723f, 0.56064367f, -1.04103339f, -4.47060108f, -2.63824081f, 3.06782055f, -2.07702565f, 3.38269401f, -1.59988797f, -3.80122590f, 2.35341501f, 2.69095278f, 3.87612104f, 1.89984226f, 0.95496917f, 3.14841127f, -5.84543085f, -7.24945450f, -2.65708590f, 2.87417006f, 0.97556210f, -3.75203967f, 1.55287778f, -7.43401051f, -1.29005826f, -3.40252638f, -4.01049423f, 2.82721639f, -1.21479535f, 8.54563904f, 7.39749908f, -0.61361837f, 7.60177565f, 1.65812778f, -0.83008504f, -3.60961151f, -7.69062138f, -1.26275063f, -4.17071676f, 5.28448200f, 4.04685593f, -1.18231702f, 1.15276611f, 1.58620787f, 6.75060844f, 3.29332161f, -0.67640316f, 5.78984785f, -3.14913464f, -6.41867924f, -2.58316016f, -2.04366302f, 2.01089478f, -3.81723452f, 3.63843751f, -5.13238430f, -3.79432917f, 4.86581373f, -1.06922054f, 3.95978498f, -0.78166616f, 8.35650539f, 5.35834265f, 0.35594034f, 9.41657066f, -0.84108615f, -6.54425859f, -3.44328952f, -6.55536795f, -0.08963367f, -1.53906262f, 0.17658240f, -0.13108420f, -0.44371247f, -0.78411150f, 2.64754868f, 9.66306782f, 1.70506203f, -0.31588936f, 4.31715870f, -6.16665173f, -10.43371868f, -3.72962189f, 4.35245228f, -1.75867891f, -4.20046234f, 8.62637043f, 1.45946813f, -3.30153608f, 0.85179043f, -2.66643381f, 3.01863337f, -2.52916121f, 8.35405540f, -0.37298933f, -0.89473486f, 6.88681793f, -4.46370125f, -7.50776386f, 3.80255938f, -3.55003357f, 1.43528831f, -2.20383263f, 2.34999895f, 2.03803205f, 1.94830751f, -1.85976326f, 0.97718471f, 5.53710842f, -0.80560827f, 0.23925614f, 5.98795223f, -2.03578377f, -7.77835321f, -2.79955530f, -1.88185954f, -2.49112058f, -0.76095992f, 2.71161270f, -0.55918610f, 0.83789903f, -1.42063200f, -0.61528748f, -4.18273115f, 1.76384258f, 4.21265936f, 5.50964785f, -0.93324339f, 3.83215356f, 1.52210593f, -0.91594946f, 1.31148386f, 3.20160103f, 1.24493563f, -0.72693497f, 1.84716725f, 3.09897518f, -1.34605026f, -1.17511916f, -1.05526352f, -1.08590937f, -1.41319299f, -3.75052118f, -2.67095542f, -0.76179552f, -3.32081509f, -1.04692316f, -1.30194843f, -1.98795474f, 5.01223469f, 0.21895903f, -1.85535169f, 3.12362719f, 0.16198632f, -3.86784005f, -2.03062248f, -0.15415624f, 8.22020721f, 4.83055592f, 4.50315666f, 4.19443417f, 0.42727345f, -4.67786789f, -5.18739986f, 2.53988838f, 3.19683266f, 1.80313504f, 1.94664574f, 0.59795094f, -4.21626759f, 0.50492239f, -0.41232634f, -0.99224532f, -3.94929314f, 1.74060190f, -0.92474866f, -1.00664830f, -6.17397356f, -1.33146775f, -3.78111315f, -4.91876888f, 2.50303864f, -0.34890354f, -1.25013232f, 0.38168997f, -1.84135628f, -4.46107960f, -4.05920792f, -2.61709857f, 0.71046209f, 9.80566883f, 6.34086990f, 2.73394704f, -2.03342366f, -2.21424174f, -5.56514263f, -4.74755144f, -2.20672894f, 0.09010231f, 1.70423889f, 3.19200158f, -6.99027634f, 1.14216340f, 0.05824995f, -0.76996505f, -6.51575899f, -0.41109252f, 0.78229940f, 1.36170781f, -5.65170193f, 1.12221193f, -4.60430050f, -4.40174437f, 4.01805925f, 0.10774946f, -2.77991009f, -0.18023163f, 0.02151692f, -1.77023101f, -1.86639869f, -0.69443607f, 4.92290831f, 6.83520412f, 4.27372265f, 6.54272366f, -7.59249687f, -1.40776849f, -3.52368808f, 1.01398587f, -3.58802676f, -0.35658866f, 1.14716864f, 3.75847244f, -2.30159235f, -0.72130895f, -0.24564353f, -1.77531350f, -3.08677864f, -0.73486501f, -1.20357263f, 0.60789430f, -3.46990204f, -0.20668676f, -5.46096087f, -5.22016764f, 0.98259866f, 1.81012678f, 3.92534304f, -2.94997001f, 1.65154219f, 2.27040243f, 0.99095678f, 0.09144652f, -0.99103236f, -1.11210847f, 0.78181303f, 2.38706732f, 2.96695375f, -0.17279971f, 0.31143007f, 1.35465562f, 2.03586054f, 6.19515753f, -3.14652419f, -2.89027119f, -3.26665854f, -1.93043876f, -0.46601450f, 1.07655203f, 1.74946189f, 4.02148342f, 0.69275337f, 0.50094581f, -4.07613230f, 2.98369169f, 4.24537849f, 0.49480581f, -2.02408123f, -2.02068973f, 6.54505825f, -5.19377470f, -0.12596917f, -0.70204186f, -0.98308045f, -3.19708824f, 1.63609934f, 1.35475993f, 0.16313422f, 4.13918924f, 7.69187021f, 3.72601676f, -1.97790039f, -1.16739464f, -3.31835508f, 8.14553452f, -1.78718984f, 1.21505618f, -3.84255409f, -3.21992350f, 0.07376552f, -0.81223297f, 3.57002878f, 1.48521733f, -0.45995998f, 0.30551746f, -3.33944130f, 1.39538884f, 1.84758544f, -0.21494150f, -2.27316713f, -4.37771225f, 6.48841667f, -5.00251961f, -0.45162797f, -5.01056004f, 0.70199943f, -4.60057783f, -2.22394514f, 0.07777429f, -1.49820781f, 3.47308421f, 6.13231564f, 1.18605387f, -4.78924608f, -3.49548388f, -2.73382568f, 6.24617863f, -2.74291611f, -1.03833354f, -2.20752788f, -2.33219409f, 1.48633552f, 1.65796840f, 4.95045471f, 2.58479190f, -0.90922785f, 0.71312457f, -4.44465590f, 1.37020862f, 2.37683725f, 0.18805164f, -3.28422308f, -1.64939332f, 3.64181972f, -3.75277281f, 3.67203593f, -0.11204052f, 2.24140930f, -3.90657187f, 2.56883717f, -1.44016707f, -2.83842611f, -0.29104578f, 2.17757058f, -0.71431804f, 1.36911654f, 0.85083604f, -1.60110259f, -1.97247636f, -1.61163378f, -0.81236130f, -0.38993555f, -3.03631902f, -0.38213277f, 0.06394482f, 3.19348621f, 0.36771113f, 1.36763072f, 2.49159527f, -0.39599860f, -2.69996762f, -0.97561121f, -2.97563028f, -0.49662948f, -0.17564940f, -2.79042959f, 0.72395414f, 2.07260203f, -0.99439794f, -2.20248008f, -0.07389921f, 0.65536159f, 4.73054695f, -0.63917702f, 0.58788192f, -3.60156059f, 6.59609890f, 3.88419437f, -3.38469863f, -3.56237841f, -2.03295064f, 0.07279694f, 3.71804547f, 0.79928309f, -2.13411403f, -1.13909864f, -0.34193408f, -1.00338125f, -1.44231665f, -5.39835978f, -0.45086145f, 1.16064668f, 2.58335257f, 2.10072684f, 4.64244223f, 7.10090065f, 1.01974952f, -4.44687223f, 2.99792576f, 1.10303724f, -1.22736573f, -3.91514421f, 3.07458854f, 2.18765211f, 3.34481716f, 2.46166849f, 2.99648619f, -0.94046807f, 5.55028200f, 0.92199719f, -0.83934361f, -0.72042274f, 0.84869325f, 1.46914721f, 0.85937387f, 4.77306223f, -4.06436539f, -2.59847593f, 2.44828081f, 0.50484699f, -2.71092367f, -6.39010477f, 0.91778028f, 3.25469685f, 1.30310678f, 1.35258150f, 3.56171441f, 7.82435083f, -2.51527429f, -4.24328852f, 2.36876059f, 1.94595242f, -2.59290171f, -6.62389565f, 3.32567835f, 2.13659120f, 4.09299326f, 3.48293996f, 2.64965177f, -3.19157362f, 13.37204266f, -0.50297594f, -4.57448196f, 3.95582604f, -0.69038916f, 0.10098404f, 1.18737555f, 3.65761185f, -5.69623756f, -2.03357077f, 1.02868807f, -1.38448596f, -0.05690211f, -8.48874187f, 0.56755424f, 1.45485961f, 0.66273880f, 0.06495565f, 1.79539490f, 8.46864319f, -1.22696662f, -1.87585378f, -0.99768794f, 2.72801924f, -0.66980243f, -2.31924677f, 0.33271110f, 0.11666083f, 1.86980045f, 5.95332909f, 7.38583708f, -2.80956483f, 6.79227638f, -6.78070831f, 1.21884382f, -1.40695429f, 0.90236962f, -1.13695288f, 0.50760663f, 1.00955284f, -5.39029121f, 0.24987072f, 2.24283314f, -4.02145576f, 2.18057394f, -3.35627747f, 1.26061773f, 1.30342579f, 0.11311233f, -1.11199212f, -4.06509686f, 5.82649660f, -1.24059582f, 5.51652861f, -1.90937877f, 1.10658336f, -0.47065550f, -2.39167786f, -1.95931304f, 4.12717247f, 1.15396059f, 1.26015663f, 7.97836876f, 7.33633423f, 2.27785325f, -2.83802366f, -2.74850106f, 0.86126029f, 6.18781090f, -1.43707538f, -6.97134876f, -3.25486469f, -1.95214593f, 0.91066706f, 0.89637989f, 1.06481194f, 6.25791073f, 0.81779671f, -1.08384395f, -3.21191931f, 2.04216075f, 4.76030350f, -2.37217665f, -1.42571259f, -6.35876131f, 4.62536526f, -5.40060568f, -3.14868999f, -1.00587153f, 1.80662942f, -7.03201485f, 6.08373499f, 0.99862772f, 2.21717811f, 4.06814623f, 6.02428913f, 5.33422756f, -0.87013257f, -2.22477579f, -2.51505303f, 5.82925224f, -0.82854009f, -4.30698347f, -1.75007713f, 2.08352375f, -2.25235629f, 1.17517352f, 5.77717733f, 2.27472878f, 2.72778273f, -1.95411634f, -4.52602863f, 1.13983536f, 1.16340065f, -2.02740526f, -3.11290503f, -1.94906235f, 1.54855204f, -4.52984142f, 1.97465122f, -1.79415476f, 4.03510094f, -8.45349979f, 10.87430096f, 2.19863629f, -5.39083815f, 5.86213875f, 6.25744534f, 6.52600002f, -4.72149038f, -1.75254321f, -5.51459169f, 7.03155518f, -2.01889277f, -4.58441257f, -3.61226106f, 0.42395937f, -0.93263882f, 2.28703761f, 2.80611467f, 2.59498215f, 0.65989012f, -1.51268566f, -4.49465561f, -4.70453882f, 5.44696808f, -4.37603617f, 0.46670085f, 2.82488608f, 2.18854523f, -2.04817152f, 1.19557285f, 1.53618634f, 4.44758606f, -7.31593513f, 7.43966007f, -3.55480957f, -5.29834652f, 2.14622784f, 1.65194583f, 2.71262598f, -4.86145496f, 0.79726243f, -8.88541985f, 1.19627261f, 0.79660845f, -1.98016644f, 1.03741014f, -3.93128228f, 1.05535269f, 2.01378822f, -0.46086323f, -0.77754641f, -1.43942690f, 0.49809402f, -2.27861357f, -3.29815221f, 0.38201320f, -3.98481083f, 4.88261318f, -0.44555628f, -2.57224536f, 2.35001850f, -2.65835261f, -2.43422794f, -2.97889376f, 1.07349825f, 1.88157082f, 4.74075413f, 0.60376728f, -0.48894715f, -1.15800071f, 4.68110943f, -0.86976886f, 1.49192941f, 0.62665290f, 0.20652676f, 0.53916287f, -1.45706177f, 0.66133004f, 1.34405875f, -4.27689552f, -0.20838106f, -5.14266443f, -1.29718637f, -1.74506426f, -0.86022055f, -3.57553625f, 0.46880072f, -1.25287139f, 3.28596354f, 11.33191013f, 1.23942876f, -3.87616491f, 7.57880497f, -0.22940339f, -5.68512678f, -1.94969654f, 5.85449600f, 3.75705457f, 4.24395847f, 1.60086083f, 2.62553668f, -0.93964291f, 5.84753895f, -0.79931092f, 0.48274064f, 2.07170033f, 3.02243996f, 2.63509989f, -0.76043403f, -1.64048159f, -6.17683458f, -3.09974527f, -2.12773156f, -0.89379883f, 2.82242465f, -1.99981332f, -0.08763933f, 0.01921120f, -1.94142103f, 2.48067307f, 0.41083777f, 8.24922180f, -1.84516132f, -1.39224625f, 5.03956223f, 0.49562740f, -5.28296328f, -0.20005548f, 3.13672113f, 0.51187158f, 7.11563921f, 6.43059587f, 3.48430967f, -5.37095928f, 8.03863049f, -5.53923941f, -2.16421175f, -3.77641368f, 3.29633045f, 5.04030085f, 2.25945377f, -3.04169011f, -2.16198015f, -2.49559617f, -0.26252726f, -6.99201345f, 2.87374353f, -0.12568980f, 0.23314142f, -1.32087135f, 4.39030552f, -0.24638844f, -4.37242651f, 14.09276772f, 1.23987353f, -1.72249663f, 0.31124914f, -2.13725138f, -3.74915648f, -1.87147236f, 0.47318631f, 1.13337576f, 3.00416899f, 8.82548523f, 4.80538750f, -5.28486395f, 5.51870108f, -5.15801477f, 0.95712411f, -1.50416136f, 2.34657240f, 4.20726633f, 5.56757259f, -3.30645251f, -3.39945269f, -2.68488026f, -2.53525281f, -3.15145874f, 2.74529529f, -0.96283442f, 2.87778258f, 0.22186530f, 1.24905694f, -7.07941198f, -5.45916176f, 3.46988297f, 0.92430985f, -0.98330998f, -2.23672342f, -3.03262734f, 0.73941302f, 0.98004431f, 0.83219361f, 7.17411804f, 4.27849865f, 0.14765590f, 8.61269569f, 9.04497051f, 1.53991723f, -2.08305025f, -4.34939337f, 0.63786775f, 2.60098696f, 0.02432060f, -1.48516297f, -4.06825686f, 5.12420368f, -0.75312757f, 1.96927559f, 4.91575956f, 3.41533065f, 3.62557888f, -4.35002136f, -5.91343403f, 0.45026422f, 4.93286371f, 3.45830250f, -4.39032364f, -0.51697755f, -7.41543341f, -3.06703568f, 1.01196158f, 2.47106576f, 5.54014874f, -4.65312243f, 8.61000633f, 8.25905323f, -1.41497111f, 8.69221878f, 0.40090930f, 1.11325574f, -1.67089832f, -4.01080132f, 1.07925677f, 2.68086481f, -0.73093414f, -1.35081220f, -7.85765076f, -5.98989439f, -0.04651213f, 4.63693142f, 2.07757711f, -0.22652936f, 3.45525455f, -0.69198442f, -10.39761639f, -2.02106953f, 4.77755499f, -2.67665577f, -1.72481167f, 4.49634743f, -2.55717134f, -4.55044937f, 0.46377492f, -3.08933020f, 3.86891365f, -2.79104614f, 8.36974335f, 0.86471701f, -5.39342690f, 12.54906940f, -0.41536295f, -5.29502535f, -3.94430566f, -5.67391300f, -4.65079165f, 2.22505951f, -0.30000746f, 2.27855444f, -4.81604433f, -1.73440599f, 4.68784523f, 5.00208044f, 0.18863934f, -1.74989462f, 3.17923450f, -1.59773099f, -12.59962940f, -1.54495025f, -0.00576371f, 1.79913878f, -2.43449807f, 1.49516344f, -3.90507102f, 1.68647158f, 4.50177765f, -5.32286358f, 3.47539330f, -2.90529680f, 1.61576962f, 0.83679676f, -5.55615807f, 3.78939056f, -4.46644831f, -5.95550919f, 0.37808037f, 0.51334500f, 1.74658906f, -0.82085419f, -0.65387219f, 3.67790437f, 0.03758264f, -2.42622781f, 1.83335185f, 4.73835945f, -0.83536482f, -0.03993917f, 3.78230667f, -4.81265640f, -8.26869011f, -1.30363441f, -2.09106350f, -3.96769738f, -1.89037073f, 0.38682747f, 0.05434489f, 5.72213697f, 0.55685395f, -3.47729349f, -1.11535001f, 2.09416127f, 5.08877802f, 5.72183466f, 1.29632664f, 0.16822398f, -2.43180108f, 3.49967623f, 2.15753818f, -0.26548505f, 3.24446392f, -0.00599277f, 1.08215356f, -0.23225522f, -2.40723038f, 0.18496060f, -3.70608735f, -0.19918591f, -1.64028871f, 0.80792952f, -0.85334057f, -2.52314138f, -3.12099195f, 0.17949918f, -0.82650864f, 2.32224989f, 9.56476116f, -0.20134282f, -0.48428559f, 2.86784410f, 0.07289505f, -3.92880869f, -2.11887884f, 0.59164631f, 6.31267452f, 7.49149418f, 2.88749456f, 2.40504885f, -3.57608175f, -1.48019314f, -0.69410253f, 0.90275228f, -0.34111357f, 2.19190216f, 3.39090061f, 3.39631820f, -5.19105434f, 2.67546582f, -2.56549048f, -0.59797800f, -4.21802664f, 0.63918972f, -0.69969130f, 0.47496963f, -4.30976725f, 0.16531238f, -3.59595251f, -0.76877379f, 11.79971790f, -0.93276632f, -1.48630571f, 8.04754066f, 2.09168458f, -3.77018499f, -4.19337654f, 0.26171905f, 1.99359691f, 8.96759701f, 8.39609814f, 6.19231987f, -5.36037970f, 4.69818354f, -4.22453928f, -4.61665344f, -2.52073431f, 1.34026706f, 2.80182385f, 2.56681514f, -4.04676390f, -3.01466990f, -4.10480118f, 0.38737059f, -0.37146521f, -2.26529670f, -1.72867084f, 0.93472683f, -2.47562981f, 0.89871657f, -1.67618203f, -0.28950238f, 5.30124855f, -0.14731219f, -0.81319761f, -1.11265934f, 0.11356127f, -2.52802444f, -1.93826056f, 1.06187987f, 1.48062325f, 4.28070498f, 5.69893932f, 9.26904392f, -4.23773003f, 5.78582096f, -6.18445301f, -2.85200453f, -5.30461454f, -4.16009140f, -0.07239690f, 4.11531162f, -1.12266588f, -1.50265646f, 0.47661865f, -1.90043914f, -6.48978710f, 1.71005368f, 0.18256521f, -0.88272136f, -0.51324779f, -0.78045660f, -5.21036625f, -4.11805344f, 3.99454761f, -1.04999924f, -6.99629354f, -5.02737141f, 0.94748145f, -2.35882139f, 4.13982439f, -1.41835535f, 7.56763077f, 3.97024012f, -4.08156776f, 6.90305424f, 0.53571963f, -2.22625160f, -2.09144926f, -4.98530245f, -0.15102190f, 0.59995949f, 3.28562784f, 0.77991986f, -3.08389306f, 3.34046674f, 0.41394949f, 5.10031366f, 2.99692893f, 0.17706826f, 2.85998058f, -6.68330860f, -6.72653008f, -0.04071128f, 3.71085787f, 3.17834806f, -4.88019037f, 6.74075413f, -7.41782188f, -5.22026348f, -1.94595623f, -3.61318684f, 1.85610664f, 1.08613706f, 6.41580677f, 1.46376514f, -4.11524010f, 9.59146214f, -2.92772651f, -1.70753336f, -1.51594138f, -4.88185692f, 1.47331417f, -2.23893595f, 4.98459148f, 1.29359996f, -2.29221845f, -0.99594390f, 3.05759239f, 6.86030054f, 2.40487719f, 3.28339863f, 7.72739315f, -3.60563445f, -9.73502827f, -1.51672328f, -0.08473521f, -2.43673515f, -3.26616001f, 3.63767886f, -11.25394535f, -5.17597103f, -1.27523947f, -7.82669783f, 0.67929745f, -4.50530529f, 5.49323797f, 6.78993320f, -2.28033876f, 4.61412525f, 2.55109429f, -12.38607693f, -0.63024014f, -3.45992327f, -0.84092742f, -0.03252453f, 4.58635283f, 5.28213978f, -1.28417206f, -1.71185923f, -0.26850975f, 8.28257561f, 4.47432184f, 2.72818279f, 8.42217731f, -4.22216320f, -8.95128918f, -1.57179546f, 1.34253705f, -5.47035217f, -5.50866985f, 4.64156532f, -6.11207914f, -5.46734476f, 3.54298997f, -2.79237103f, -0.70766860f, -3.62739944f, 3.22660995f, -2.02262759f, 0.11224222f, 2.63832402f, -0.91955596f, -4.65958309f, -0.29729855f, -1.78957534f, -0.40749407f, 0.51688713f, 0.83725226f, 0.30945438f, 1.20769620f, -1.75219965f, 2.59689760f, 5.01501608f, -1.59034789f, 0.58155286f, 3.75831509f, -5.26110506f, -8.65382767f, -6.19066620f, -0.61932850f, -2.71863723f, -0.87443137f, 3.40582991f, -1.27868056f, 3.51236677f, -2.07806540f, -0.85076392f, -1.14599180f, 1.16361260f, 1.86411846f, 5.86179352f, 0.69029891f, -0.06060839f, 1.54649436f, -0.60351688f, 1.51970077f, 0.04187265f, 1.64540339f, 2.75502157f, 2.46308279f, 1.69071770f, -3.23827076f, 0.92096543f, -3.09458661f, -1.23823690f, 0.24035048f, -0.74456501f, -1.85476089f, -0.32914662f, -2.10325241f, 1.19795251f, -2.05372071f, 1.02114081f, 2.56286955f, 0.42165697f, -1.65826249f, 4.00724554f, -2.18727994f, -1.05848944f, -0.52338278f, -0.28714985f, 8.08780861f, 5.04444599f, 3.51866961f, 3.37445784f, -1.96067202f, -1.21509445f, -3.96595931f, -0.80801201f, 0.76944816f, 1.80147493f, 4.14419460f, -0.12201095f, -2.77788162f, 1.13284469f, -2.05441403f, -0.61129224f, -2.69690657f, 1.91634214f, -2.17146754f, -0.22308528f, -6.02561045f, 0.49161875f, -6.74280357f, -4.62689781f, 2.47910833f, 1.86534905f, -3.24152899f, -1.39898300f, 0.29427958f, -2.16338181f, 0.90073711f, 1.75551236f, 4.42651892f, 8.34437466f, 5.50070190f, 5.68162251f, 1.65345454f, -2.72315669f, -5.43411493f, -0.29380533f, 1.07508349f, -1.73533511f, 2.56912184f, 3.62010550f, -6.30422783f, 1.74158525f, -1.22070909f, -0.80982518f, -4.14757967f, 4.29217434f, 0.70600843f, -2.09282112f, -5.09018898f, -0.11623126f, -5.99775553f, -4.66743088f, 1.61512172f, -1.30276895f, -3.17103505f, -0.26310229f, -1.00843918f, -0.77664804f, -2.05240250f, 0.04728425f, 1.15720487f, 4.01001406f, 7.24615860f, 2.55452180f, -5.76347876f, 0.34683830f, -6.05540276f, -4.70677900f, -0.93182588f, -4.37759733f, 2.93209839f, 1.63947964f, -2.43563962f, 1.35213876f, 0.00670356f, -0.02742785f, -2.16460943f, 1.39449501f, 0.23929763f, 2.37476778f, -4.17733765f, -0.81475425f, -6.15027046f, -5.74441719f, 3.53978682f, 0.66798484f}); nd4j::ops::deconv2d_tf op; auto result = op.execute({&input0, &input1, &input2}, {}, {7,7, 2,2, 0,0, 1,1, 1,1}); @@ -368,10 +368,10 @@ TYPED_TEST(TypedConvolutionTests2, sconv2d_bp_4) { auto bias = NDArrayFactory::create('c', {oC}, {1,2,3,4}); auto gradO = NDArrayFactory::create('c', {bS, oH, oW, oC}); - auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{0.07 , 0.19 , 0.348, 0.652, 0.588, 0.956, 0.387, 0.687, 1.326, 2.022, 1.878, 2.67 , 1.071, 1.515, 2.982, 3.966, 3.534, 4.614, 1.606, 1.982, 3.932, 4.748, 4.428, 5.308, - 1.126, 1.63 , 3.228, 4.3 , 3.468, 4.604, 3.123, 3.999, 7.95 , 9.798, 8.502, 10.446, 3.807, 4.827, 9.606, 11.742,10.158, 12.39 , 4.198, 4.958, 9.884, 11.468,10.38 , 12.028}); + auto expGradI = NDArrayFactory::create('c', {bS, iH, iW, iC},{0.07f, 0.19f, 0.348f, 0.652f, 0.588f, 0.956f, 0.387f, 0.687f, 1.326f, 2.022f, 1.878f, 2.67f, 1.071f, 1.515f, 2.982f, 3.966f, 3.534f, 4.614f, 1.606f, 1.982f, 3.932f, 4.748f, 4.428f, 5.308f, + 1.126f, 1.63f, 3.228f, 4.3f, 3.468f, 4.604f, 3.123f, 3.999f, 7.95f, 9.798f, 8.502f, 10.446f, 3.807f, 4.827f, 9.606f, 11.742f,10.158f, 12.39f, 4.198f, 4.958f, 9.884f, 11.468f,10.38f, 12.028f}); - auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, mC},{19.08, 19.44,19.8 , 20.16,12.24, 12.48,12.72, 12.96,22.56, 23.04,23.52, 24. ,14.4 , 14.72,15.04, 15.36,14.76, 15.12,15.48, 15.84, 9.36, 9.6 , 9.84, 10.08}); + auto expGradW = NDArrayFactory::create('c', {kH, kW, iC, mC},{19.08f, 19.44f, 19.8f, 20.16f, 12.24f, 12.48f, 12.72f, 12.96f, 22.56f, 23.04f, 23.52f, 24.f, 14.4f, 14.72f, 15.04f, 15.36f, 14.76f, 15.12f, 15.48f, 15.84f, 9.36f, 9.6f, 9.84f, 10.08f}); input = 2.; weightsDepth.linspace(0.1, 0.1); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp index 5375f8fca..1d4bf7338 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp @@ -1092,15 +1092,15 @@ TEST_F(DeclarableOpsTests10, NTH_Element_Test_06) { /////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, NTH_Element_Test_7) { - NDArray input = NDArrayFactory::create('c', {2, 3, 4}, {0.7788, 0.8012, 0.7244, 0.2309, - 0.7271, 0.1804, 0.5056, 0.8925, - 0.5461, 0.9234, 0.0856, 0.7938, + NDArray input = NDArrayFactory::create('c', {2, 3, 4}, {0.7788f, 0.8012f, 0.7244f, 0.2309f, + 0.7271f, 0.1804f, 0.5056f, 0.8925f, + 0.5461f, 0.9234f, 0.0856f, 0.7938f, - 0.6591, 0.5555, 0.1596, 0.3087, - 0.1548, 0.4695, 0.9939, 0.6113, - 0.6765, 0.1800, 0.6750, 0.2246}); + 0.6591f, 0.5555f, 0.1596f, 0.3087f, + 0.1548f, 0.4695f, 0.9939f, 0.6113f, + 0.6765f, 0.1800f, 0.6750f, 0.2246f}); NDArray n = NDArrayFactory::create(2); - NDArray exp = NDArrayFactory::create('c', {2,3}, {0.7788, 0.7271, 0.7938, 0.5555, 0.6113, 0.675}); + NDArray exp = NDArrayFactory::create('c', {2,3}, {0.7788f, 0.7271f, 0.7938f, 0.5555f, 0.6113f, 0.675f}); //input.linspace(1.f); @@ -1119,15 +1119,15 @@ TEST_F(DeclarableOpsTests10, NTH_Element_Test_7) { /////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, NTH_Element_Test_8) { - NDArray input = NDArrayFactory::create('c', {2, 3, 4}, {0.7788, 0.8012, 0.7244, 0.2309, - 0.7271, 0.1804, 0.5056, 0.8925, - 0.5461, 0.9234, 0.0856, 0.7938, + NDArray input = NDArrayFactory::create('c', {2, 3, 4}, {0.7788f, 0.8012f, 0.7244f, 0.2309f, + 0.7271f, 0.1804f, 0.5056f, 0.8925f, + 0.5461f, 0.9234f, 0.0856f, 0.7938f, - 0.6591, 0.5555, 0.1596, 0.3087, - 0.1548, 0.4695, 0.9939, 0.6113, - 0.6765, 0.1800, 0.6750, 0.2246}); + 0.6591f, 0.5555f, 0.1596f, 0.3087f, + 0.1548f, 0.4695f, 0.9939f, 0.6113f, + 0.6765f, 0.1800f, 0.6750f, 0.2246f}); NDArray n = NDArrayFactory::create(2); - NDArray exp = NDArrayFactory::create('c', {2,3}, {0.7244, 0.5056, 0.5461, 0.3087, 0.4695, 0.2246}); + NDArray exp = NDArrayFactory::create('c', {2,3}, {0.7244f, 0.5056f, 0.5461f, 0.3087f, 0.4695f, 0.2246f}); //input.linspace(1.f); @@ -1359,10 +1359,10 @@ TEST_F(DeclarableOpsTests10, broadcast_to_test10) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test1) { - NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); + NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); //NDArray paddings('c', {3,2}, {0,0, 0,1, 0,0}); //NDArray expected('c', {2,4,4}, {1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.}); - NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, {1., 2., 3., 4., 2.2, 3.2, 4.2, 5.2, 3.4, 4.4, 5.4, 6.4, + NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, {1., 2., 3., 4., 2.2, 3.2, 4.2, 5.2, 3.4, 4.4, 5.4, 6.4, 4.6, 5.6, 6.6, 7.6, 5.8, 6.8, 7.8, 8.8, 7., 8., 9., 10., 8.2, 9.2, 10.2, 11.2, 9., 10., 11., 12., 9., 10., 11., 12., 9., 10., 11., 12., 3.4, 4.4, 5.4, 6.4, 4.6, 5.6, 6.6, 7.6, @@ -1416,10 +1416,10 @@ TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test2) { - NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); + NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); NDArray size = NDArrayFactory::create({10, 10}); //NDArray expected('c', {2,4,4}, {1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.}); - NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, {1., 2., 3., 4., 2.2, 3.2, 4.2, 5.2, 3.4, 4.4, 5.4, 6.4, + NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, {1., 2., 3., 4., 2.2, 3.2, 4.2, 5.2, 3.4, 4.4, 5.4, 6.4, 4.6, 5.6, 6.6, 7.6, 5.8, 6.8, 7.8, 8.8, 7., 8., 9., 10., 8.2, 9.2, 10.2, 11.2, 9., 10., 11., 12., 9., 10., 11., 12., 9., 10., 11., 12., 3.4, 4.4, 5.4, 6.4, 4.6, 5.6, 6.6, 7.6, @@ -1472,10 +1472,10 @@ TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test2) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test3) { - NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); + NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); //NDArray paddings('c', {3,2}, {0,0, 0,1, 0,0}); //NDArray expected('c', {2,4,4}, {1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.}); - NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, + NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, { 1., 2., 3., 4. , 1.8888888, 2.8888888, 3.8888888, 4.888889, 2.7777777, 3.7777777, 4.7777777, 5.7777777, @@ -1602,9 +1602,9 @@ TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test3) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, ImageResizeBilinear_Test4) { - NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); + NDArray input = NDArrayFactory::create('c', {1, 2,3,4}); NDArray size = NDArrayFactory::create({10, 10}); - NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, + NDArray expected = NDArrayFactory::create('c', {1, 10, 10, 4}, { 1., 2., 3., 4. , 1.8888888, 2.8888888, 3.8888888, 4.888889, 2.7777777, 3.7777777, 4.7777777, 5.7777777, @@ -1750,10 +1750,10 @@ TEST_F(DeclarableOpsTests10, LinSpace_Test1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, ImageResizeNeighbor_Test1) { - NDArray input = NDArrayFactory::create('c', {1, 2, 3, 4}); + NDArray input = NDArrayFactory::create('c', {1, 2, 3, 4}); //NDArray paddings('c', {3,2}, {0,0, 0,1, 0,0}); //NDArray expected('c', {2,4,4}, {1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.}); - NDArray expected = NDArrayFactory::create('c', {1, 4, 5, 4}, { 1, 2, 3, 4, + NDArray expected = NDArrayFactory::create('c', {1, 4, 5, 4}, { 1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8, 5, 6, 7, 8, @@ -1926,8 +1926,8 @@ TEST_F(DeclarableOpsTests10, Image_CropAndResize_1) { //////////////////////////////////////////////////////////////////// TEST_F(DeclarableOpsTests10, Image_CropAndResize_2) { int axis = 0; - NDArray images = NDArrayFactory::create('c', {1,2,2,1}, {1,2,3,4}); - NDArray boxes = NDArrayFactory::create('c', {1,4}, {0,0,1,1}); + NDArray images = NDArrayFactory::create('c', {1,2,2,1}, {1.f, 2.f, 3.f, 4.f}); + NDArray boxes = NDArrayFactory::create('c', {1,4}, {0.f, 0.f, 1.f, 1.f}); NDArray boxI = NDArrayFactory::create('c', {1}, {axis}); NDArray cropSize = NDArrayFactory::create({1, 1}); diff --git a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests8.cpp b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests8.cpp index 9510b3633..82b3d2db7 100644 --- a/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests8.cpp +++ b/libnd4j/tests_cpu/layers_tests/DeclarableOpsTests8.cpp @@ -3238,11 +3238,11 @@ TEST_F(DeclarableOpsTests8, Test_Moments_7) { //////////////////////////////////////////////////////////////////////////////// TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_01) { - auto x = NDArrayFactory::create('c', {1, 1, 2, 5}, { 1, 2., 3, 4, 5, - 6., 7., 8, 9, 10} + auto x = NDArrayFactory::create('c', {1, 1, 2, 5}, { 1.f, 2.f, 3.f, 4.f, 5.f, + 6.f, 7.f, 8.f, 9.f, 10.f} ); - auto exp = NDArrayFactory::create('c', {1, 1, 2, 5}, {0.2581989 , 0.3592106 , 0.40089184, 0.53935987, 0.70014, 0.4898979 , 0.46056613, 0.43971977, 0.5240003 , 0.6375767 }// 0.72760683, 0.4850712, 0.5848977, 0.67488194, + auto exp = NDArrayFactory::create('c', {1, 1, 2, 5}, {0.2581989f, 0.3592106f, 0.40089184f, 0.53935987f, 0.70014f, 0.4898979f, 0.46056613f, 0.43971977f, 0.5240003f, 0.6375767f}// 0.72760683, 0.4850712, 0.5848977, 0.67488194, // 0.7581754, 0.58321184, 0.86747235, 0.4048204} ); @@ -3262,10 +3262,10 @@ TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_01) { //////////////////////////////////////////////////////////////////////////////// TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_02) { - auto x = NDArrayFactory::create('c', {1, 1, 1, 6}, { 1, 2., 3, 4, 5, 6}); + auto x = NDArrayFactory::create('c', {1, 1, 1, 6}, { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f}); auto exp = NDArrayFactory::create('c', {1, 1, 1, 6}, { - 0.2581989 , 0.3592106 , 0.40089184, 0.4193139, 0.5360563, 0.67936623} + 0.2581989f, 0.3592106f, 0.40089184f, 0.4193139f, 0.5360563f, 0.67936623f} ); nd4j::ops::lrn op; @@ -3284,8 +3284,8 @@ TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_02) { //////////////////////////////////////////////////////////////////////////////// TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_03) { - auto x = NDArrayFactory::create('c', {1, 1, 1, 10}, { 1, 2., 3, 4, 5, 6, 7, 8, 9, 10}); - auto exp = NDArrayFactory::create('c', {1, 1, 1, 10}, {0.10425719, 0.16843036, 0.2095291 , 0.23652494, 0.25449327,0.3053919 , 0.35675305, 0.4098524 , 0.46662825, 0.52999896}); + auto x = NDArrayFactory::create('c', {1, 1, 1, 10}, { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f}); + auto exp = NDArrayFactory::create('c', {1, 1, 1, 10}, {0.10425719f, 0.16843036f, 0.2095291f, 0.23652494f, 0.25449327f, 0.3053919f, 0.35675305f, 0.4098524f, 0.46662825f, 0.52999896f}); nd4j::ops::lrn op; auto results = op.execute({&x}, {1.0, 1.0, 0.5}, {5}, {}, false, nd4j::DataType::DOUBLE); @@ -3303,17 +3303,17 @@ TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_03) { //////////////////////////////////////////////////////////////////////////////// TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_1) { - auto x = NDArrayFactory::create('c', {2, 2, 2, 2}, { 5.5, 0., 0.3, 5.5, - 8.6, 0., 0., 0.4, - 1.5, 1., 1.3, 1.5, - 2.6, 2., 3., 1.4} + auto x = NDArrayFactory::create('c', {2, 2, 2, 2}, { 5.5f, 0.f, 0.3f, 5.5f, + 8.6f, 0.f, 0.f, 0.4f, + 1.5f, 1.f, 1.3f, 1.5f, + 2.6f, 2.f, 3.f, 1.4f} ); auto exp = NDArrayFactory::create('c', {2, 2, 2, 2}, { - 0.98386997, 0., 0.05358852, 0.9824562, - 0.99330735, 0., 0., 0.37139067, - 0.72760683, 0.4850712, 0.5848977, 0.67488194, - 0.7581754, 0.58321184, 0.86747235, 0.4048204} + 0.98386997f, 0.f, 0.05358852f, 0.9824562f, + 0.99330735f, 0.f, 0.f, 0.37139067f, + 0.72760683f, 0.4850712f, 0.5848977f, 0.67488194f, + 0.7581754f, 0.58321184f, 0.86747235f, 0.4048204f} ); nd4j::ops::lrn op; @@ -3336,61 +3336,61 @@ TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_2) { x.linspace(1); auto exp = NDArrayFactory::create('c', {3, 3, 5, 5}, { - 0.2581989 ,0.3592106 , 0.40089184, 0.53935987, 0.70014, - 0.4898979 ,0.46056613, 0.43971977, 0.5240002 , 0.6375767, - 0.5274096 ,0.47771242, 0.4443308 , 0.5163977 , 0.61701745, - 0.5424508 ,0.48452914, 0.44570294, 0.5123918 , 0.6068971, - 0.5505386 ,0.4881662 , 0.4462865 , 0.5099462 , 0.60088515, + 0.2581989f, 0.3592106f, 0.40089184f, 0.53935987f, 0.70014f, + 0.4898979f, 0.46056613f, 0.43971977f, 0.5240002f, 0.6375767f, + 0.5274096f, 0.47771242f, 0.4443308f, 0.5163977f, 0.61701745f, + 0.5424508f, 0.48452914f, 0.44570294f, 0.5123918f, 0.6068971f, + 0.5505386f, 0.4881662f, 0.4462865f, 0.5099462f, 0.60088515f, - 0.5555859 , 0.49042296, 0.44658744, 0.5083028 , 0.59690416, - 0.55903524, 0.4919585 , 0.44676256, 0.5071239 , 0.59407425, - 0.5615412 , 0.49307042, 0.44687328, 0.50623745, 0.5919596 , - 0.56344414, 0.49391258, 0.4469477 , 0.5055468 , 0.59031945, - 0.56493837, 0.49457246, 0.4470002 , 0.5049936 , 0.5890103 , + 0.5555859f, 0.49042296f, 0.44658744f, 0.5083028f, 0.59690416f, + 0.55903524f, 0.4919585f, 0.44676256f, 0.5071239f, 0.59407425f, + 0.5615412f, 0.49307042f, 0.44687328f, 0.50623745f, 0.5919596f, + 0.56344414f, 0.49391258f, 0.4469477f, 0.5055468f, 0.59031945f, + 0.56493837f, 0.49457246f, 0.4470002f, 0.5049936f, 0.5890103f, - 0.56614274, 0.49510333, 0.44703856, 0.50454074, 0.5879411 , - 0.567134 , 0.49553978, 0.4470674 , 0.504163 , 0.5870515 , - 0.5679643 , 0.4959048 , 0.44708967, 0.5038433 , 0.5862998 , - 0.56866974, 0.4962146 , 0.44710726, 0.5035692 , 0.58565617, - 0.56927663, 0.49648085, 0.4471213 , 0.5033315 , 0.5850988 , + 0.56614274f, 0.49510333f, 0.44703856f, 0.50454074f, 0.5879411f, + 0.567134f, 0.49553978f, 0.4470674f, 0.504163f, 0.5870515f, + 0.5679643f, 0.4959048f, 0.44708967f, 0.5038433f, 0.5862998f, + 0.56866974f, 0.4962146f, 0.44710726f, 0.5035692f, 0.58565617f, + 0.56927663f, 0.49648085f, 0.4471213f, 0.5033315f, 0.5850988f, - 0.56980413, 0.49671215, 0.44713274, 0.50312346, 0.58461165, - 0.57026696, 0.49691492, 0.4471422 , 0.50293994, 0.58418214, - 0.5706764 , 0.49709415, 0.44715008, 0.5027767 , 0.5838005 , - 0.571041 , 0.4972537 , 0.44715673, 0.50263065, 0.58345926, - 0.57136786, 0.49739665, 0.44716236, 0.5024992 , 0.58315235, + 0.56980413f, 0.49671215f, 0.44713274f, 0.50312346f, 0.58461165f, + 0.57026696f, 0.49691492f, 0.4471422f, 0.50293994f, 0.58418214f, + 0.5706764f, 0.49709415f, 0.44715008f, 0.5027767f, 0.5838005f, + 0.571041f, 0.4972537f, 0.44715673f, 0.50263065f, 0.58345926f, + 0.57136786f, 0.49739665f, 0.44716236f, 0.5024992f, 0.58315235f, - 0.5716625 , 0.49752548, 0.4471672 , 0.5023803, 0.5828747 , - 0.5719295 , 0.49764213, 0.44717142, 0.5022721, 0.5826225 , - 0.57217246, 0.49774826, 0.44717506, 0.5021734, 0.58239233, - 0.5723947 , 0.4978453 , 0.44717824, 0.5020829, 0.58218133, - 0.57259864, 0.49793428, 0.44718108, 0.5019997, 0.5819874 , + 0.5716625f, 0.49752548f, 0.4471672f, 0.5023803f, 0.5828747f, + 0.5719295f, 0.49764213f, 0.44717142f, 0.5022721f, 0.5826225f, + 0.57217246f, 0.49774826f, 0.44717506f, 0.5021734f, 0.58239233f, + 0.5723947f, 0.4978453f, 0.44717824f, 0.5020829f, 0.58218133f, + 0.57259864f, 0.49793428f, 0.44718108f, 0.5019997f, 0.5819874f, - 0.5727864 , 0.49801624, 0.44718358, 0.5019227, 0.5818083 , - 0.57296 , 0.49809194, 0.44718578, 0.5018515, 0.5816426 , - 0.5731208 , 0.49816203, 0.44718775, 0.5017854, 0.58148885, - 0.57327026, 0.49822718, 0.4471895 , 0.5017239, 0.5813457 , - 0.57340944, 0.49828786, 0.44719115, 0.5016664, 0.581212 , + 0.5727864f, 0.49801624f, 0.44718358f, 0.5019227f, 0.5818083f, + 0.57296f, 0.49809194f, 0.44718578f, 0.5018515f, 0.5816426f, + 0.5731208f, 0.49816203f, 0.44718775f, 0.5017854f, 0.58148885f, + 0.57327026f, 0.49822718f, 0.4471895f, 0.5017239f, 0.5813457f, + 0.57340944f, 0.49828786f, 0.44719115f, 0.5016664f, 0.581212f, - 0.57353944, 0.4983446 , 0.44719255, 0.50161266, 0.58108705, - 0.5736612 , 0.49839762, 0.4471939 , 0.50156236, 0.5809699 , - 0.5737754 , 0.4984474 , 0.44719502, 0.501515 , 0.58085984, - 0.5738828 , 0.49849418, 0.4471962 , 0.50147045, 0.5807564 , - 0.5739839 , 0.49853817, 0.44719717, 0.5014284 , 0.5806588 , + 0.57353944f, 0.4983446f, 0.44719255f, 0.50161266f, 0.58108705f, + 0.5736612f, 0.49839762f, 0.4471939f, 0.50156236f, 0.5809699f, + 0.5737754f, 0.4984474f, 0.44719502f, 0.501515f, 0.58085984f, + 0.5738828f, 0.49849418f, 0.4471962f, 0.50147045f, 0.5807564f, + 0.5739839f, 0.49853817f, 0.44719717f, 0.5014284f, 0.5806588f, - 0.5740793 , 0.49857965, 0.4471981 , 0.5013887 , 0.5805666 , - 0.5741694 , 0.49861887, 0.44719887, 0.50135124, 0.58047944, - 0.57425463, 0.49865603, 0.44719967, 0.5013157 , 0.5803969 , - 0.5743354 , 0.4986912 , 0.44720036, 0.5012819 , 0.5803186 , - 0.57441217, 0.49872455, 0.44720104, 0.5012499 , 0.58024424, + 0.5740793f, 0.49857965f, 0.4471981f, 0.5013887f, 0.5805666f, + 0.5741694f, 0.49861887f, 0.44719887f, 0.50135124f, 0.58047944f, + 0.57425463f, 0.49865603f, 0.44719967f, 0.5013157f, 0.5803969f, + 0.5743354f, 0.4986912f, 0.44720036f, 0.5012819f, 0.5803186f, + 0.57441217f, 0.49872455f, 0.44720104f, 0.5012499f, 0.58024424f, - 0.57448506, 0.4987563 , 0.44720164, 0.5012194 , 0.58017343, - 0.57455444, 0.4987865 , 0.4472022 , 0.5011904 , 0.5801061, - 0.57462054, 0.49881527, 0.44720277, 0.5011627 , 0.5800419, - 0.57468355, 0.49884263, 0.44720328, 0.50113624, 0.5799805, - 0.57474375, 0.49886885, 0.44720373, 0.50111103, 0.5799219 } + 0.57448506f, 0.4987563f, 0.44720164f, 0.5012194f, 0.58017343f, + 0.57455444f, 0.4987865f, 0.4472022f, 0.5011904f, 0.5801061f, + 0.57462054f, 0.49881527f, 0.44720277f, 0.5011627f, 0.5800419f, + 0.57468355f, 0.49884263f, 0.44720328f, 0.50113624f, 0.5799805f, + 0.57474375f, 0.49886885f, 0.44720373f, 0.50111103f, 0.5799219f } ); // nd4j::ops::lrn op; @@ -3413,61 +3413,61 @@ TYPED_TEST(TypedDeclarableOpsTests8, LrnTest_3) { x.linspace(1); auto exp = NDArrayFactory::create('c', {3, 3, 5, 5}, { - 0.2581989 ,0.3592106 , 0.40089184, 0.53935987, 0.70014, - 0.4898979 ,0.46056613, 0.43971977, 0.5240002 , 0.6375767, - 0.5274096 ,0.47771242, 0.4443308 , 0.5163977 , 0.61701745, - 0.5424508 ,0.48452914, 0.44570294, 0.5123918 , 0.6068971, - 0.5505386 ,0.4881662 , 0.4462865 , 0.5099462 , 0.60088515, + 0.2581989f, 0.3592106f, 0.40089184f, 0.53935987f, 0.70014f, + 0.4898979f, 0.46056613f, 0.43971977f, 0.5240002f, 0.6375767f, + 0.5274096f, 0.47771242f, 0.4443308f, 0.5163977f, 0.61701745f, + 0.5424508f, 0.48452914f, 0.44570294f, 0.5123918f, 0.6068971f, + 0.5505386f, 0.4881662f, 0.4462865f, 0.5099462f, 0.60088515f, - 0.5555859 , 0.49042296, 0.44658744, 0.5083028 , 0.59690416, - 0.55903524, 0.4919585 , 0.44676256, 0.5071239 , 0.59407425, - 0.5615412 , 0.49307042, 0.44687328, 0.50623745, 0.5919596 , - 0.56344414, 0.49391258, 0.4469477 , 0.5055468 , 0.59031945, - 0.56493837, 0.49457246, 0.4470002 , 0.5049936 , 0.5890103 , + 0.5555859f, 0.49042296f, 0.44658744f, 0.5083028f, 0.59690416f, + 0.55903524f, 0.4919585f, 0.44676256f, 0.5071239f, 0.59407425f, + 0.5615412f, 0.49307042f, 0.44687328f, 0.50623745f, 0.5919596f, + 0.56344414f, 0.49391258f, 0.4469477f, 0.5055468f, 0.59031945f, + 0.56493837f, 0.49457246f, 0.4470002f, 0.5049936f, 0.5890103f, - 0.56614274, 0.49510333, 0.44703856, 0.50454074, 0.5879411 , - 0.567134 , 0.49553978, 0.4470674 , 0.504163 , 0.5870515 , - 0.5679643 , 0.4959048 , 0.44708967, 0.5038433 , 0.5862998 , - 0.56866974, 0.4962146 , 0.44710726, 0.5035692 , 0.58565617, - 0.56927663, 0.49648085, 0.4471213 , 0.5033315 , 0.5850988 , + 0.56614274f, 0.49510333f, 0.44703856f, 0.50454074f, 0.5879411f, + 0.567134f, 0.49553978f, 0.4470674f, 0.504163f, 0.5870515f, + 0.5679643f, 0.4959048f, 0.44708967f, 0.5038433f, 0.5862998f, + 0.56866974f, 0.4962146f, 0.44710726f, 0.5035692f, 0.58565617f, + 0.56927663f, 0.49648085f, 0.4471213f, 0.5033315f, 0.5850988f, - 0.56980413, 0.49671215, 0.44713274, 0.50312346, 0.58461165, - 0.57026696, 0.49691492, 0.4471422 , 0.50293994, 0.58418214, - 0.5706764 , 0.49709415, 0.44715008, 0.5027767 , 0.5838005 , - 0.571041 , 0.4972537 , 0.44715673, 0.50263065, 0.58345926, - 0.57136786, 0.49739665, 0.44716236, 0.5024992 , 0.58315235, + 0.56980413f, 0.49671215f, 0.44713274f, 0.50312346f, 0.58461165f, + 0.57026696f, 0.49691492f, 0.4471422f, 0.50293994f, 0.58418214f, + 0.5706764f, 0.49709415f, 0.44715008f, 0.5027767f, 0.5838005f, + 0.571041f, 0.4972537f, 0.44715673f, 0.50263065f, 0.58345926f, + 0.57136786f, 0.49739665f, 0.44716236f, 0.5024992f, 0.58315235f, - 0.5716625 , 0.49752548, 0.4471672 , 0.5023803, 0.5828747 , - 0.5719295 , 0.49764213, 0.44717142, 0.5022721, 0.5826225 , - 0.57217246, 0.49774826, 0.44717506, 0.5021734, 0.58239233, - 0.5723947 , 0.4978453 , 0.44717824, 0.5020829, 0.58218133, - 0.57259864, 0.49793428, 0.44718108, 0.5019997, 0.5819874 , + 0.5716625f, 0.49752548f, 0.4471672f, 0.5023803f, 0.5828747f, + 0.5719295f, 0.49764213f, 0.44717142f, 0.5022721f, 0.5826225f, + 0.57217246f, 0.49774826f, 0.44717506f, 0.5021734f, 0.58239233f, + 0.5723947f, 0.4978453f, 0.44717824f, 0.5020829f, 0.58218133f, + 0.57259864f, 0.49793428f, 0.44718108f, 0.5019997f, 0.5819874f, - 0.5727864 , 0.49801624, 0.44718358, 0.5019227, 0.5818083 , - 0.57296 , 0.49809194, 0.44718578, 0.5018515, 0.5816426 , - 0.5731208 , 0.49816203, 0.44718775, 0.5017854, 0.58148885, - 0.57327026, 0.49822718, 0.4471895 , 0.5017239, 0.5813457 , - 0.57340944, 0.49828786, 0.44719115, 0.5016664, 0.581212 , + 0.5727864f, 0.49801624f, 0.44718358f, 0.5019227f, 0.5818083f, + 0.57296f, 0.49809194f, 0.44718578f, 0.5018515f, 0.5816426f, + 0.5731208f, 0.49816203f, 0.44718775f, 0.5017854f, 0.58148885f, + 0.57327026f, 0.49822718f, 0.4471895f, 0.5017239f, 0.5813457f, + 0.57340944f, 0.49828786f, 0.44719115f, 0.5016664f, 0.581212f, - 0.57353944, 0.4983446 , 0.44719255, 0.50161266, 0.58108705, - 0.5736612 , 0.49839762, 0.4471939 , 0.50156236, 0.5809699 , - 0.5737754 , 0.4984474 , 0.44719502, 0.501515 , 0.58085984, - 0.5738828 , 0.49849418, 0.4471962 , 0.50147045, 0.5807564 , - 0.5739839 , 0.49853817, 0.44719717, 0.5014284 , 0.5806588 , + 0.57353944f, 0.4983446f, 0.44719255f, 0.50161266f, 0.58108705f, + 0.5736612f, 0.49839762f, 0.4471939f, 0.50156236f, 0.5809699f, + 0.5737754f, 0.4984474f, 0.44719502f, 0.501515f, 0.58085984f, + 0.5738828f, 0.49849418f, 0.4471962f, 0.50147045f, 0.5807564f, + 0.5739839f, 0.49853817f, 0.44719717f, 0.5014284f, 0.5806588f, - 0.5740793 , 0.49857965, 0.4471981 , 0.5013887 , 0.5805666 , - 0.5741694 , 0.49861887, 0.44719887, 0.50135124, 0.58047944, - 0.57425463, 0.49865603, 0.44719967, 0.5013157 , 0.5803969 , - 0.5743354 , 0.4986912 , 0.44720036, 0.5012819 , 0.5803186 , - 0.57441217, 0.49872455, 0.44720104, 0.5012499 , 0.58024424, + 0.5740793f, 0.49857965f, 0.4471981f, 0.5013887f, 0.5805666f, + 0.5741694f, 0.49861887f, 0.44719887f, 0.50135124f, 0.58047944f, + 0.57425463f, 0.49865603f, 0.44719967f, 0.5013157f, 0.5803969f, + 0.5743354f, 0.4986912f, 0.44720036f, 0.5012819f, 0.5803186f, + 0.57441217f, 0.49872455f, 0.44720104f, 0.5012499f, 0.58024424f, - 0.57448506, 0.4987563 , 0.44720164, 0.5012194 , 0.58017343, - 0.57455444, 0.4987865 , 0.4472022 , 0.5011904 , 0.5801061, - 0.57462054, 0.49881527, 0.44720277, 0.5011627 , 0.5800419, - 0.57468355, 0.49884263, 0.44720328, 0.50113624, 0.5799805, - 0.57474375, 0.49886885, 0.44720373, 0.50111103, 0.5799219 } + 0.57448506f, 0.4987563f, 0.44720164f, 0.5012194f, 0.58017343f, + 0.57455444f, 0.4987865f, 0.4472022f, 0.5011904f, 0.5801061f, + 0.57462054f, 0.49881527f, 0.44720277f, 0.5011627f, 0.5800419f, + 0.57468355f, 0.49884263f, 0.44720328f, 0.50113624f, 0.5799805f, + 0.57474375f, 0.49886885f, 0.44720373f, 0.50111103f, 0.5799219f } ); // nd4j::ops::lrn op; diff --git a/libnd4j/tests_cpu/layers_tests/MultiDeviceTests.cpp b/libnd4j/tests_cpu/layers_tests/MultiDeviceTests.cpp new file mode 100644 index 000000000..efd48311b --- /dev/null +++ b/libnd4j/tests_cpu/layers_tests/MultiDeviceTests.cpp @@ -0,0 +1,68 @@ +/******************************************************************************* + * Copyright (c) 2015-2019 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +// +// @author raver119@gmail.com +// + +#include "testlayers.h" +#include +#include +#include +#include +#include +#include +#include + + +using namespace nd4j; + +class MultiDeviceTests : public testing::Test { +public: + +}; + +void createArrays(int limit, std::vector &arrays) { + auto deviceId = AffinityManager::currentDeviceId(); + auto numDevices = AffinityManager::numberOfDevices(); + + for (int e = 0; e < limit; e++) { + auto value = deviceId * limit + e; + arrays[value] = NDArrayFactory::create_('c', {10}); + arrays[value]->assign(value); + //nd4j_printf("device_%i; value: [%i]; mean: [%f]\n", deviceId, value, arrays[value]->meanNumber().e(0)); + } +} + +TEST_F(MultiDeviceTests, test_multi_device_migration_1) { + auto deviceId = AffinityManager::currentDeviceId(); + auto numDevices = AffinityManager::numberOfDevices(); + auto numArrays = 10; + std::vector arrays(numDevices * numArrays); + + // filling list of arrays on multiple threads + for (int e = 0; e < numDevices; e++) { + std::thread t1(createArrays, numArrays, std::ref(arrays)); + + t1.join(); + } + + // at this moment all arrays are build, so we can test migration + for (int e = 0; e < arrays.size(); e++) { + ASSERT_NEAR((float) e, arrays[e]->meanNumber().e(0), 1e-5f); + delete arrays[e]; + } +} diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java index ad4cad0b0..873e97809 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/allocator/impl/AtomicAllocator.java @@ -438,6 +438,7 @@ public class AtomicAllocator implements Allocator { Long allocId = objectsTracker.getAndIncrement(); point.setObjectId(allocId); point.setConstant(true); + point.setDeviceId(Nd4j.getAffinityManager().getDeviceForCurrentThread()); allocationsMap.put(allocId, point); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java index 30bd2dd93..cf362c460 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/concurrency/CudaAffinityManager.java @@ -68,11 +68,7 @@ public class CudaAffinityManager extends BasicAffinityManager { */ @Override public Integer getDeviceForCurrentThread() { - val id = NativeOpsHolder.getInstance().getDeviceNativeOps().getDevice(); - if (!affinityMap.containsKey(Thread.currentThread().getId())) - affinityMap.put(Thread.currentThread().getId(), id); - - return id; + return NativeOpsHolder.getInstance().getDeviceNativeOps().getDevice(); } /** @@ -205,7 +201,6 @@ public class CudaAffinityManager extends BasicAffinityManager { int currentDeviceId = getDeviceForCurrentThread(); if (currentDeviceId != deviceId.intValue()) { - Nd4j.getMemoryManager().releaseCurrentContext(); unsafeSetDevice(deviceId); } @@ -215,7 +210,6 @@ public class CudaAffinityManager extends BasicAffinityManager { INDArray result = Nd4j.createArrayFromShapeBuffer(newDataBuffer, newShapeBuffer); if (currentDeviceId != deviceId.intValue()) { - Nd4j.getMemoryManager().releaseCurrentContext(); unsafeSetDevice(currentDeviceId); } @@ -238,7 +232,6 @@ public class CudaAffinityManager extends BasicAffinityManager { int currentDeviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread(); if (currentDeviceId != deviceId) { - Nd4j.getMemoryManager().releaseCurrentContext(); Nd4j.getAffinityManager().unsafeSetDevice(deviceId); } @@ -246,7 +239,6 @@ public class CudaAffinityManager extends BasicAffinityManager { AtomicAllocator.getInstance().memcpy(dstBuffer, buffer); if (currentDeviceId != deviceId) { - Nd4j.getMemoryManager().releaseCurrentContext(); Nd4j.getAffinityManager().unsafeSetDevice(currentDeviceId); } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java index 07cad5269..fb4510f1b 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/flow/impl/SynchronousFlowController.java @@ -188,13 +188,15 @@ public class SynchronousFlowController implements FlowController { } if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) { - ((JCublasNDArray) result).setShapeInfoDataBuffer( - Nd4j.getConstantHandler().relocateConstantSpace(result.shapeInfoDataBuffer())); + ((JCublasNDArray) result).setShapeInfoDataBuffer(Nd4j.getExecutioner().createShapeInfo(result.shape(), result.stride(), result.elementWiseStride(), result.ordering(), result.dataType(), result.isEmpty())); } allocator.getAllocationPoint(result).setCurrentContext(context); } + if (operands == null) + return context; + for (INDArray operand : operands) { if (operand == null || operand.isEmpty()) continue; @@ -213,8 +215,7 @@ public class SynchronousFlowController implements FlowController { } if (pointShape.getDeviceId() != cId && pointShape.getDeviceId() >= 0) { - ((JCublasNDArray) operand).setShapeInfoDataBuffer( - Nd4j.getConstantHandler().relocateConstantSpace(operand.shapeInfoDataBuffer())); + ((JCublasNDArray) operand).setShapeInfoDataBuffer(Nd4j.getExecutioner().createShapeInfo(operand.shape(), operand.stride(), operand.elementWiseStride(), operand.ordering(), operand.dataType(), operand.isEmpty())); } prepareDelayedMemory(operand); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java index e35628728..67f58ba94 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/jita/handler/impl/CudaZeroHandler.java @@ -819,10 +819,10 @@ public class CudaZeroHandler implements MemoryHandler { val ohPtr = dstPoint.getHostPointer(); // FIXME: cross-thread access, might cause problems - if (!dstPoint.isActualOnHostSide()) + if (dstPoint.getHostPointer() != null && !dstPoint.isActualOnHostSide()) AtomicAllocator.getInstance().synchronizeHostData(buffer); - if (!dstPoint.isActualOnHostSide()) + if (dstPoint.getHostPointer() != null && !dstPoint.isActualOnHostSide()) throw new RuntimeException("Buffer synchronization failed"); if (buffer.isAttached() || dstPoint.isAttached()) { @@ -832,10 +832,14 @@ public class CudaZeroHandler implements MemoryHandler { if (workspace == null) { // if we're out of workspace, we should mark our buffer as detached, so gc will pick it up eventually - val pairH = alloc(AllocationStatus.HOST, dstPoint, dstPoint.getShape(), false); + // host part is optional + if (dstPoint.getHostPointer() != null) { + val pairH = alloc(AllocationStatus.HOST, dstPoint, dstPoint.getShape(), false); + dstPoint.getPointers().setHostPointer(pairH.getHostPointer()); + } + val pairD = alloc(AllocationStatus.DEVICE, dstPoint, dstPoint.getShape(), false); dstPoint.getPointers().setDevicePointer(pairD.getDevicePointer()); - dstPoint.getPointers().setHostPointer(pairH.getHostPointer()); //log.info("New host pointer: {}; Old host pointer: {}", dstPoint.getHostPointer().address(), ohPtr.address()); @@ -869,7 +873,11 @@ public class CudaZeroHandler implements MemoryHandler { Nd4j.getMemoryManager().memcpy(nBuffer, buffer); dstPoint.getPointers().setDevicePointer(nBuffer.getAllocationPoint().getDevicePointer()); - dstPoint.getPointers().setHostPointer(nBuffer.getAllocationPoint().getHostPointer()); + + if (dstPoint.getHostPointer() != null) { + dstPoint.getPointers().setHostPointer(nBuffer.getAllocationPoint().getHostPointer()); + } + dstPoint.setDeviceId(deviceId); dstPoint.tickDeviceRead(); @@ -885,6 +893,17 @@ public class CudaZeroHandler implements MemoryHandler { throw new RuntimeException("Can't relocateObject() for constant buffer"); } else { // log.info("Free relocateObject: deviceId: {}, pointer: {}", deviceId, dstPoint.getPointers().getDevicePointer().address()); + val context = getCudaContext(); + if (dstPoint.getHostPointer() == null) { + ((BaseCudaDataBuffer) buffer).lazyAllocateHostPointer(); + + if (nativeOps.memcpyAsync(dstPoint.getHostPointer(), dstPoint.getDevicePointer(), + buffer.length() * buffer.getElementSize(), 2, context.getSpecialStream()) == 0) + throw new ND4JIllegalStateException("memcpyAsync failed"); + + context.syncSpecialStream(); + } + memoryProvider.free(dstPoint); deviceMemoryTracker.subFromAllocation(Thread.currentThread().getId(), dstPoint.getDeviceId(), AllocationUtils.getRequiredMemory(dstPoint.getShape())); @@ -893,7 +912,6 @@ public class CudaZeroHandler implements MemoryHandler { val profD = PerformanceTracker.getInstance().helperStartTransaction(); - CudaContext context = getCudaContext(); if (nativeOps.memcpyAsync(dstPoint.getDevicePointer(), dstPoint.getHostPointer(), buffer.length() * buffer.getElementSize(), 1, context.getSpecialStream()) == 0) throw new ND4JIllegalStateException("memcpyAsync failed"); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java index cd0356d18..170f22a52 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/JCublasNDArray.java @@ -17,6 +17,7 @@ package org.nd4j.linalg.jcublas; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.nd4j.jita.allocator.enums.AllocationStatus; import org.nd4j.jita.allocator.enums.CudaConstants; @@ -54,7 +55,7 @@ import java.util.concurrent.atomic.AtomicLong; * @author Adam Gibson * @author raver119@gmail.com */ - +@Slf4j public class JCublasNDArray extends BaseNDArray { @@ -737,14 +738,12 @@ public class JCublasNDArray extends BaseNDArray { if (!this.isView()) { Nd4j.getExecutioner().commit(); - DataBuffer buffer = Nd4j.createBuffer(this.lengthLong(), false); + val buffer = Nd4j.createBuffer(this.dataType(), this.lengthLong(), false); - AllocationPoint pointDst = AtomicAllocator.getInstance().getAllocationPoint(buffer); - AllocationPoint pointSrc = AtomicAllocator.getInstance().getAllocationPoint(this.data); + val pointDst = AtomicAllocator.getInstance().getAllocationPoint(buffer); + val pointSrc = AtomicAllocator.getInstance().getAllocationPoint(this.data); -// CudaContext context = (CudaContext) AtomicAllocator.getInstance().getDeviceContext().getContext(); - - CudaContext context = AtomicAllocator.getInstance().getFlowController().prepareAction(pointDst, pointSrc); + val context = AtomicAllocator.getInstance().getFlowController().prepareAction(pointDst, pointSrc); MemcpyDirection direction = MemcpyDirection.DEVICE_TO_DEVICE; val perfD = PerformanceTracker.getInstance().helperStartTransaction(); @@ -764,14 +763,12 @@ public class JCublasNDArray extends BaseNDArray { PerformanceTracker.getInstance().helperRegisterTransaction(pointDst.getDeviceId(), perfD, pointDst.getNumberOfBytes(), direction); if (pointDst.getDeviceId() != Nd4j.getMemoryManager().getCurrentWorkspace().getDeviceId()) { - //log.info("Swapping [{}] -> [{}]", pointDst.getDeviceId(), Nd4j.getMemoryManager().getCurrentWorkspace().getDeviceId()); pointDst.setDeviceId(Nd4j.getMemoryManager().getCurrentWorkspace().getDeviceId()); } copy = Nd4j.createArrayFromShapeBuffer(buffer, this.shapeInfoDataBuffer()); // tag buffer as valid on device side - pointDst.tickHostRead(); pointDst.tickDeviceWrite(); AtomicAllocator.getInstance().getFlowController().registerAction(context, pointDst, pointSrc); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java index 77093a5bb..72e089e45 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/buffer/factory/CudaDataBufferFactory.java @@ -25,6 +25,7 @@ import org.bytedeco.javacpp.indexer.*; import org.nd4j.linalg.api.buffer.DataBuffer; import org.nd4j.linalg.api.buffer.DataType; import org.nd4j.linalg.api.buffer.LongBuffer; +import org.nd4j.linalg.api.buffer.Utf8Buffer; import org.nd4j.linalg.api.buffer.factory.DataBufferFactory; import org.nd4j.linalg.api.memory.MemoryWorkspace; import org.nd4j.linalg.exception.ND4JIllegalStateException; @@ -92,6 +93,8 @@ public class CudaDataBufferFactory implements DataBufferFactory { return new CudaByteDataBuffer(underlyingBuffer, length, offset); case BOOL: return new CudaBoolDataBuffer(underlyingBuffer, length, offset); + case UTF8: + return new Utf8Buffer(underlyingBuffer, length, offset); default: throw new ND4JIllegalStateException("Unknown data buffer type: " + underlyingBuffer.dataType().toString()); } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java index 1a964bef5..7d41ba986 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaExecutioner.java @@ -233,7 +233,7 @@ public class CudaExecutioner extends DefaultOpExecutioner { throw new ND4JIllegalStateException("Op target dimension " + Arrays.toString(dimension) + " contains element that higher then rank of op.X: [" + op.x().rank() + "]"); - CudaContext context = AtomicAllocator.getInstance().getFlowController().prepareAction(op.z(), op.x(), op.y()); + val context = AtomicAllocator.getInstance().getFlowController().prepareAction(op.z(), op.x(), op.y()); if (CudaEnvironment.getInstance().getConfiguration().isDebug()) lastOp.set(op.opName()); @@ -2491,6 +2491,11 @@ public class CudaExecutioner extends DefaultOpExecutioner { nativeOps.execCustomOp2(null, op.opHash(), context.contextPointer()); + for (val arr:op.outputArguments()) + AtomicAllocator.getInstance().registerAction(ctx, arr); + + AtomicAllocator.getInstance().registerAction(ctx, null, op.inputArguments()); + profilingConfigurableHookOut(op, st); if (context.getOutputArrays().isEmpty()) diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java index 749f5cc96..26d363f32 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/linalg/jcublas/ops/executioner/CudaOpContext.java @@ -81,10 +81,8 @@ public class CudaOpContext extends BaseOpContext implements OpContext { @Override public void setInputArray(int index, @NonNull INDArray array) { - // FIXME: remove - Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.EVERYWHERE); + val ctx = AtomicAllocator.getInstance().getFlowController().prepareAction(null, array); - val ctx = AtomicAllocator.getInstance().getDeviceContext(); nativeOps.setGraphContextInputArray(context, index, array.isEmpty() ? null : array.data().addressPointer(), array.shapeInfoDataBuffer().addressPointer(), array.isEmpty() ? null : AtomicAllocator.getInstance().getPointer(array, ctx), AtomicAllocator.getInstance().getPointer(array.shapeInfoDataBuffer())); super.setInputArray(index, array); @@ -92,9 +90,8 @@ public class CudaOpContext extends BaseOpContext implements OpContext { @Override public void setOutputArray(int index, @NonNull INDArray array) { - Nd4j.getAffinityManager().ensureLocation(array, AffinityManager.Location.EVERYWHERE); + val ctx = AtomicAllocator.getInstance().getFlowController().prepareAction(array, null); - val ctx = AtomicAllocator.getInstance().getDeviceContext(); nativeOps.setGraphContextOutputArray(context, index, array.isEmpty() ? null : array.data().addressPointer(), array.shapeInfoDataBuffer().addressPointer(), array.isEmpty() ? null : AtomicAllocator.getInstance().getPointer(array, ctx), AtomicAllocator.getInstance().getPointer(array.shapeInfoDataBuffer())); super.setOutputArray(index, array); diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java index b15e4455e..886b258ae 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCuda.java @@ -9898,6 +9898,72 @@ public static final int PREALLOC_SIZE = 33554432; // #endif //LIBND4J_OPREGISTRATOR_H +// Parsed from execution/ContextBuffers.h + +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +// +// @author raver119@gmail.com +// + +// #ifndef LIBND4J_CONTEXTBUFFERS_H +// #define LIBND4J_CONTEXTBUFFERS_H + +// #include +// #include + @Namespace("nd4j") @NoOffset public static class ContextBuffers extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ContextBuffers(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public ContextBuffers(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public ContextBuffers position(long position) { + return (ContextBuffers)super.position(position); + } + + public ContextBuffers() { super((Pointer)null); allocate(); } + private native void allocate(); + public ContextBuffers(Pointer rPointer, Pointer sPointer, Pointer aPointer, @Cast("bool") boolean isOwner/*=false*/) { super((Pointer)null); allocate(rPointer, sPointer, aPointer, isOwner); } + private native void allocate(Pointer rPointer, Pointer sPointer, Pointer aPointer, @Cast("bool") boolean isOwner/*=false*/); + public ContextBuffers(Pointer rPointer, Pointer sPointer, Pointer aPointer) { super((Pointer)null); allocate(rPointer, sPointer, aPointer); } + private native void allocate(Pointer rPointer, Pointer sPointer, Pointer aPointer); + + public native Pointer reductionBuffer(); + public native Pointer scalarBuffer(); + public native Pointer allocationBuffer(); + + public native Pointer execStream(); + public native Pointer specialStream(); + + public native void setReductionBuffer(Pointer pointer); + public native void setScalarBuffer(Pointer pointer); + public native void setAllocationBuffer(Pointer pointer); + + public native void triggerOwnership(@Cast("bool") boolean isOwner); + + public native int deviceId(); + } + + + +// #endif //DEV_TESTS_CONTEXTBUFFERS_H + + // Parsed from execution/LaunchContext.h /******************************************************************************* @@ -9971,6 +10037,9 @@ public static final int PREALLOC_SIZE = 33554432; public static native LaunchContext defaultContext(); + + public static native void swapContextBuffers(@ByRef ContextBuffers buffers); + } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java index 51b9ce7e4..4466cf4b5 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-cuda/src/main/java/org/nd4j/nativeblas/Nd4jCudaPresets.java @@ -76,6 +76,7 @@ import org.bytedeco.javacpp.tools.InfoMapper; "ops/declarable/BooleanOp.h", "ops/declarable/LogicOp.h", "ops/declarable/OpRegistrator.h", + "execution/ContextBuffers.h", "execution/LaunchContext.h", "array/ShapeDescriptor.h", "array/TadDescriptor.h", diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java index d05fd1682..2f1189aea 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpu.java @@ -22808,6 +22808,72 @@ public static final int TAD_THRESHOLD = TAD_THRESHOLD(); // #endif +// Parsed from execution/ContextBuffers.h + +/******************************************************************************* + * Copyright (c) 2015-2018 Skymind, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +// +// @author raver119@gmail.com +// + +// #ifndef LIBND4J_CONTEXTBUFFERS_H +// #define LIBND4J_CONTEXTBUFFERS_H + +// #include +// #include + @Namespace("nd4j") @NoOffset public static class ContextBuffers extends Pointer { + static { Loader.load(); } + /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */ + public ContextBuffers(Pointer p) { super(p); } + /** Native array allocator. Access with {@link Pointer#position(long)}. */ + public ContextBuffers(long size) { super((Pointer)null); allocateArray(size); } + private native void allocateArray(long size); + @Override public ContextBuffers position(long position) { + return (ContextBuffers)super.position(position); + } + + public ContextBuffers() { super((Pointer)null); allocate(); } + private native void allocate(); + public ContextBuffers(Pointer rPointer, Pointer sPointer, Pointer aPointer, @Cast("bool") boolean isOwner/*=false*/) { super((Pointer)null); allocate(rPointer, sPointer, aPointer, isOwner); } + private native void allocate(Pointer rPointer, Pointer sPointer, Pointer aPointer, @Cast("bool") boolean isOwner/*=false*/); + public ContextBuffers(Pointer rPointer, Pointer sPointer, Pointer aPointer) { super((Pointer)null); allocate(rPointer, sPointer, aPointer); } + private native void allocate(Pointer rPointer, Pointer sPointer, Pointer aPointer); + + public native Pointer reductionBuffer(); + public native Pointer scalarBuffer(); + public native Pointer allocationBuffer(); + + public native Pointer execStream(); + public native Pointer specialStream(); + + public native void setReductionBuffer(Pointer pointer); + public native void setScalarBuffer(Pointer pointer); + public native void setAllocationBuffer(Pointer pointer); + + public native void triggerOwnership(@Cast("bool") boolean isOwner); + + public native int deviceId(); + } + + + +// #endif //DEV_TESTS_CONTEXTBUFFERS_H + + // Parsed from execution/LaunchContext.h /******************************************************************************* @@ -22872,6 +22938,9 @@ public static final int TAD_THRESHOLD = TAD_THRESHOLD(); public static native LaunchContext defaultContext(); + + public static native void swapContextBuffers(@ByRef ContextBuffers buffers); + } diff --git a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java index dd47eb25d..271fa662d 100644 --- a/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java +++ b/nd4j/nd4j-backends/nd4j-backend-impls/nd4j-native/src/main/java/org/nd4j/nativeblas/Nd4jCpuPresets.java @@ -100,6 +100,7 @@ import java.util.Scanner; "ops/declarable/headers/bitwise.h", "ops/declarable/headers/loss.h", "ops/declarable/headers/datatypes.h", + "execution/ContextBuffers.h", "execution/LaunchContext.h", "array/ShapeDescriptor.h", "array/TadDescriptor.h", diff --git a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/crash/SpecialTests.java b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/crash/SpecialTests.java index e3aeb6fbd..75a91263d 100644 --- a/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/crash/SpecialTests.java +++ b/nd4j/nd4j-backends/nd4j-tests/src/test/java/org/nd4j/linalg/crash/SpecialTests.java @@ -198,27 +198,43 @@ public class SpecialTests extends BaseNd4jTest { val list = new CopyOnWriteArrayList(); val threads = new ArrayList(); - for (int e = 0; e< Nd4j.getAffinityManager().getNumberOfDevices(); e++) { + val devices = Nd4j.getAffinityManager().getNumberOfDevices(); + for (int e = 0; e < devices; e++) { val f = e; val t = new Thread(new Runnable() { @Override public void run() { + val deviceId = Nd4j.getAffinityManager().getDeviceForCurrentThread(); + log.info("Current device: {}", deviceId); for (int i = 0; i < 10; i++) { - list.add(Nd4j.create(100, 100).assign(1.0f)); + val ar = Nd4j.create(100, 100).assign(1.0f); + + assertEquals(deviceId, Nd4j.getAffinityManager().getDeviceForArray(ar)); + list.add(ar); Nd4j.getExecutioner().commit(); } } }); t.start(); + t.join(); threads.add(t); + + log.info("------------------------"); } for (val t:threads) t.join(); - for (val a:list) - assertEquals(1.0f, a.meanNumber().floatValue(), 1e-5); + for (val a:list) { + val device = Nd4j.getAffinityManager().getDeviceForArray(a); + try { + assertEquals(1.0f, a.meanNumber().floatValue(), 1e-5); + } catch (Exception e) { + log.error("Failed for array from device [{}]", device); + throw e; + } + } } @Test