cavis/libnd4j/include/memory/cpu/Workspace.cpp

221 lines
6.6 KiB
C++
Raw Normal View History

2021-02-01 13:31:45 +01:00
/* ******************************************************************************
*
2019-06-06 14:21:15 +02:00
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
2021-02-01 13:31:45 +01:00
* See the NOTICE file distributed with this work for additional
* information regarding copyright ownership.
2019-06-06 14:21:15 +02:00
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
//
// CPU workspaces implementation
//
// @author raver119@gmail.com
//
#include <system/op_boilerplate.h>
2019-06-06 14:21:15 +02:00
#include <atomic>
#include <stdio.h>
#include <stdlib.h>
#include "../Workspace.h"
#include <helpers/logger.h>
#include <math/templatemath.h>
2019-06-06 14:21:15 +02:00
#include <cstring>
namespace sd {
2019-06-06 14:21:15 +02:00
namespace memory {
Workspace::Workspace(ExternalWorkspace *external) {
if (external->sizeHost() > 0) {
_ptrHost = (char *) external->pointerHost();
_ptrDevice = (char *) external->pointerDevice();
_initialSize = external->sizeHost();
_currentSize = external->sizeHost();
_offset = 0L;
_offsetSecondary = 0L;
this->_cycleAllocations = 0;
this->_spillsSize = 0;
_externalized = true;
}
};
Workspace::Workspace(Nd4jLong initialSize, Nd4jLong secondaryBytes) {
if (initialSize > 0) {
this->_ptrHost = (char *) malloc(initialSize);
CHECK_ALLOC(this->_ptrHost, "Failed to allocate new workspace", initialSize);
memset(this->_ptrHost, 0, initialSize);
this->_allocatedHost = true;
} else
this->_allocatedHost = false;
this->_initialSize = initialSize;
this->_currentSize = initialSize;
[WIP] More of CUDA (#95) * initial commit Signed-off-by: raver119 <raver119@gmail.com> * Implementation of hashcode cuda helper. Working edition. * Fixed parallel test input arangements. * Fixed tests for hashcode op. * Fixed shape calculation for image:crop_and_resize op and test. * NativeOps tests. Initial test suite. * Added tests for indexReduce methods. * Added test on execBroadcast with NDArray as dimensions. * Added test on execBroadcastBool with NDArray as dimensions. * Added tests on execPairwiseTransform and execPairwiseTransofrmBool. * Added tests for execReduce with scalar results. * Added reduce tests for non-empty dims array. * Added tests for reduce3. * Added tests for execScalar. * Added tests for execSummaryStats. * - provide cpu/cuda code for batch_to_space - testing it Signed-off-by: Yurii <yurii@skymind.io> * - remove old test for batch_to_space (had wrong format and numbers were not checked) Signed-off-by: Yurii <yurii@skymind.io> * Fixed complilation errors with test. * Added test for execTransformFloat. * Added test for execTransformSame. * Added test for execTransformBool. * Added test for execTransformStrict. * Added tests for execScalar/execScalarBool with TADs. * Added test for flatten. * - provide cpu/cuda code for space_to_Batch operaion Signed-off-by: Yurii <yurii@skymind.io> * Added test for concat. * comment unnecessary stuff in s_t_b Signed-off-by: Yurii <yurii@skymind.io> * Added test for specialConcat. * Added tests for memcpy/set routines. * Fixed pullRow cuda test. * Added pullRow test. * Added average test. * - correct typo in NDArray::applyPairwiseTransform(nd4j::pairwise::BoolOps op...) Signed-off-by: Yurii <yurii@skymind.io> * - debugging and fixing cuda tests in JavaInteropTests file Signed-off-by: Yurii <yurii@skymind.io> * - correct some tests Signed-off-by: Yurii <yurii@skymind.io> * Added test for shuffle. * Fixed ops declarations. * Restored omp and added shuffle test. * Added convertTypes test. * Added tests for execRandom. Eliminated usage of RandomBuffer with NativeOps. * Added sort tests. * Added tests for execCustomOp. * - further debuging and fixing tests terminated with crash Signed-off-by: Yurii <yurii@skymind.io> * Added tests for calculateOutputShapes. * Addded Benchmarks test. * Commented benchmark tests. * change assertion Signed-off-by: raver119 <raver119@gmail.com> * Added tests for apply_sgd op. Added cpu helper for that op. * Implement cuda helper for aplly_sgd op. Fixed tests for NativeOps. * Added test for assign broadcastable. * Added tests for assign_bp op. * Added tests for axpy op. * - assign/execScalar/execTransformAny signature change - minor test fix Signed-off-by: raver119 <raver119@gmail.com> * Fixed axpy op. * meh Signed-off-by: raver119 <raver119@gmail.com> * - fix tests for nativeOps::concat Signed-off-by: Yurii <yurii@skymind.io> * sequential transform/scalar Signed-off-by: raver119 <raver119@gmail.com> * allow nested parallelism Signed-off-by: raver119 <raver119@gmail.com> * assign_bp leak fix Signed-off-by: raver119 <raver119@gmail.com> * block setRNG fix Signed-off-by: raver119 <raver119@gmail.com> * enable parallelism by default Signed-off-by: raver119 <raver119@gmail.com> * enable nested parallelism by default Signed-off-by: raver119 <raver119@gmail.com> * Added cuda implementation for row_count helper. * Added implementation for tnse gains op helper. * - take into account possible situations when input arrays are empty in reduce_ cuda stuff Signed-off-by: Yurii <yurii@skymind.io> * Implemented tsne/edge_forces op cuda-based helper. Parallelized cpu-based helper for edge_forces. * Added kernel for tsne/symmetrized op heleper. * Implementation of tsne/symmetrized op cuda helper. Working edition. * Eliminated waste printfs. * Added test for broadcastgradientargs op. * host-only fallback for empty reduce float Signed-off-by: raver119 <raver119@gmail.com> * - some tests fixes Signed-off-by: Yurii <yurii@skymind.io> * - correct the rest of reduce_ stuff Signed-off-by: Yurii <yurii@skymind.io> * - further correction of reduce_ stuff Signed-off-by: Yurii <yurii@skymind.io> * Added test for Cbow op. Also added cuda implementation for cbow helpers. * - improve code of stack operation for scalar case Signed-off-by: Yurii <yurii@skymind.io> * - provide cuda kernel for gatherND operation Signed-off-by: Yurii <yurii@skymind.io> * Implementation of cbow helpers with cuda kernels. * minor tests tweaks Signed-off-by: raver119 <raver119@gmail.com> * minor tests tweaks Signed-off-by: raver119 <raver119@gmail.com> * - further correction of cuda stuff Signed-off-by: Yurii <yurii@skymind.io> * Implementatation of cbow op helper with cuda kernels. Working edition. * Skip random testing for cudablas case. * lstmBlockCell context fix Signed-off-by: raver119 <raver119@gmail.com> * Added tests for ELU and ELU_BP ops. * Added tests for eq_scalar, gt_scalar, gte_scalar and lte_scalar ops. * Added tests for neq_scalar. * Added test for noop. * - further work on clipbynorm_bp Signed-off-by: Yurii <yurii@skymind.io> * - get rid of concat op call, use instead direct concat helper call Signed-off-by: Yurii <yurii@skymind.io> * lstmBlockCell context fix Signed-off-by: raver119 <raver119@gmail.com> * Added tests for lrelu and lrelu_bp. * Added tests for selu and selu_bp. * Fixed lrelu derivative helpers. * - some corrections in lstm Signed-off-by: Yurii <yurii@skymind.io> * operator * result shape fix Signed-off-by: raver119 <raver119@gmail.com> * - correct typo in lstmCell Signed-off-by: Yurii <yurii@skymind.io> * few tests fixed Signed-off-by: raver119 <raver119@gmail.com> * CUDA inverse broadcast bool fix Signed-off-by: raver119 <raver119@gmail.com> * disable MMAP test for CUDA Signed-off-by: raver119 <raver119@gmail.com> * BooleanOp syncToDevice Signed-off-by: raver119 <raver119@gmail.com> * meh Signed-off-by: raver119 <raver119@gmail.com> * additional data types for im2col/col2im Signed-off-by: raver119 <raver119@gmail.com> * Added test for firas_sparse op. * one more RandomBuffer test excluded Signed-off-by: raver119 <raver119@gmail.com> * Added tests for flatten op. * Added test for Floor op. * bunch of tests fixed Signed-off-by: raver119 <raver119@gmail.com> * mmulDot tests fixed Signed-off-by: raver119 <raver119@gmail.com> * more tests fixed Signed-off-by: raver119 <raver119@gmail.com> * Implemented floordiv_bp op and tests. * Fixed scalar case with cuda implementation for bds. * - work on cuda kernel for clip_by_norm backprop op is completed Signed-off-by: Yurii <yurii@skymind.io> * Eliminate cbow crach. * more tests fixed Signed-off-by: raver119 <raver119@gmail.com> * more tests fixed Signed-off-by: raver119 <raver119@gmail.com> * Eliminated abortion with batched nlp test. * more tests fixed Signed-off-by: raver119 <raver119@gmail.com> * Fixed shared flag initializing. * disabled bunch of cpu workspaces tests Signed-off-by: raver119 <raver119@gmail.com> * scalar operators fix: missing registerSpecialUse call Signed-off-by: raver119 <raver119@gmail.com> * Fixed logdet for cuda and tests. * - correct clipBynorm_bp Signed-off-by: Yurii <yurii@skymind.io> * Fixed crop_and_resize shape datatype. * - correct some mmul tests Signed-off-by: Yurii <yurii@skymind.io>
2019-08-02 19:01:03 +02:00
this->_currentSizeSecondary = 0;
this->_spillsSizeSecondary = 0;
2019-06-06 14:21:15 +02:00
this->_offset = 0;
this->_offsetSecondary = 0;
this->_cycleAllocations = 0;
this->_spillsSize = 0;
}
void Workspace::init(Nd4jLong bytes, Nd4jLong secondaryBytes) {
if (this->_currentSize < bytes) {
if (this->_allocatedHost && !_externalized)
free((void *)this->_ptrHost);
this->_ptrHost =(char *) malloc(bytes);
CHECK_ALLOC(this->_ptrHost, "Failed to allocate new workspace", bytes);
memset(this->_ptrHost, 0, bytes);
this->_currentSize = bytes;
this->_allocatedHost = true;
}
}
void Workspace::expandBy(Nd4jLong numBytes, Nd4jLong secondaryBytes) {
this->init(_currentSize + numBytes, _currentSizeSecondary + secondaryBytes);
}
void Workspace::expandTo(Nd4jLong numBytes, Nd4jLong secondaryBytes) {
this->init(numBytes, secondaryBytes);
}
void Workspace::freeSpills() {
_spillsSize = 0;
if (_spills.size() < 1)
return;
for (auto v:_spills)
free(v);
_spills.clear();
}
Workspace::~Workspace() {
if (this->_allocatedHost && !_externalized)
free((void *)this->_ptrHost);
freeSpills();
}
Nd4jLong Workspace::getUsedSize() {
return getCurrentOffset();
}
Nd4jLong Workspace::getCurrentSize() {
return _currentSize;
}
Nd4jLong Workspace::getCurrentOffset() {
return _offset.load();
}
void* Workspace::allocateBytes(Nd4jLong numBytes) {
if (numBytes < 1)
throw allocation_exception::build("Number of bytes for allocation should be positive", numBytes);
//numBytes += 32;
void* result = nullptr;
this->_cycleAllocations += numBytes;
this->_mutexAllocation.lock();
if (_offset.load() + numBytes > _currentSize) {
nd4j_debug("Allocating %lld bytes in spills\n", numBytes);
this->_mutexAllocation.unlock();
void *p = malloc(numBytes);
CHECK_ALLOC(p, "Failed to allocate new workspace", numBytes);
_mutexSpills.lock();
_spills.push_back(p);
_mutexSpills.unlock();
_spillsSize += numBytes;
return p;
}
result = (void *)(_ptrHost + _offset.load());
_offset += numBytes;
//memset(result, 0, (int) numBytes);
nd4j_debug("Allocating %lld bytes from workspace; Current PTR: %p; Current offset: %lld\n", numBytes, result, _offset.load());
this->_mutexAllocation.unlock();
return result;
}
Nd4jLong Workspace::getAllocatedSize() {
return getCurrentSize() + getSpilledSize();
}
void Workspace::scopeIn() {
freeSpills();
init(_cycleAllocations.load());
_cycleAllocations = 0;
}
void Workspace::scopeOut() {
_offset = 0;
_offsetSecondary = 0;
}
Nd4jLong Workspace::getSpilledSize() {
return _spillsSize.load();
}
void* Workspace::allocateBytes(sd::memory::MemoryType type, Nd4jLong numBytes) {
2019-06-06 14:21:15 +02:00
if (type == DEVICE)
throw std::runtime_error("CPU backend doesn't have device memory");
return this->allocateBytes(numBytes);
}
Nd4jLong Workspace::getAllocatedSecondarySize() {
return 0L;
}
Nd4jLong Workspace::getCurrentSecondarySize() {
return 0L;
}
Nd4jLong Workspace::getCurrentSecondaryOffset() {
return 0L;
}
Nd4jLong Workspace::getSpilledSecondarySize() {
return 0L;
}
Nd4jLong Workspace::getUsedSecondarySize() {
return 0L;
}
Workspace* Workspace::clone() {
// for clone we take whatever is higher: current allocated size, or allocated size of current loop
return new Workspace(sd::math::nd4j_max<Nd4jLong >(this->getCurrentSize(), this->_cycleAllocations.load()));
2019-06-06 14:21:15 +02:00
}
}
}