cavis/libnd4j/include/array/impl/DataBuffer.cpp

339 lines
11 KiB
C++

/*******************************************************************************
* Copyright (c) 2015-2018 Skymind, Inc.
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
* SPDX-License-Identifier: Apache-2.0
******************************************************************************/
//
// @author raver119@gmail.com
// @author Yurii Shyrma (iuriish@yahoo.com)
//
#include <array/DataBuffer.h>
#include <helpers/logger.h>
#include <array/DataTypeUtils.h>
#include <execution/AffinityManager.h>
#include <memory/MemoryCounter.h>
#include <exceptions/allocation_exception.h>
namespace sd {
///// IMLEMENTATION OF COMMON METHODS /////
////////////////////////////////////////////////////////////////////////
// default constructor
DataBuffer::DataBuffer() {
_primaryBuffer = nullptr;
_specialBuffer = nullptr;
_lenInBytes = 0;
_dataType = INT8;
_workspace = nullptr;
_isOwnerPrimary = false;
_isOwnerSpecial = false;
_deviceId = sd::AffinityManager::currentDeviceId();
setCountersToZero();
}
////////////////////////////////////////////////////////////////////////
// copy constructor
DataBuffer::DataBuffer(const DataBuffer &other) {
throw std::runtime_error("DataBuffer copy constructor: we don't expect using of this constructor!");
_lenInBytes = other._lenInBytes;
_dataType = other._dataType;
_workspace = other._workspace;
_primaryBuffer = nullptr;
_specialBuffer = nullptr;
_deviceId.store(other._deviceId.load());
setCountersToZero();
allocateBuffers();
copyBufferFrom(other);
}
////////////////////////////////////////////////////////////////////////
DataBuffer::DataBuffer(void* primary, void* special,
const size_t lenInBytes, const DataType dataType,
const bool isOwnerPrimary, const bool isOwnerSpecial,
memory::Workspace* workspace) {
if (primary == nullptr && special == nullptr)
throw std::runtime_error("DataBuffer constructor: can't be initialized with both nullptr buffers !");
_primaryBuffer = primary;
_specialBuffer = special;
_lenInBytes = lenInBytes;
_dataType = dataType;
_workspace = workspace;
_isOwnerPrimary = isOwnerPrimary;
_isOwnerSpecial = isOwnerSpecial;
_deviceId = sd::AffinityManager::currentDeviceId();
setCountersToZero();
if(primary != nullptr)
readPrimary();
if(special != nullptr)
readSpecial();
}
////////////////////////////////////////////////////////////////////////
DataBuffer::DataBuffer(void* primary, const size_t lenInBytes, const DataType dataType, const bool isOwnerPrimary, memory::Workspace* workspace):
DataBuffer(primary, nullptr, lenInBytes, dataType, isOwnerPrimary, false, workspace) {
syncToSpecial(true);
}
////////////////////////////////////////////////////////////////////////
// copies data from hostBuffer to own memory buffer
DataBuffer::DataBuffer(const void* hostBuffer, const DataType dataType, const size_t lenInBytes, memory::Workspace* workspace) {
if (hostBuffer == nullptr)
throw std::runtime_error("DataBuffer constructor: can't be initialized with nullptr host buffer !");
if (lenInBytes == 0)
throw std::runtime_error("DataBuffer constructor: can't be initialized with zero length !");
_primaryBuffer = nullptr;
_specialBuffer = nullptr;
_lenInBytes = lenInBytes;
_dataType = dataType;
_workspace = workspace;
_deviceId = sd::AffinityManager::currentDeviceId();
setCountersToZero();
allocateBuffers();
copyBufferFromHost(hostBuffer, lenInBytes);
}
////////////////////////////////////////////////////////////////////////
DataBuffer::DataBuffer(const size_t lenInBytes, const DataType dataType, memory::Workspace* workspace, const bool allocBoth) {
_dataType = dataType;
_workspace = workspace;
_lenInBytes = lenInBytes;
_primaryBuffer = nullptr;
_specialBuffer = nullptr;
_deviceId = sd::AffinityManager::currentDeviceId();
setCountersToZero();
if(lenInBytes != 0) {
allocateBuffers(allocBoth);
writeSpecial();
}
}
////////////////////////////////////////////////////////////////////////
// move constructor
DataBuffer::DataBuffer(DataBuffer&& other) {
_primaryBuffer = other._primaryBuffer;
_specialBuffer = other._specialBuffer;
_lenInBytes = other._lenInBytes;
_dataType = other._dataType;
_workspace = other._workspace;
_isOwnerPrimary = other._isOwnerPrimary;
_isOwnerSpecial = other._isOwnerSpecial;
_deviceId.store(other._deviceId);
copyCounters(other);
other._primaryBuffer = other._specialBuffer = nullptr;
other.setAllocFlags(false, false);
other._lenInBytes = 0;
}
////////////////////////////////////////////////////////////////////////
// assignment operator
DataBuffer& DataBuffer::operator=(const DataBuffer& other) {
if (this == &other)
return *this;
deleteBuffers();
_lenInBytes = other._lenInBytes;
_dataType = other._dataType;
_workspace = other._workspace;
allocateBuffers();
copyBufferFrom(other);
return *this;
}
////////////////////////////////////////////////////////////////////////
// move assignment operator
DataBuffer& DataBuffer::operator=(DataBuffer&& other) noexcept {
if (this == &other)
return *this;
deleteBuffers();
_primaryBuffer = other._primaryBuffer;
_specialBuffer = other._specialBuffer;
_lenInBytes = other._lenInBytes;
_dataType = other._dataType;
_workspace = other._workspace;
_isOwnerPrimary = other._isOwnerPrimary;
_isOwnerSpecial = other._isOwnerSpecial;
copyCounters(other);
other._primaryBuffer = other._specialBuffer = nullptr;
other.setAllocFlags(false, false);
other._lenInBytes = 0;
return *this;
}
////////////////////////////////////////////////////////////////////////
void* DataBuffer::primary() {
return _primaryBuffer;
}
////////////////////////////////////////////////////////////////////////
void* DataBuffer::special() {
return _specialBuffer;
}
////////////////////////////////////////////////////////////////////////
DataType DataBuffer::getDataType() {
return _dataType;
}
////////////////////////////////////////////////////////////////////////
size_t DataBuffer::getLenInBytes() const {
return _lenInBytes;
}
////////////////////////////////////////////////////////////////////////
void DataBuffer::allocatePrimary() {
if (_primaryBuffer == nullptr && getLenInBytes() > 0) {
auto deviceId = sd::AffinityManager::currentDeviceId();
// check if this allocation won't bring us above limit
if (_workspace == nullptr) {
if (Environment::getInstance()->isCPU()) {
// on cpu backend we validate against device 0 for now
if (!sd::memory::MemoryCounter::getInstance()->validate(getLenInBytes()))
throw sd::allocation_exception::build("Requested amount exceeds HOST device limits", sd::memory::MemoryCounter::getInstance()->deviceLimit(deviceId), getLenInBytes());
} else {
// in heterogenous mode we valdate against device group
if (!sd::memory::MemoryCounter::getInstance()->validateGroup(sd::memory::MemoryType::HOST, getLenInBytes()))
throw sd::allocation_exception::build("Requested amount exceeds HOST group limits", sd::memory::MemoryCounter::getInstance()->groupLimit(sd::memory::MemoryType::HOST), getLenInBytes());
}
}
ALLOCATE(_primaryBuffer, _workspace, getLenInBytes(), int8_t);
_isOwnerPrimary = true;
// count in towards current deviceId if we're not in workspace mode
if (_workspace == nullptr) {
if (Environment::getInstance()->isCPU()) // we don't want this counter to be added to CUDA device
sd::memory::MemoryCounter::getInstance()->countIn(deviceId, getLenInBytes());
sd::memory::MemoryCounter::getInstance()->countIn(sd::memory::MemoryType::HOST, getLenInBytes());
}
}
}
////////////////////////////////////////////////////////////////////////
void DataBuffer::setAllocFlags(const bool isOwnerPrimary, const bool isOwnerSpecial) {
_isOwnerPrimary = isOwnerPrimary;
_isOwnerSpecial = isOwnerSpecial;
}
////////////////////////////////////////////////////////////////////////
void DataBuffer::deletePrimary() {
if(_isOwnerPrimary && _primaryBuffer != nullptr && getLenInBytes() != 0) {
auto p = reinterpret_cast<int8_t*>(_primaryBuffer);
RELEASE(p, _workspace);
_primaryBuffer = nullptr;
_isOwnerPrimary = false;
// count out towards DataBuffer device, only if we're not in workspace
if (_workspace == nullptr) {
if (Environment::getInstance()->isCPU())
sd::memory::MemoryCounter::getInstance()->countOut(_deviceId, getLenInBytes());
sd::memory::MemoryCounter::getInstance()->countOut(sd::memory::MemoryType::HOST, getLenInBytes());
}
}
}
////////////////////////////////////////////////////////////////////////
void DataBuffer::deleteBuffers() {
deletePrimary();
deleteSpecial();
_lenInBytes = 0;
}
////////////////////////////////////////////////////////////////////////
DataBuffer::~DataBuffer() {
deleteBuffers();
}
void DataBuffer::setPrimaryBuffer(void *buffer, size_t length) {
if (_primaryBuffer != nullptr && _isOwnerPrimary) {
deletePrimary();
}
_primaryBuffer = buffer;
_isOwnerPrimary = false;
_lenInBytes = length * DataTypeUtils::sizeOf(_dataType);
}
void DataBuffer::setSpecialBuffer(void *buffer, size_t length) {
if (_specialBuffer != nullptr && _isOwnerSpecial) {
deleteSpecial();
}
this->setSpecial(buffer, false);
_lenInBytes = length * DataTypeUtils::sizeOf(_dataType);
}
void DataBuffer::setDataType(DataType dataType) {
_dataType = dataType;
}
int DataBuffer::deviceId() const {
return _deviceId.load();
}
void DataBuffer::close() {
this->deleteBuffers();
}
void DataBuffer::setDeviceId(int deviceId) {
_deviceId = deviceId;
}
}