cavis/libnd4j/include/ops/declarable/helpers/cpu/softmax.cpp

/* ******************************************************************************
 *
 *
 * This program and the accompanying materials are made available under the
 * terms of the Apache License, Version 2.0 which is available at
 * https://www.apache.org/licenses/LICENSE-2.0.
 *
 *  See the NOTICE file distributed with this work for additional
 *  information regarding copyright ownership.
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 ******************************************************************************/

//
// @author Yurii Shyrma (iuriish@yahoo.com), created on 19.04.2018
// @author raver119@gmail.com
//

#include <ops/declarable/helpers/activations.h>
#include <helpers/ShapeUtils.h>
#include <numeric>
#include <helpers/ConstantTadHelper.h>
#include <execution/Threads.h>

namespace sd {
    namespace ops {
        namespace helpers {

            template <typename T>
            static void softMaxForVector_(void const* input, Nd4jLong const* inShapeInfo, void *output, Nd4jLong const* outShapeInfo) {

                auto inBuff  = reinterpret_cast<T const*>(input);
                auto outBuff = reinterpret_cast<T *>(output);

                T max = -DataTypeUtils::max<T>();
                T sum = 0.;
                int inEWS = shape::elementWiseStride(inShapeInfo);
                int outEWS = shape::elementWiseStride(outShapeInfo);
                int length = shape::length(inShapeInfo);

                if (inEWS >= 1 && outEWS >= 1) {

                    if (inEWS == 1 && outEWS == 1) {

                        for (int i = 0; i < length; i++)
                            max = sd::math::nd4j_max<T>(max, inBuff[i]);

                        for (int i = 0; i < length; i++) {
                            outBuff[i] = sd::math::nd4j_exp<T, T>(inBuff[i] - max);
                            sum += outBuff[i];
                        }

                        for (int i = 0; i < length; i++)
                            outBuff[i] /= sum;
                    }
                    else {

                        for (int i = 0; i < length; i++)
                            max = sd::math::nd4j_max<T>(max, inBuff[i * inEWS]);

                        for (int i = 0; i < length; i++) {
                            T r = sd::math::nd4j_exp<T, T>(inBuff[i * inEWS] - max);
                            outBuff[i * outEWS] = r;
                            sum += r;
                        }

                        for (int i = 0; i < length; i++)
                            outBuff[i * outEWS] /= sum;
                    }
                }
            }

            ///////////////////////////////////////////////////////////////////
            void softMaxForVector(sd::LaunchContext * context, const NDArray& input, NDArray& output) {

                if(!input.isVector() || !output.isVector())
                    throw std::runtime_error("ops::helpers::softMaxForVector function: input and output arrays must be vectors !");

                auto xType = input.dataType();
                BUILD_SINGLE_SELECTOR(xType, softMaxForVector_, (input.buffer(), input.shapeInfo(), output.buffer(), output.shapeInfo()), FLOAT_TYPES);
            }

            template <typename T>
            void softmax_loop(const T* input, T *output, const Nd4jLong * offsets, Nd4jLong numOfSubArrs, uint32_t tadLen);

#ifdef _OPENMP
            template <>
            FORCEINLINE void softmax_loop(const float* input, float *output, const Nd4jLong * offsets, Nd4jLong numOfSubArrs, uint32_t tadLen) {
#pragma omp parallel for default(shared)
                    for (Nd4jLong i = 0; i < numOfSubArrs; i++) {
                        auto inBuff = input + offsets[i];
                        auto outBuff = output + offsets[i];

                        float max = -DataTypeUtils::max<float>();
                        float sum = 0.f;

#pragma omp simd reduction(max:max)
                        for (uint j = 0; j < tadLen; ++j)
                            max = sd::math::nd4j_max<float>(max, inBuff[j]);

#pragma omp simd reduction(+:sum)
                        for (uint j = 0; j < tadLen; ++j) {
                            float temp = sd::math::nd4j_exp<float, float>(inBuff[j] - max);
                            outBuff[j] = temp;
                            sum += temp;
                        }

                        for (uint j = 0; j < tadLen; ++j)
                            outBuff[j] /= sum;
                    }
            }
#else
            template <>
            FORCEINLINE void softmax_loop(const float *input, float *output, const Nd4jLong *offsets, Nd4jLong numOfSubArrs, uint32_t tadLen) {
                auto func = PRAGMA_THREADS_FOR {
                    for (auto i = start; i < stop; i++) {
                        auto inBuff = input + offsets[i];
                        auto outBuff = output + offsets[i];

                        float max = -DataTypeUtils::max<float>();
                        float sum = 0.f;

                        for (uint j = 0; j < tadLen; ++j)
                            max = sd::math::nd4j_max<float>(max, inBuff[j]);

                        for (uint j = 0; j < tadLen; ++j) {
                            float temp = sd::math::nd4j_exp<float, float>(inBuff[j] - max);
                            outBuff[j] = temp;
                            sum += temp;
                        }

                        for (uint j = 0; j < tadLen; ++j)
                            outBuff[j] /= sum;
                    }
                };

                samediff::Threads::parallel_tad(func,0, numOfSubArrs);
            }

#endif


            template <typename T>
            FORCEINLINE void softmax_loop(const T *input, T *output, const Nd4jLong *offsets, Nd4jLong numOfSubArrs, uint32_t tadLen) {
                auto func = PRAGMA_THREADS_FOR {
                    for (auto i = start; i < stop; i++) {
                        auto inBuff = input + offsets[i];
                        auto outBuff = output + offsets[i];

                        T max = -DataTypeUtils::max<T>();
                        T sum(0.f);

#pragma omp simd reduction(maxT:max)
                        for (uint j = 0; j < tadLen; ++j)
                            max = sd::math::nd4j_max<T>(max, inBuff[j]);

#pragma omp simd reduction(sumT:sum)
                        for (uint j = 0; j < tadLen; ++j) {
                            T temp = sd::math::nd4j_exp<T, T>(inBuff[j] - max);
                            outBuff[j] = temp;
                            sum += temp;
                        }

                        for (uint j = 0; j < tadLen; ++j)
                            outBuff[j] /= sum;
                    }
                };

                samediff::Threads::parallel_tad(func,0, numOfSubArrs);
            }

//////////////////////////////////////////////////////////////////////////
            template <typename T>
            static void softmax_(sd::LaunchContext * context, const NDArray& input, NDArray& output, const int dimension) {

                const int rank = input.rankOf();

                if(input.isVector()) {

                    if(rank == 1 || input.sizeAt(dimension) != 1)
                        softMaxForVector_<T>(input.buffer(), input.shapeInfo(), output.buffer(), output.shapeInfo());
                    else
                        output = 1.;
                }
                else if(input.isSameShapeStrict(output)) {

                    TadPack tadPack  = sd::ConstantTadHelper::getInstance().tadForDimensions(input.shapeInfo(), dimension);
                    auto tadShapeInfo  = tadPack.primaryShapeInfo();
                    auto tadOffsets    = tadPack.primaryOffsets();
                    const uint numOfSubArrs = tadPack.numberOfTads();
                    const uint tadLen       = shape::length(tadShapeInfo);

                    if(shape::elementWiseStride(tadShapeInfo) == 1){
                        auto inBuff = input.bufferAsT<T>();
                        T *outBuff = output.bufferAsT<T>();

                        softmax_loop(inBuff, outBuff, tadOffsets, numOfSubArrs, tadLen);
                    }
                    else {

                        uint inShapeInfoCast[MAX_RANK];
                        bool canCast = sd::DataTypeUtils::castShapeInfo(tadShapeInfo, inShapeInfoCast);

                        auto offsets = new Nd4jLong[tadLen];
                        shape::calcOffsets(tadShapeInfo, offsets);

                        auto func = PRAGMA_THREADS_FOR {
                            for (auto i = start; i < stop; i++) {
                                auto inBuff = input.bufferAsT<T>() + tadOffsets[i];
                                auto outBuff = output.bufferAsT<T>() + tadOffsets[i];

                                T max = -DataTypeUtils::max<T>();
                                T sum = 0.f;

                                for (uint j = 0; j < tadLen; ++j)
                                    max = sd::math::nd4j_max<T>(max, inBuff[offsets[j]]);

                                for (uint j = 0; j < tadLen; ++j) {
                                    T temp = sd::math::nd4j_exp<T, T>(inBuff[offsets[j]] - max);
                                    outBuff[offsets[j]] = temp;
                                    sum += temp;
                                }

                                for (uint j = 0; j < tadLen; ++j)
                                    outBuff[offsets[j]] /= sum;
                            }
                        };

                        samediff::Threads::parallel_tad(func, 0, numOfSubArrs);

                        delete []offsets;
                    }
                }
                else {
                    NDArray max = input.reduceAlongDimension(sd::reduce::Max, {dimension}, true);
                    input.applyTrueBroadcast(sd::BroadcastOpsTuple::Subtract(), max, output, false);
                    output.applyTransform(sd::transform::Exp, output);
                    NDArray sum = output.reduceAlongDimension(sd::reduce::Sum, {dimension}, true);
                    output /= sum;
                }
            }


            ///////////////////////////////////////////////////////////////////
            void softmax(sd::LaunchContext * context, const NDArray& input, NDArray& output, const int dimension) {

                BUILD_SINGLE_SELECTOR(input.dataType(), softmax_, (context, input, output, dimension), FLOAT_TYPES);
            }

        }
    }
}