|
| template<Index size> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::calcColLoops (const bfloat16 *&indexA, Index &row, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexB, Index strideB, Index offsetA, Index offsetB, Index bigSuffix, float *result) |
| |
| template<typename LhsMapper , typename RhsMapper , bool linear> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::calcVecColLoops (Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<typename LhsMapper , typename RhsMapper > |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::calcVecLoops (Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows> |
| void | Eigen::internal::colLoopBody (Index &col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *indexB, Index strideB, Index offsetB, float *result) |
| |
| template<const Index num_packets, bool rhsExtraCols, bool lhsExtraRows> |
| void | Eigen::internal::colLoopBodyExtra (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result) |
| |
| template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colLoopBodyExtraN (Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result) |
| |
| template<const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows, bool multiIter = false> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colLoopBodyIter (Index depth, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *indexB, Index strideB, Index offsetB, float *result, const Index extra_cols, const Index extra_rows) |
| |
| template<const Index num_packets, bool lhsExtraRows = false> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colLoops (Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16 *indexA, const bfloat16 *blockB, Index strideB, Index offsetB, float *result) |
| |
| template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear> |
| void | Eigen::internal::colVecColLoopBody (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<typename LhsMapper , typename RhsMapper , bool extraRows, bool linear> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colVecColLoopBodyExtra (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<const Index num_acc, typename LhsMapper , typename RhsMapper , bool extraRows, bool linear> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colVecColLoopBodyExtraN (Index &row, Index cend, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<const Index num_acc, typename LhsMapper , typename RhsMapper > |
| void | Eigen::internal::colVecLoopBody (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<typename LhsMapper , typename RhsMapper > |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colVecLoopBodyExtra (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<const Index num_acc, typename LhsMapper , typename RhsMapper > |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::colVecLoopBodyExtraN (Index &row, Index cols, Index rows, LhsMapper &lhs, RhsMapper &rhs, const Packet4f pAlpha, float *result) |
| |
| template<typename DataMapper > |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::convertArrayF32toBF16 (float *result, Index cols, Index rows, const DataMapper &res) |
| |
| template<typename DataMapper , const Index size> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::convertArrayF32toBF16Col (float *result, Index col, Index rows, const DataMapper &res) |
| |
| template<bool non_unit_stride = false> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::convertArrayPointerF32toBF16 (float *result, Index rows, bfloat16 *dst, Index resInc=1) |
| |
| EIGEN_ALWAYS_INLINE Packet8bf | Eigen::internal::convertF32toBF16 (const float *res) |
| |
| template<const Index size, bool non_unit_stride = false> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::convertPointerF32toBF16 (Index &i, float *result, Index rows, bfloat16 *&dst, Index resInc=1) |
| |
| template<Index num_acc> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::disassembleAccumulators (__vector_quad(&quad_acc)[num_acc], Packet4f(&acc)[num_acc][4]) |
| |
| template<typename DataMapper > |
| void | Eigen::internal::gemmMMAbfloat16 (const DataMapper &res, const bfloat16 *indexA, const bfloat16 *indexB, Index rows, Index depth, Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) |
| |
| template<typename LhsMapper , typename RhsMapper > |
| void | Eigen::internal::gemvMMA_bfloat16_col (Index rows, Index cols, const LhsMapper &alhs, const RhsMapper &rhs, bfloat16 *res, Index resIncr, bfloat16 alpha) |
| |
| template<typename LhsMapper , typename RhsMapper > |
| void | Eigen::internal::gemvMMA_bfloat16_row (Index rows, Index cols, const LhsMapper &alhs, const RhsMapper &rhs, bfloat16 *res, Index resIncr, bfloat16 alpha) |
| |
| template<Index num_acc, Index num_packets, bool zero, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::KLoop (const bfloat16 *indexA, const bfloat16 *indexB, __vector_quad(&quad_acc)[num_acc], Index strideB, Index k, Index offsetB, Index extra_cols, Index extra_rows) |
| |
| template<bool zero> |
| EIGEN_ALWAYS_INLINE Packet8bf | Eigen::internal::loadBfloat16 (const bfloat16 *indexA) |
| |
| template<bool zero> |
| EIGEN_ALWAYS_INLINE Packet8bf | Eigen::internal::loadRhsBfloat16 (const bfloat16 *blockB, Index strideB, Index i) |
| |
| template<Index num_acc, typename LhsMapper , bool zero> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::loadVecLoop (Index k, LhsMapper &lhs, Packet8bf(&a0)[num_acc], Packet8bf b1) |
| |
| template<Index num_acc> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::multVec (__vector_quad(&quad_acc)[num_acc], Packet8bf(&a0)[num_acc], Packet8bf b0) |
| |
| template<Index num_acc, typename LhsMapper , typename RhsMapper , bool extra> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::multVecLoop (__vector_quad(&quad_acc)[num_acc], const LhsMapper &lhs, RhsMapper &rhs, Index j, Index extra_cols) |
| |
| template<Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::outputResults (Packet4f(&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float *result, const Index extra_cols, Index extra_rows) |
| |
| template<Index num_acc> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::preduxVecResults (Packet4f(&acc)[num_acc][4]) |
| |
| template<Index num_acc> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::preduxVecResults2 (Packet4f(&acc)[num_acc][4], Index k) |
| |
| template<Index num_acc, typename LhsMapper , typename RhsMapper , bool zero, bool linear> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::vecColLoop (Index j, LhsMapper &lhs, RhsMapper &rhs, __vector_quad(&quad_acc)[num_acc]) |
| |
| template<Index num_acc, typename LhsMapper , typename RhsMapper > |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::vecLoop (Index cols, const LhsMapper &lhs, RhsMapper &rhs, __vector_quad(&quad_acc)[num_acc], Index extra_cols) |
| |
| template<Index num_acc> |
| EIGEN_ALWAYS_INLINE void | Eigen::internal::zeroAccumulators (__vector_quad(&quad_acc)[num_acc]) |
| |