/*
 *     Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
 *
 * NVIDIA CORPORATION and its licensors retain all intellectual property
 * and proprietary rights in and to this software, related documentation
 * and any modifications thereto.  Any use, reproduction, disclosure or
 * distribution of this software and related documentation without an express
 * license agreement from NVIDIA CORPORATION is strictly prohibited.
 *
 */

#ifndef __NVHPC_CUDA_WMMA_RUNTIME_H
#define __NVHPC_CUDA_WMMA_RUNTIME_H

#if !defined(PGI_COMPILE_BITCODE) || defined(__NVHPC_NG) || (__CUDACC_VER_MAJOR__ < 12)
#define __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
#endif

#include <mma.h>

#ifndef NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG
#define NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __device__ static __inline__
#endif

#include "nvhpc_common.h"

#define __HALF_TO_US(var) *(reinterpret_cast<unsigned short *>(&(var)))
#define __HALF2_TO_UI(var) *(reinterpret_cast<unsigned int *>(&(var)))
#define __HALF2_TO_US2(var) *(reinterpret_cast<ushort2 *>(&(var)))
#define __UI_TO_HALF2(var) *(reinterpret_cast<__half2 *>(&(var)))

/* Basically following the order in mma.hpp */
/* Load fragment, m16n16k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m16n16k16_rmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m16n16k16_ld_a((int*)frag, (const int*)p, lda, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m16n16k16_cmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m16n16k16_ld_a((int*)frag, (const int*)p, lda, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m16n16k16_rmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m16n16k16_ld_b((int*)frag, (const int*)p, ldb, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m16n16k16_cmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m16n16k16_ld_b((int*)frag, (const int*)p, ldb, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k16_r2(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m16n16k16_ld_c_f16((int*)frag, (const int*)p, ldc, 0);
  if (layout == 1)
    __hmma_m16n16k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k16_r2_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m16n16k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k16_r4(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m16n16k16_ld_c_f32((float*)frag, (const float*)p, ldc, 0);
  if (layout == 1)
    __hmma_m16n16k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k16_r4_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m16n16k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

/* Load fragment, m32n8k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m32n8k16_rmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m32n8k16_ld_a((int*)frag, (const int*)p, lda, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m32n8k16_cmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m32n8k16_ld_a((int*)frag, (const int*)p, lda, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m32n8k16_rmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m32n8k16_ld_b((int*)frag, (const int*)p, ldb, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m32n8k16_cmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m32n8k16_ld_b((int*)frag, (const int*)p, ldb, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m32n8k16_r2(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m32n8k16_ld_c_f16((int*)frag, (const int*)p, ldc, 0);
  if (layout == 1)
    __hmma_m32n8k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m32n8k16_r2_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m32n8k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m32n8k16_r4(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m32n8k16_ld_c_f32((float*)frag, (const float*)p, ldc, 0);
  if (layout == 1)
    __hmma_m32n8k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m32n8k16_r4_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m32n8k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

/* Load fragment, m8n32k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m8n32k16_rmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m8n32k16_ld_a((int*)frag, (const int*)p, lda, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m8n32k16_cmjr(signed char* frag, signed char* p, int lda)
{
  __hmma_m8n32k16_ld_a((int*)frag, (const int*)p, lda, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m8n32k16_rmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m8n32k16_ld_b((int*)frag, (const int*)p, ldb, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m8n32k16_cmjr(signed char* frag, signed char* p, int ldb)
{
  __hmma_m8n32k16_ld_b((int*)frag, (const int*)p, ldb, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n32k16_r2(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m8n32k16_ld_c_f16((int*)frag, (const int*)p, ldc, 0);
  if (layout == 1)
    __hmma_m8n32k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n32k16_r2_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m8n32k16_ld_c_f16((int*)frag, (const int*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n32k16_r4(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m8n32k16_ld_c_f32((float*)frag, (const float*)p, ldc, 0);
  if (layout == 1)
    __hmma_m8n32k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n32k16_r4_cmjr(signed char* frag, signed char* p, int ldc)
{
  __hmma_m8n32k16_ld_c_f32((float*)frag, (const float*)p, ldc, 1);
}

#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
/* Load fragment, m16n16k8 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m16n16k8_rmjr(signed char* frag, signed char* p, int lda)
{
  __mma_tf32_m16n16k8_ld_a((int*)frag, (const int*)p, lda, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m16n16k8_cmjr(signed char* frag, signed char* p, int lda)
{
  __mma_tf32_m16n16k8_ld_a((int*)frag, (const int*)p, lda, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m16n16k8_rmjr(signed char* frag, signed char* p, int ldb)
{
  __mma_tf32_m16n16k8_ld_b((int*)frag, (const int*)p, ldb, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m16n16k8_cmjr(signed char* frag, signed char* p, int ldb)
{
  __mma_tf32_m16n16k8_ld_b((int*)frag, (const int*)p, ldb, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k8(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __mma_tf32_m16n16k8_ld_c((float*)frag, (const float*)p, ldc, 0);
  if (layout == 1)
    __mma_tf32_m16n16k8_ld_c((float*)frag, (const float*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m16n16k8_cmjr(signed char* frag, signed char* p, int ldc)
{
  __mma_tf32_m16n16k8_ld_c((float*)frag, (const float*)p, ldc, 1);
}

/* Load fragment, m8n8k4 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m8n8k4_rmjr(signed char* frag, signed char* p, int lda)
{
  __dmma_m8n8k4_ld_a((double *)frag, (const double*)p, lda, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_a_m8n8k4_cmjr(signed char* frag, signed char* p, int lda)
{
  __dmma_m8n8k4_ld_a((double *)frag, (const double*)p, lda, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m8n8k4_rmjr(signed char* frag, signed char* p, int ldb)
{
  __dmma_m8n8k4_ld_b((double *)frag, (const double*)p, ldb, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_b_m8n8k4_cmjr(signed char* frag, signed char* p, int ldb)
{
  __dmma_m8n8k4_ld_b((double *)frag, (const double*)p, ldb, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n8k4(signed char* frag, signed char* p, int ldc, int layout)
{
  if (layout == 0)
    __dmma_m8n8k4_ld_c((double *)frag, (const double*)p, ldc, 0);
  if (layout == 1)
    __dmma_m8n8k4_ld_c((double *)frag, (const double*)p, ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_ld_c_m8n8k4_cmjr(signed char* frag, signed char* p, int ldc)
{
  __dmma_m8n8k4_ld_c((double *)frag, (const double*)p, ldc, 1);
}
#endif

/* -------------------------------------------------------------------------- */

/* Store fragment, m16n16k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k16_r2(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m16n16k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m16n16k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k16_r2_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m16n16k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k16_r4(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m16n16k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m16n16k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k16_r4_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m16n16k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

/* Store fragment, m32n8k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m32n8k16_r2(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m32n8k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m32n8k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m32n8k16_r2_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m32n8k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m32n8k16_r4(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m32n8k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m32n8k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m32n8k16_r4_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m32n8k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

/* Store fragment, m8n32k16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n32k16_r2(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m8n32k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m8n32k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n32k16_r2_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m8n32k16_st_c_f16((int*)p, (int*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n32k16_r4(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __hmma_m8n32k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __hmma_m8n32k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n32k16_r4_cmjr(signed char* p, signed char* frag, int ldc)
{
  __hmma_m8n32k16_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
/* Store fragment, m16n16k8 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k8_r4(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __mma_m16n16k8_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __mma_m16n16k8_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m16n16k8_r4_cmjr(signed char* p, signed char* frag, int ldc)
{
  __mma_m16n16k8_st_c_f32((float*)p, (float*)frag, (unsigned int) ldc, 1);
}

/* Store fragment, m8n8k4 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n8k4_r8(signed char* p, signed char* frag, int ldc, int layout)
{
  if (layout == 0)
    __dmma_m8n8k4_st_c_f64((double*)p, (double*)frag, (unsigned int) ldc, 0);
  if (layout == 1)
    __dmma_m8n8k4_st_c_f64((double*)p, (double*)frag, (unsigned int) ldc, 1);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_st_c_m8n8k4_r8_cmjr(signed char* p, signed char* frag, int ldc)
{
  __dmma_m8n8k4_st_c_f64((double*)p, (double*)frag, (unsigned int) ldc, 1);
}
#endif

/* MMA functions for shape m16n16k16, no saturation */
/* -------------------------------------------------------------------------- */
/* D fp16 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}

/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m16n16k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 3, 0);
}

/* D fp32 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}

/* D fp32 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}

/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r4_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m16n16k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 3, 0);
}

/* D fp16 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k16_r2_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m16n16k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}

/* MMA functions for shape m32n8k16, no saturation */
/* -------------------------------------------------------------------------- */
/* D fp16 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}
/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m32n8k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 3, 0);
}

/* D fp32 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}

/* D fp32 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}
/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r4_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m32n8k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 3, 0);
}

/* D fp16 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m32n8k16_r2_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m32n8k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}

/* MMA functions for shape m8n32k16, no saturation */
/* -------------------------------------------------------------------------- */
/* D fp16 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}
/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  int ival = 0;
  ((int*)(d))[0] = ival; ((int*)(d))[1] = ival;
  ((int*)(d))[2] = ival; ((int*)(d))[3] = ival;
  __hmma_m8n32k16_mma_f16f16((int*)d, (int*)a, (int*)b, (int*)d, 3, 0);
}

/* D fp32 C fp16 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_rmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_cmjr_r2(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f16((float*)d, (int*)a, (int*)b, (int*)c, 3, 0);
}

/* D fp32 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}
/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r4_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __hmma_m8n32k16_mma_f32f32((float*)d, (int*)a, (int*)b, (float*)d, 3, 0);
}

/* D fp16 C fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n32k16_r2_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __hmma_m8n32k16_mma_f16f32((int*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}

#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
/* MMA functions for shape m16n16k8, tf32 */
/* -------------------------------------------------------------------------- */
/* A, B, C, and D fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_rmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_rmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_cmjr_rmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_cmjr_cmjr_r4(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)c, 3, 0);
}

/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m16n16k8_r4_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 8; i++ )
    ((float*)(d))[i] = 0.0;
  __mma_tf32_m16n16k8_mma_f32((float*)d, (int*)a, (int*)b, (float*)d, 3, 0);
}

/* MMA functions for shape m8n8k4, double */
/* -------------------------------------------------------------------------- */
/* A, B, C, and D fp32 */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_rmjr_rmjr_r8(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)c, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_rmjr_cmjr_r8(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)c, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_cmjr_rmjr_r8(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)c, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_cmjr_cmjr_r8(signed char* d, signed char* a, signed char* b, signed char* c)
{
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)c, 3, 0);
}

/* 3 operand */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_rmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 2; i++ )
    ((double*)(d))[i] = 0.0;
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)d, 0, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_rmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 2; i++ )
    ((double*)(d))[i] = 0.0;
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)d, 1, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_cmjr_rmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 2; i++ )
    ((double*)(d))[i] = 0.0;
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)d, 2, 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_m8n8k4_r8_cmjr_cmjr(signed char* d, signed char* a, signed char* b)
{
  for (int i = 0; i < 2; i++ )
    ((double*)(d))[i] = 0.0;
  __dmma_m8n8k4_mma_f64((double*)d, (double*)a, (double*)b, (double*)d, 3, 0);
}
#endif

/* That's it for load, store, compute.  The rest are helpers */

/* These were guarded with cplusplus starting in CUDA 9.0 */

/* Assignment and conversion */
/* ------------------------- */
/* These might need pgi definitions because we are not using "half" */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __CONST_PURE float
__pgi_half2float(unsigned short h)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __half2float(__ushort_as_half(h));
#else
  float val;
  asm volatile("cvt.f32.f16 %0, %1;"
               : "=f"(val)  : "h"(h));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __CONST_PURE unsigned short
__pgi_float2us_rn(float x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __half_as_ushort(__float2half_rn(x));
#else
  unsigned short val;
  asm volatile("cvt.rn.f16.f32 %0, %1;"
               : "=h"(val)  : "f"(x));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __CONST_PURE __half
__pgi_float2half_rn(float x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __float2half_rn(x);
#else
  __half val;
  asm volatile("cvt.rn.f16.f32 %0, %1;"
               : "=h"(__HALF_TO_US(val))  : "f"(x));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __CONST_PURE unsigned int
__pgi_float2half2_rn(float x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __float2half2_rn(x);
  return __HALF2_TO_UI(r);
#else
  unsigned int val;
  asm volatile("{.reg .f16 low;\n"
               "cvt.rn.f16.f32 low, %1;\n"
               "mov.b32 %0, {low,low};}\n" : "=r"(val)  : "f"(x));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_real2_assign2real4(signed char *y, signed char *x)
{
  unsigned short a = *((unsigned short *)x);
  ((float *)y)[0] = __pgi_half2float(a);
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_real4_assign2real2(signed char *y, signed char *x)
{
  float a = *((float *)x);
  ((unsigned short *)y)[0] = __pgi_float2us_rn(a);
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_halves2half2(signed char *z, signed char *y, signed char *x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  *(__half2 *)z = __halves2half2(*(__half *)x, *(__half *)y);
#else
  unsigned int val;
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  asm volatile("mov.b32 %0, {%1, %2};"
               : "=r"(val)  : "h"(a), "h"(b));
  ((unsigned int *)z)[0] = val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_make_v2real2(signed char *x, signed char *y)
{
  ushort2 val2;
  val2.x = *((unsigned short *)x);
  val2.y = *((unsigned short *)y);
  return val2;
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG unsigned int
__pgi_make_v2real2from1(unsigned short x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half h = __ushort_as_half(x);
  __half2 r = __halves2half2(h, h);
  return __HALF2_TO_UI(r);
#else
  unsigned int val;
  asm volatile("mov.b32 %0, {%1, %2};"
               : "=r"(val)  : "h"(x), "h"(x));
  return val;
#endif
}

/* Fill routines called either through API or assignment statements */
/* ------------------------------------------------------------------------- */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_accr2(signed char* frag, signed char* val)
{
  /* Fill 8 elements, two at a time */
  unsigned int ival = __pgi_make_v2real2from1(*((unsigned short *)val));
  ((unsigned int*)(frag))[0] = ival;
  ((unsigned int*)(frag))[1] = ival;
  ((unsigned int*)(frag))[2] = ival;
  ((unsigned int*)(frag))[3] = ival;
} /* wmma_fill_frag_accr2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_accr4(signed char* frag, float val)
{
  /* Fill 8 elements, one at a time */
  ((float*)(frag))[0] = val;
  ((float*)(frag))[1] = val;
  ((float*)(frag))[2] = val;
  ((float*)(frag))[3] = val;
  ((float*)(frag))[4] = val;
  ((float*)(frag))[5] = val;
  ((float*)(frag))[6] = val;
  ((float*)(frag))[7] = val;
} /* wmma_fill_frag_accr4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_accr8(signed char* frag, double val)
{
  /* Fill 8 elements, one at a time */
  ((double*)(frag))[0] = val;
  ((double*)(frag))[1] = val;
} /* wmma_fill_frag_accr8 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_ab(signed char* frag, signed char* val)
{
  /* Fill 16 elements, two at a time */
  unsigned int ival = __pgi_make_v2real2from1(*((unsigned short *)val));
  ((unsigned int*)(frag))[0] = ival;
  ((unsigned int*)(frag))[1] = ival;
  ((unsigned int*)(frag))[2] = ival;
  ((unsigned int*)(frag))[3] = ival;
  ((unsigned int*)(frag))[4] = ival;
  ((unsigned int*)(frag))[5] = ival;
  ((unsigned int*)(frag))[6] = ival;
  ((unsigned int*)(frag))[7] = ival;
} /* wmma_fill_frag_ab */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_ab_r4(signed char* frag, float val)
{
  /* Fill 8 elements, one at a time */
  ((float*)(frag))[0] = val;
  ((float*)(frag))[1] = val;
  ((float*)(frag))[2] = val;
  ((float*)(frag))[3] = val;
} /* wmma_fill_frag_ab_r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_fill_frag_ab_r8(signed char* frag, double val)
{
  ((double*)(frag))[0] = val;
} /* wmma_fill_frag_ab_r8 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_copy_frag_accr2(signed char* y, signed char* x)
{
  /* Fill 8 elements, two at a time */
  ((unsigned int*)(y))[0] = ((unsigned int*)(x))[0];
  ((unsigned int*)(y))[1] = ((unsigned int*)(x))[1];
  ((unsigned int*)(y))[2] = ((unsigned int*)(x))[2];
  ((unsigned int*)(y))[3] = ((unsigned int*)(x))[3];
} /* wmma_copy_frag_accr2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_copy_frag_accr4(signed char* y, signed char* x)
{
  /* Fill 8 elements, one at a time */
  ((float*)(y))[0] = ((float*)(x))[0];
  ((float*)(y))[1] = ((float*)(x))[1];
  ((float*)(y))[2] = ((float*)(x))[2];
  ((float*)(y))[3] = ((float*)(x))[3];
  ((float*)(y))[4] = ((float*)(x))[4];
  ((float*)(y))[5] = ((float*)(x))[5];
  ((float*)(y))[6] = ((float*)(x))[6];
  ((float*)(y))[7] = ((float*)(x))[7];
} /* wmma_copy_frag_accr4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_copy_frag_accr8(signed char* y, signed char* x)
{
  /* Fill 8 elements, one at a time */
  ((double*)(y))[0] = ((double*)(x))[0];
  ((double*)(y))[1] = ((double*)(x))[1];
} /* wmma_copy_frag_accr8 */

 /* Add these five lines for building with nvcc to generate .ll file */
 struct __device_builtin__ __align__(8) char8
 {
   signed char x0, y0, z0, w0, x1, y1, z1, w1;
 };
 typedef __device_builtin__ struct char8 char8;
 
/* wmma comparison functions */
/* ------------------------- */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpeq_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __heq2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpeq_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __heq2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpeq_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] == val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpeq_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __heq2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpeq_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] == ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpeq_cm8n8k4r8s(signed char* sa, double val)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] == val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpeq_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] == ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpeq_cm8n8k4r8 */

/* ----- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpne_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hne2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpne_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hne2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpne_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] != val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpne_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hne2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpne_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] != ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpne_cm8n8k4r8s(signed char* sa, double val)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] != val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpne_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] != ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpne_cm8n8k4r8 */

/* ----- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmplt_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hlt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.lt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmplt_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hlt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.lt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmplt_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] < val ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmplt_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hlt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.lt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmplt_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] < ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmplt_cm8n8k4r8s(signed char* sa, double val)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] < val ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmplt_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] < ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmplt_cm8n8k4r8 */

/* ----- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpgt_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hgt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.gt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpgt_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hgt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.gt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpgt_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] > val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpgt_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hgt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.gt.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpgt_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] > ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpgt_cm8n8k4r8s(signed char* sa, double val)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] > val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char2
__pgi_wmma_cmpgt_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char2 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] > ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpgt_cm8n8k4r8 */

/* ----- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hle2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.le.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hle2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.le.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] <= val ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hle2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.le.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] <= ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm8n8k4r8s(signed char* sa, double val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] <= val ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmple_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] <= ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmple_cm8n8k4r8 */

/* ----- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm16n16k16r2s(signed char* sa, float bval)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_float2half2_rn(bval);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hge2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ge.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm16n16k16r2s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm16n16k16r2sr2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, val;
  unsigned int b = __pgi_make_v2real2from1((unsigned short)*sb);
  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hge2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ge.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm16n16k16r4s(signed char* sa, float val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] >= val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm16n16k16r4s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm16n16k16r2(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  unsigned int a, b, val;

  for (int i = 0; i < 4; i++) {
    a = ((unsigned int*)(sa))[i];
    b = ((unsigned int*)(sb))[i];
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    val = __hge2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
    asm volatile("set.ge.f16x2.f16x2 %0, %1, %2;"
                 : "=r"(val)  : "r"(a), "r"(b));
#endif
    lres[2*i]   = (val & 0x0000ffff) ? 1 : 0;
    lres[2*i+1] = (val & 0xffff0000) ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm16n16k16r2 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm16n16k16r4(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 8; i++) {
    lres[i] = ((float*)(sa))[i] >= ((float*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm16n16k16r4 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm8n8k4r8s(signed char* sa, double val)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] >= val ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm8n8k4r8s */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE char8
__pgi_wmma_cmpge_cm8n8k4r8(signed char* sa, signed char* sb)
{
  char8 llres;
  signed char *lres = (signed char *)&llres;
  for (int i = 0; i < 2; i++) {
    lres[i] = ((double*)(sa))[i] >= ((double*)(sb))[i] ? 1 : 0;
  }
  return llres;
} /* wmma_cmpge_cm8n8k4r8 */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG int   __pgi_shfl_xori2(int var, int lanemask);
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG float __pgi_shfl_xorf2(float var, int lanemask);
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG double __pgi_shfl_xord2(double var, int lanemask);

/* Logical reductions        */
/* ------------------------- */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_shfl_wmmalogical(int lcount)
{
  int lnbr = 0;
  lnbr = __pgi_shfl_xori2(lcount, (unsigned int)1);
  lcount = lcount + lnbr;
  lnbr = __pgi_shfl_xori2(lcount, (unsigned int)2);
  lcount = lcount + lnbr;
  lnbr = __pgi_shfl_xori2(lcount, (unsigned int)4);
  lcount = lcount + lnbr;
  lnbr = __pgi_shfl_xori2(lcount, (unsigned int)8);
  lcount = lcount + lnbr;
  lnbr = __pgi_shfl_xori2(lcount, (unsigned int)16);
  lcount = lcount + lnbr;
  return lcount;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_any_wmmalogical1(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 8; i++)
    if (lvals[i]) lcount++;
  return (__pgi_wmma_shfl_wmmalogical(lcount) > 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_all_wmmalogical1(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 8; i++)
    if (lvals[i]) lcount++;
  return (__pgi_wmma_shfl_wmmalogical(lcount) == 256);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_count_wmmalogical1(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 8; i++)
    if (lvals[i]) lcount++;
  return __pgi_wmma_shfl_wmmalogical(lcount);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_any_wmmalogical5(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 2; i++)
    if (lvals[i]) lcount++;
  return (__pgi_wmma_shfl_wmmalogical(lcount) > 0);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_all_wmmalogical5(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 2; i++)
    if (lvals[i]) lcount++;
  return (__pgi_wmma_shfl_wmmalogical(lcount) == 64);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE int
__pgi_wmma_count_wmmalogical5(signed char *lvals)
{
  int lcount = 0;
  for (int i = 0; i < 2; i++)
    if (lvals[i]) lcount++;
  return __pgi_wmma_shfl_wmmalogical(lcount);
}

/* wmma merge functions      */
/* ------------------------- */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r2r2r2m(signed char* d, signed char* t, signed char* f, signed char* l)
{
  for (int i = 0; i < 8; i++) {
    ((unsigned short*)(d))[i] = l[i] ?
                        ((unsigned short*)(t))[i] : ((unsigned short*)(f))[i];
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r2sr2m(signed char* d, signed char* t, signed char* f, signed char* l)
{
  unsigned short ts = ((unsigned short*)(t))[0];
  for (int i = 0; i < 8; i++) {
    ((unsigned short*)(d))[i] = l[i] ? ts : ((unsigned short*)(f))[i];
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r2r2sm(signed char* d, signed char* t, signed char* f, signed char* l)
{
  unsigned short fs = ((unsigned short*)(f))[0];
  for (int i = 0; i < 8; i++) {
    ((unsigned short*)(d))[i] = l[i] ? ((unsigned short*)(t))[i] : fs;
  }
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r2ssm(signed char* d, signed char* t, signed char* f, signed char* l)
{
  unsigned short ts = ((unsigned short*)(t))[0];
  unsigned short fs = ((unsigned short*)(f))[0];
  for (int i = 0; i < 8; i++) {
    ((unsigned short*)(d))[i] = l[i] ? ts : fs;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r4r4r4m(signed char* d, signed char* t, signed char* f, signed char* l)
{
  for (int i = 0; i < 8; i++) {
    ((float*)(d))[i] = l[i] ?
                        ((float*)(t))[i] : ((float*)(f))[i];
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r4sr4m(signed char* d, float t, signed char* f, signed char* l)
{
  for (int i = 0; i < 8; i++) {
    ((float*)(d))[i] = l[i] ? t : ((float*)(f))[i] ;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r4r4sm(signed char* d, signed char* t, float f, signed char* l)
{
  for (int i = 0; i < 8; i++) {
    ((float*)(d))[i] = l[i] ? ((float*)(t))[i] : f;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm16n16k16r4ssm(signed char* d, float t, float f, signed char* l)
{
  for (int i = 0; i < 8; i++) {
    ((float*)(d))[i] = l[i] ? t : f;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm8n8k4r8r8r8m(signed char* d, signed char* t, signed char* f, signed char* l)
{
  for (int i = 0; i < 2; i++) {
    ((double*)(d))[i] = l[i] ?
                        ((double*)(t))[i] : ((double*)(f))[i];
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm8n8k4r8sr8m(signed char* d, double t, signed char* f, signed char* l)
{
  for (int i = 0; i < 2; i++) {
    ((double*)(d))[i] = l[i] ? t : ((double*)(f))[i] ;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm8n8k4r8r8sm(signed char* d, signed char* t, double f, signed char* l)
{
  for (int i = 0; i < 2; i++) {
    ((double*)(d))[i] = l[i] ? ((double*)(t))[i] : f;
  }
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_wmma_merge_cm8n8k4r8ssm(signed char* d, double t, double f, signed char* l)
{
  for (int i = 0; i < 2; i++) {
    ((double*)(d))[i] = l[i] ? t : f;
  }
}

/* arithmetic on real(2) */
/* -------------------------------------------------------------------------*/

/* Trouble with LLVM caused me to make these return __half */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_add_real2(signed char *x, signed char *y)
{
  __half val;
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  val = __hadd(*(__half *)x, *(__half *)y);
#else
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  asm volatile("add.f16 %0, %1, %2;"
               : "=h"(__HALF_TO_US(val))  : "h"(a), "h"(b));
#endif
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_sub_real2(signed char *x, signed char *y)
{
  __half val;
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  val = __hsub(*(__half *)x, *(__half *)y);
#else
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  asm volatile("sub.f16 %0, %1, %2;"
               : "=h"(__HALF_TO_US(val))  : "h"(a), "h"(b));
#endif
  return val;
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_mul_real2(signed char *x, signed char *y)
{
  __half val;
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  val = __hmul(*(__half *)x, *(__half *)y);
#else
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  asm volatile("mul.f16 %0, %1, %2;"
               : "=h"(__HALF_TO_US(val))  : "h"(a), "h"(b));
#endif
  return val;
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_div_real2(signed char *x, signed char *y)
{
  __half val;
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  val = __hdiv(*(__half *)x, *(__half *)y);
#else
  float fa, fb, fv, rcp;
  unsigned short abs;
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  const unsigned short den = 0x008f;

  fa = __pgi_half2float(a);
  fb = __pgi_half2float(b);

  asm volatile("rcp.approx.f32 %0, %1;" : "=f"(rcp)  : "f"(fb));

  fv = rcp * fa;
  val = __pgi_float2half_rn(fv);
  abs = __HALF_TO_US(val) & 0x7fff;
  if ((abs < den) && (!(abs == 0x0))) {
    float err = __fmaf_rn(-fb, fv, fa);
    fv = __fmaf_rn(rcp, err, fv);
    val = __pgi_float2half_rn(fv);
  }
#endif
  return val;
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_fma_real2(signed char *x, signed char *y, signed char *z)
{
  __half val;
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  val = __hfma(*(__half *)x, *(__half *)y, *(__half *)z);
#else
  const unsigned short a = *((unsigned short *)x);
  const unsigned short b = *((unsigned short *)y);
  const unsigned short c = *((unsigned short *)z);
  asm volatile("fma.rn.f16 %0, %1, %2, %3;"
               : "=h"(__HALF_TO_US(val))  : "h"(a), "h"(b), "h"(c));
#endif
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_realint_real2(int x, int kind)
{
  __half h;
  __HALF_TO_US(h) = (unsigned short) 0;
  if (kind == 2) {
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
    h = __int2half_rn(x);
#else
    asm volatile("cvt.rn.f16.s32 %0, %1;" : "=h"(__HALF_TO_US(h)) : "r"(x));
#endif
  }
  return h;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE __half
__pgi_wmma_realreal_real2(float x, int kind)
{
  __half h;
  __HALF_TO_US(h) = (unsigned short) 0;
  if (kind == 2) {
    h = __pgi_float2half_rn(x);
  }
  return h;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_wmma_realreal2_real(signed char *x, int kind)
{
  unsigned short a = *((unsigned short *)x);
  return __pgi_half2float(a);
}

/* vector(2) of real(2) */
/* -------------------------------------------------------------------------*/
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_negate_v2real2(signed char *x)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  unsigned int a = 0x0;
  __half2 r = __hsub2(__UI_TO_HALF2(a), *(__half2 *)x);
  return __HALF2_TO_US2(r);
#else
  ushort2 val;
  const unsigned int a = 0x0;
  const unsigned int b = *((unsigned int *)x);
  asm volatile("sub.f16x2 %0, %1, %2;"
               : "=r"(__HALF2_TO_UI(val))  : "r"(a), "r"(b));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_add_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __hadd2(*(__half2 *)x, *(__half2 *)y);
  return __HALF2_TO_US2(r);
#else
  ushort2 val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("add.f16x2 %0, %1, %2;"
               : "=r"(__HALF2_TO_UI(val))  : "r"(a), "r"(b));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_sub_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __hsub2(*(__half2 *)x, *(__half2 *)y);
return __HALF2_TO_US2(r);
#else
  ushort2 val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("sub.f16x2 %0, %1, %2;"
               : "=r"(__HALF2_TO_UI(val))  : "r"(a), "r"(b));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_mul_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __hmul2(*(__half2 *)x, *(__half2 *)y);
  return __HALF2_TO_US2(r);
#else
  ushort2 val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("mul.f16x2 %0, %1, %2;"
               : "=r"(__HALF2_TO_UI(val))  : "r"(a), "r"(b));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_div_v2real2(signed char *x, signed char *y)
{
  __half a, b;
  a = __pgi_wmma_div_real2(x, y);
  b = __pgi_wmma_div_real2(x+2, y+2);
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __halves2half2(a, b);
  return __HALF2_TO_US2(r);
#else
  ushort2 val;
  asm volatile("mov.b32 %0, {%1, %2};"
                : "=r"(__HALF2_TO_UI(val))  : "h"(__HALF_TO_US(a)), "h"(__HALF_TO_US(b)));
  return val;
#endif
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE ushort2
__pgi_fma_v2real2(signed char *x, signed char *y, signed char*z)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  __half2 r = __hfma2(*(__half2 *)x, *(__half2 *)y, *(__half2 *)z);
  return __HALF2_TO_US2(r);
#else
  ushort2 val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  const unsigned int c = *((unsigned int *)z);
  asm volatile("fma.rn.f16x2 %0, %1, %2, %3;"
               : "=r"(__HALF2_TO_UI(val))  : "r"(a), "r"(b), "r"(c));
  return val;
#endif
}

/* comparison of vector(2) of real(2), returning v2logical2 */
/* -------------------------------------------------------------------------*/
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpeq_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __heq2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpne_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hne2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpgt_v2real2_byvalue(unsigned int a, unsigned int b)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hgt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
  unsigned int val;
  asm volatile("set.gt.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpgt_v2real2(signed char *x, signed char *y)
{
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  return __pgi_wmma_cmpgt_v2real2_byvalue(a, b);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmplt_v2real2_byvalue(unsigned int a, unsigned int b)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hlt2_mask(__UI_TO_HALF2(a), __UI_TO_HALF2(b));
#else
  unsigned int val;
  asm volatile("set.lt.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmplt_v2real2(signed char *x, signed char *y)
{
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  return __pgi_wmma_cmplt_v2real2_byvalue(a, b);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpge_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hge2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.ge.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmple_v2real2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hle2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.le.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

/* comparison of vector(2) of real(2) and real(4), returning v2logical2 */
/* -------------------------------------------------------------------------*/
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpeq_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __heq2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpne_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hne2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpgt_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hgt2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.gt.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmplt_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hlt2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.lt.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmpge_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hge2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.ge.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_wmma_cmple_v2realr(signed char *x, float y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hle2_mask(*(__half2 *)x, __float2half2_rn(y));
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = __pgi_float2half2_rn(y);
  asm volatile("set.le.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

/* logical expressions of v2logical2 type */
/* -------------------------------------------------------------------------*/
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_lognot_v2logical2(signed char *x)
{
  unsigned int val = *((unsigned int *)x);
  return (val ^ 0x3c003c00);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_logand_v2logical2(signed char *x, signed char *y)
{
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  val = a & b;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_logor_v2logical2(signed char *x, signed char *y)
{
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  val = a | b;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_logeqv_v2logical2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __heq2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.eq.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_logneqv_v2logical2(signed char *x, signed char *y)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return __hne2_mask(*(__half2 *)x, *(__half2 *)y);
#else
  unsigned int val;
  const unsigned int a = *((unsigned int *)x);
  const unsigned int b = *((unsigned int *)y);
  asm volatile("set.ne.f16x2.f16x2 %0, %1, %2;"
               : "=r"(val)  : "r"(a), "r"(b));
  return val;
#endif
}

/* compression of v2logical2 using any, all, and count */
/* -------------------------------------------------------------------------*/
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_any_v2logical2(signed char *x)
{
  unsigned int val = *((unsigned int *)x);
  val = (val & 0xffff0000) || (val & 0x0000ffff);
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_all_v2logical2(signed char *x)
{
  unsigned int val = *((unsigned int *)x);
  val = (val & 0xffff0000) && (val & 0x0000ffff);
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned int
__pgi_count_v2logical2(signed char *x)
{
  unsigned int val = *((unsigned int *)x);
  unsigned int count = 0;
  if (val & 0xffff0000) count++;
  if (val & 0x0000ffff) count++;
  return count;
}

/* Maxval */
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned short
__pgi_maxval_cm16n16k16r2(signed char* x)
{
  unsigned int val = ((unsigned int *)(x))[0];
  unsigned int y, cres;
  for (int i = 1; i < 4; i++) {
    y = ((unsigned int *)(x))[i];
    cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
    if ((cres & 0x0000ffff)!=0) cres = cres | 0x0000ffff;
    if ((cres & 0xffff0000)!=0) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  for (int i = 1; i < 32; i*=2) {
    y = __pgi_shfl_xori2(val, (unsigned int)i);
    cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
    if ((cres & 0x0000ffff)!=0) cres = cres | 0x0000ffff;
    if ((cres & 0xffff0000)!=0) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  y = ((val & 0xffff0000) >> 16) | ((val & 0x0000ffff) << 16);
  cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
  if ((cres & 0x0000ffff)!=0) 
    return ((unsigned short) (val & 0x0000ffff));
  else
    return ((unsigned short) (  y & 0x0000ffff));
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_maxval_cm16n16k16r4(signed char* frag)
{
  float val, nbr;
  val = ((float*)(frag))[0];
  for (int i = 1; i < 8; i++) 
    if (((float*)(frag))[i] > val) val  = ((float*)(frag))[i];
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr > val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned short
__pgi_maxval_cm16n16k16r2_wm(signed char* x, signed char* mask)
{
  unsigned int val = 0xfc00fc00;  /* -Inf */
  unsigned int y, cres;
  for (int i = 0; i < 4; i++) {
    y = ((unsigned int *)(x))[i];
    cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
    if (((cres & 0x0000ffff)!=0) || (*mask++ == 0)) cres = cres | 0x0000ffff;
    if (((cres & 0xffff0000)!=0) || (*mask++ == 0)) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  for (int i = 1; i < 32; i*=2) {
    y = __pgi_shfl_xori2(val, (unsigned int)i);
    cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
    if ((cres & 0x0000ffff)!=0) cres = cres | 0x0000ffff;
    if ((cres & 0xffff0000)!=0) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  y = ((val & 0xffff0000) >> 16) | ((val & 0x0000ffff) << 16);
  cres = __pgi_wmma_cmpgt_v2real2_byvalue(val, y);
  if ((cres & 0x0000ffff)!=0) 
    return ((unsigned short) (val & 0x0000ffff));
  else
    return ((unsigned short) (  y & 0x0000ffff));
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_maxval_cm16n16k16r4_wm(signed char* frag, signed char* mask)
{
  float val, nbr;
  if (mask[0])
    val = ((float*)(frag))[0];
  else
    val = __int_as_float(0xff7fffff);
  for (int i = 1; i < 8; i++) {
    if (mask[i]) {
      if (((float*)(frag))[i] > val) val  = ((float*)(frag))[i];
    }
  }
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr > val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE double
__pgi_maxval_cm8n8k4r8(signed char* frag)
{
  double val, nbr;
  val = ((double*)(frag))[0];
  if (((double*)(frag))[1] > val) val  = ((double*)(frag))[1];
  nbr = __pgi_shfl_xord2(val, (unsigned int)1);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)2);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)4);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)8);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)16);
  if (nbr > val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE double
__pgi_maxval_cm8n8k4r8_wm(signed char* frag, signed char *mask)
{
  double val, nbr;
  if (mask[0])
    val = ((double*)(frag))[0];
  else
    val = __longlong_as_double(0xffefffffffffffffll);
  if (mask[1]) {
    if (((double*)(frag))[1] > val) val  = ((double*)(frag))[1];
  }
  nbr = __pgi_shfl_xord2(val, (unsigned int)1);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)2);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)4);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)8);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)16);
  if (nbr > val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_maxabsval_cm16n16k16r4(signed char* frag)
{
  float val, nbr;
  val = ((float*)(frag))[0];
  if (val < 0.0) val = -val;
  for (int i = 1; i < 8; i++) {
    nbr = ((float*)(frag))[i];
    if (nbr < 0.0) nbr = -nbr;
    if (nbr > val) val  = nbr;
  }
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr > val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr > val) val = nbr;
  return val;
}

/* ---- */

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE unsigned short
__pgi_minval_cm16n16k16r2(signed char* x)
{
  unsigned int val = ((unsigned int *)(x))[0];
  unsigned int y, cres;
  for (int i = 1; i < 4; i++) {
    y = ((unsigned int *)(x))[i];
    cres = __pgi_wmma_cmplt_v2real2_byvalue(val, y);
    if ((cres & 0x0000ffff)!=0) cres = cres | 0x0000ffff;
    if ((cres & 0xffff0000)!=0) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  for (int i = 1; i < 32; i*=2) {
    y = __pgi_shfl_xori2(val, (unsigned int)i);
    cres = __pgi_wmma_cmplt_v2real2_byvalue(val, y);
    if ((cres & 0x0000ffff)!=0) cres = cres | 0x0000ffff;
    if ((cres & 0xffff0000)!=0) cres = cres | 0xffff0000;
    val = (val & cres) | (y & ~cres);
  }
  y = ((val & 0xffff0000) >> 16) | ((val & 0x0000ffff) << 16);
  cres = __pgi_wmma_cmplt_v2real2_byvalue(val, y);
  if ((cres & 0x0000ffff)!=0) 
    return ((unsigned short) (val & 0x0000ffff));
  else
    return ((unsigned short) (  y & 0x0000ffff));
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_minval_cm16n16k16r4(signed char* frag)
{
  float val, nbr;
  val = ((float*)(frag))[0];
  for (int i = 1; i < 8; i++) 
    if (((float*)(frag))[i] < val) val  = ((float*)(frag))[i];
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr < val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_minval_cm16n16k16r4_wm(signed char* frag, signed char* mask)
{
  float val, nbr;
  if (mask[0])
    val = ((float*)(frag))[0];
  else
    val = __int_as_float(0x7f7fffff);
  for (int i = 1; i < 8; i++) {
    if (mask[i]) {
      if (((float*)(frag))[i] < val) val  = ((float*)(frag))[i];
    }
  }
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr < val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE double
__pgi_minval_cm8n8k4r8(signed char* frag)
{
  double val, nbr;
  val = ((double*)(frag))[0];
  if (((double*)(frag))[1] < val) val  = ((double*)(frag))[1];
  nbr = __pgi_shfl_xord2(val, (unsigned int)1);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)2);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)4);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)8);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)16);
  if (nbr < val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE double
__pgi_minval_cm8n8k4r8_wm(signed char* frag, signed char *mask)
{
  double val, nbr;
  if (mask[0])
    val = ((double*)(frag))[0];
  else
    val = __longlong_as_double(0x7fefffffffffffffll);
  if (mask[1]) {
    if (((double*)(frag))[1] < val) val  = ((double*)(frag))[1];
  }
  nbr = __pgi_shfl_xord2(val, (unsigned int)1);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)2);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)4);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)8);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xord2(val, (unsigned int)16);
  if (nbr < val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __PURE float
__pgi_minabsval_cm16n16k16r4(signed char* frag)
{
  float val, nbr;
  val = ((float*)(frag))[0];
  if (val < 0.0) val = -val;
  for (int i = 1; i < 8; i++) {
    nbr = ((float*)(frag))[i];
    if (nbr < 0.0) nbr = -nbr;
    if (nbr < val) val  = nbr;
  }
  nbr = __pgi_shfl_xorf2(val, (unsigned int)1);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)2);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)4);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)8);
  if (nbr < val) val = nbr;
  nbr = __pgi_shfl_xorf2(val, (unsigned int)16);
  if (nbr < val) val = nbr;
  return val;
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_assgn1_v2real2(signed char *y, signed char *x)
{
  /* real(2), dimension(2), assigned to type(v2real2) */
  unsigned int algn = ((unsigned long long)((void *)x)) & 0xffff;
  if (__ffs(algn | 0x10) > 2) {
    ((unsigned int *)y)[0] = ((unsigned int *)x)[0];
  } else {
    __pgi_halves2half2(y, x, x+2);
  }
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_assgn2_v2real2(signed char *y, signed char *x)
{
  /* type(v2real2) assigned to real(2), dimension(2) */
  unsigned int algn = ((unsigned long long)((void *)y)) & 0xffff;
  if (__ffs(algn | 0x10) > 2) {
    ((unsigned int *)y)[0] = ((unsigned int *)x)[0];
  } else {
    ((unsigned short *)y)[0] = ((unsigned short *)x)[0];
    ((unsigned short *)y)[1] = ((unsigned short *)x)[1];
  }
}
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_assgn3_v2real2(signed char *y, float x)
{
  /* float assigned to both halves of type(v2real2) */
  ((unsigned int *)y)[0] = __pgi_float2half2_rn(x);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_assgn_v4real4(signed char *y, signed char *x)
{
  ((float4 *)y)[0] = ((float4 *)x)[0];
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__pgi_assgn_v8real2(signed char *y, signed char *x)
{
  ((unsigned long long *)y)[0] = ((unsigned long long *)x)[0];
  ((unsigned long long *)y)[1] = ((unsigned long long *)x)[1];
}

#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG float
__pgi_float_to_tf32(float in)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return nvcuda::wmma::__float_to_tf32(in);
#else
  float ret;
  asm("{\n  .reg .b32 __$1;"
      "\n   cvt.rna.tf32.f32 __$1, %1;"
      "\n   mov.b32 %0, __$1;\n}\n" : "=f"(ret) : "f"(in) );
  return ret;
#endif
}
#endif

#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800

#include "cuda_pipeline.h"

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__nvf_wmma_memcpy_async_r16(signed char* dst, signed char* p)
{
  _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_memcpy_async<16, 16>(
                         (void*)dst, (const void*)p);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__nvf_wmma_memcpy_async_r8(signed char* dst, signed char* p)
{
  _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_memcpy_async<8, 8>(
                         (void*)dst, (const void*)p);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__nvf_wmma_memcpy_async_r4(signed char* dst, signed char* p)
{
  _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_memcpy_async<4, 4>(
                         (void*)dst, (const void*)p);
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__nvf_wmma_pipeline_commit()
{
  _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_commit();
}

NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG void
__nvf_wmma_pipeline_wait_prior(size_t prior)
{
    switch (prior) {
    case  0 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<0>(); return;
    case  1 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<1>(); return;
    case  2 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<2>(); return;
    case  3 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<3>(); return;
    case  4 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<4>(); return;
    case  5 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<5>(); return;
    case  6 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<6>(); return;
    case  7 : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<7>(); return;
    default : _CUDA_PIPELINE_INTERNAL_NAMESPACE::pipeline_wait_prior<8>(); return;
    }
}

// This function is already defined within clang under `include/__clang_cuda_intrinsics.h`
#if !defined(__clang__)
NVHPC_CUDA_WMMA_RUNTIME_DEV_SIG __attribute__((nv_weak)) uint32_t
__nvvm_get_smem_pointer(void *p)
{
#ifndef __NVHPC_CUDA_WMMA_RUNTIME_USE_PTX
  return (uint32_t)__cvta_generic_to_shared(p);
#else
  uint32_t res;
  uint64_t val;
  asm volatile("cvta.to.shared.u64 %0, %1;"
               : "=l"(val)  : "l"(p));
  asm volatile("cvt.u32.u64 %0, %1;"
               : "=r"(res)  : "l"(val));
  return res;
#endif
}
#endif

#endif

#endif // __NVHPC_CUDA_WMMA_RUNTIME_H
