/*
 *     Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 *
 * NVIDIA CORPORATION and its licensors retain all intellectual property
 * and proprietary rights in and to this software, related documentation
 * and any modifications thereto.  Any use, reproduction, disclosure or
 * distribution of this software and related documentation without an express
 * license agreement from NVIDIA CORPORATION is strictly prohibited.
 *
 */

#ifndef __NVHPC_UTILS_RUNTIME_H
#define __NVHPC_UTILS_RUNTIME_H

#include <stdarg.h>

#if !defined(PGI_COMPILE_BITCODE) || defined(__NVHPC_NG) || (__CUDACC_VER_MAJOR__ < 12)
#define __NVHPC_CUDA_UTILS_USE_PTX
#endif

#ifndef __NVHPC_CUDA_UTILS_USE_PTX
#include <cuda_fp16.h>
#include <cuda/ptx>
#if (CC >= 70)
#include <cuda/std/barrier>
#endif
#endif

#include "nvhpc_common.h"

#ifndef NVHPC_UTILRT_DEV_SIG
#define NVHPC_UTILRT_DEV_SIG __device__ static __inline__
#else
NVHPC_UTILRT_DEV_SIG void __pgi_memcpy(signed char *dst, signed char *src,
        size_t num_bytes);
NVHPC_UTILRT_DEV_SIG void pgf90_sect_alldim_i8(void *pd, void *pa, long long *lower,
        long long *upper, long long *stride, long long flags, int rank, int wrank);
#endif

NVHPC_UTILRT_DEV_SIG void
pgf90_str_copy_klen(int n, signed char *to, long long to_len, ...)
{
  va_list va;
  signed char *from, *tmpbuf, *savbuf;
  long long from_len, dumb_len;

  if (to_len <= 0) return;

  va_start(va, to_len);
  if (n == 1) {
    /* N==1 is the most common case */
    from = va_arg(va, signed char *);
    from_len = va_arg(va, long long);
    if (from_len < 0) from_len = 0;
    /* Do not expect overlap to occur */
    if ((from < to) && ((from+from_len) > to)) {
      tmpbuf = (signed char *) malloc(from_len);
      __pgi_memcpy(tmpbuf, from, from_len);
      from = tmpbuf;
    } else {
      tmpbuf = 0;
    }
    while (to_len-- > 0) {
      if (from_len > 0) {
        *to++ = *from++;
        from_len--;
      } else
        *to++ = ' ';
    }
    if (tmpbuf) free(tmpbuf);
  } else {
    /* N>1 is not a common case, just use dumb algorithm, one buffer */
    dumb_len = to_len;
    tmpbuf = (signed char *) malloc(dumb_len);
    savbuf = tmpbuf;
    while (n-- > 0) {
      from = va_arg(va, signed char *);
      from_len = va_arg(va, long long);
      if (from_len > dumb_len) from_len = dumb_len;
      if (from_len > 0) {
        __pgi_memcpy(tmpbuf, from, (size_t) from_len);
        tmpbuf += from_len;
        dumb_len -= from_len;
      }
    }
    while (dumb_len-- > 0)
      *tmpbuf++ = ' ';
    __pgi_memcpy(to, savbuf, to_len);
    free(savbuf);
  }
}

NVHPC_UTILRT_DEV_SIG void
pgf90_sect_i8(void *pd, void *pa, void *prank, ...)
{
  va_list va;
  long long lower[7], upper[7], stride[7], flags;
  int ax, rank, wrank;
  typedef struct F90_DumDesc_la {
    long long tag;
    long long rank;
  } F90_DumDesc_la;

  /* Just need to pull the rank out of this descriptor */
  F90_DumDesc_la *a = (F90_DumDesc_la *)pa;

  va_start(va, prank);
  wrank = (int)(*((long long *)(prank)));
  for (ax = 0; ax < wrank; ++ax) {
    lower[ax] = *va_arg(va, long long *);
    upper[ax] = *va_arg(va, long long *);
    stride[ax] = *va_arg(va, long long *);
  }
  flags = *va_arg(va, long long *);
  va_end(va);

  rank = (flags & 0x55) + (flags >> 1 & 0x15);
  rank = (rank & 0x33) + (rank >> 2 & 0x13);
  rank += rank >> 4;
  rank &= 0x7;
  /* Just need to pull the rank out of this descriptor */
  wrank = a->rank;
  
  pgf90_sect_alldim_i8(pd, pa, lower, upper, stride, flags, rank, wrank);
}

/* Device versions for loads/stores with cache operators */

/* ldca */
NVHPC_UTILRT_DEV_SIG __PURE int
__ldca_i4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldca((int *)ptr);
#else
  unsigned int ret;
  asm ("ld.global.ca.s32 %0, [%1];"  : "=r"(ret) : "l"(ptr) : "memory");
  return (int)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE long long
__ldca_i8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldca((long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.ca.s64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return (long long)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned long long
__ldca_cd(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldca((unsigned long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.ca.u64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned short
__ldca_r2(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __half ret = __ldca((__half *)ptr);
  return __half_as_ushort(ret);
#else
  unsigned short ret;
  asm ("ld.global.ca.b16 %0, [%1];"  : "=h"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE float
__ldca_r4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldca((float *)ptr);
#else
  float ret;
  asm ("ld.global.ca.f32 %0, [%1];"  : "=f"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE double
__ldca_r8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldca((double *)ptr);
#else
  double ret;
  asm ("ld.global.ca.f64 %0, [%1];"  : "=d"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_c4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float2 *)dst = __ldca((const float2 *)src);
#else
  float rx, ry;
  asm ("ld.global.ca.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE cmplx2
__ldca_c4x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  float2 r = __ldca((const float2 *)src);
  cmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  cmplx2 ret;
  float rx, ry;
  asm ("ld.global.ca.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_c8_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldca((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.ca.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE dcmplx2
__ldca_c8x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  double2 r = __ldca((const double2 *)src);
  dcmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  dcmplx2 ret;
  double rx, ry;
  asm ("ld.global.ca.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_i4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(int4 *)dst = __ldca((const int4 *)src);
#else
  int rx, ry, rz, rw;
  asm ("ld.global.ca.v4.s32 {%0,%1,%2,%3}, [%4];"  : "=r"(rx), "=r"(ry), "=r"(rz), "=r"(rw) : "l"(src) : "memory");
  ((int *)dst)[0] = rx;
  ((int *)dst)[1] = ry;
  ((int *)dst)[2] = rz;
  ((int *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_i8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(longlong2 *)dst = __ldca((const longlong2 *)src);
#else
  long long rx, ry;
  asm ("ld.global.ca.v2.s64 {%0,%1}, [%2];"  : "=l"(rx), "=l"(ry) : "l"(src) : "memory");
  ((long long *)dst)[0] = rx;
  ((long long *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_r2x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(__half2 *)dst = __ldca((const __half2 *)src);
#else
  unsigned int rx;
  asm ("ld.global.ca.b32 {%0}, [%1];"  : "=r"(rx) : "l"(src) : "memory");
  ((unsigned int *)dst)[0] = rx;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_r4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float4 *)dst = __ldca((const float4 *)src);
#else
  float rx, ry, rz, rw;
  asm ("ld.global.ca.v4.f32 {%0,%1,%2,%3}, [%4];"  : "=f"(rx), "=f"(ry), "=f"(rz), "=f"(rw) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
  ((float *)dst)[2] = rz;
  ((float *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldca_r8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldca((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.ca.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

/* ldcg */
NVHPC_UTILRT_DEV_SIG __PURE int
__ldcg_i4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcg((int *)ptr);
#else
  unsigned int ret;
  asm ("ld.global.cg.s32 %0, [%1];"  : "=r"(ret) : "l"(ptr) : "memory");
  return (int)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE long long
__ldcg_i8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcg((long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cg.s64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return (long long)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned long long
__ldcg_cd(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcg((unsigned long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cg.u64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned short
__ldcg_r2(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __half ret = __ldcg((__half *)ptr);
  return __half_as_ushort(ret);
#else
  unsigned short ret;
  asm ("ld.global.cg.b16 %0, [%1];"  : "=h"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE float
__ldcg_r4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcg((float *)ptr);
#else
  float ret;
  asm ("ld.global.cg.f32 %0, [%1];"  : "=f"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE double
__ldcg_r8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcg((double *)ptr);
#else
  double ret;
  asm ("ld.global.cg.f64 %0, [%1];"  : "=d"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_c4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float2 *)dst = __ldcg((const float2 *)src);
#else
  float rx, ry;
  asm ("ld.global.cg.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE cmplx2
__ldcg_c4x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  float2 r = __ldcg((const float2 *)src);
  cmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  cmplx2 ret;
  float rx, ry;
  asm ("ld.global.cg.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_c8_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcg((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cg.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE dcmplx2
__ldcg_c8x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  double2 r = __ldcg((const double2 *)src);
  dcmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  dcmplx2 ret;
  double rx, ry;
  asm ("ld.global.cg.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_i4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(int4 *)dst = __ldcg((const int4 *)src);
#else
  int rx, ry, rz, rw;
  asm ("ld.global.cg.v4.s32 {%0,%1,%2,%3}, [%4];"  : "=r"(rx), "=r"(ry), "=r"(rz), "=r"(rw) : "l"(src) : "memory");
  ((int *)dst)[0] = rx;
  ((int *)dst)[1] = ry;
  ((int *)dst)[2] = rz;
  ((int *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_i8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(longlong2 *)dst = __ldcg((const longlong2 *)src);
#else
  long long rx, ry;
  asm ("ld.global.cg.v2.s64 {%0,%1}, [%2];"  : "=l"(rx), "=l"(ry) : "l"(src) : "memory");
  ((long long *)dst)[0] = rx;
  ((long long *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_r2x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(__half2 *)dst = __ldcg((const __half2 *)src);
#else
  unsigned int rx;
  asm ("ld.global.cg.b32 {%0}, [%1];"  : "=r"(rx) : "l"(src) : "memory");
  ((unsigned int *)dst)[0] = rx;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_r4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float4 *)dst = __ldcg((const float4 *)src);
#else
  float rx, ry, rz, rw;
  asm ("ld.global.cg.v4.f32 {%0,%1,%2,%3}, [%4];"  : "=f"(rx), "=f"(ry), "=f"(rz), "=f"(rw) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
  ((float *)dst)[2] = rz;
  ((float *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcg_r8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcg((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cg.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

/* ldcs */
NVHPC_UTILRT_DEV_SIG __PURE int
__ldcs_i4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcs((int *)ptr);
#else
  unsigned int ret;
  asm ("ld.global.cs.s32 %0, [%1];"  : "=r"(ret) : "l"(ptr) : "memory");
  return (int)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE long long
__ldcs_i8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcs((long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cs.s64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return (long long)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned long long
__ldcs_cd(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcs((unsigned long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cs.u64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned short
__ldcs_r2(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __half ret = __ldcs((__half *)ptr);
  return __half_as_ushort(ret);
#else
  unsigned short ret;
  asm ("ld.global.cs.b16 %0, [%1];"  : "=h"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE float
__ldcs_r4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcs((float *)ptr);
#else
  float ret;
  asm ("ld.global.cs.f32 %0, [%1];"  : "=f"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE double
__ldcs_r8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcs((double *)ptr);
#else
  double ret;
  asm ("ld.global.cs.f64 %0, [%1];"  : "=d"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_c4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float2 *)dst = __ldcs((const float2 *)src);
#else
  float rx, ry;
  asm ("ld.global.cs.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE cmplx2
__ldcs_c4x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  float2 r = __ldcs((const float2 *)src);
  cmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  cmplx2 ret;
  float rx, ry;
  asm ("ld.global.cs.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_c8_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcs((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cs.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE dcmplx2
__ldcs_c8x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  double2 r = __ldcs((const double2 *)src);
  dcmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  dcmplx2 ret;
  double rx, ry;
  asm ("ld.global.cs.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_i4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(int4 *)dst = __ldcs((const int4 *)src);
#else
  int rx, ry, rz, rw;
  asm ("ld.global.cs.v4.s32 {%0,%1,%2,%3}, [%4];"  : "=r"(rx), "=r"(ry), "=r"(rz), "=r"(rw) : "l"(src) : "memory");
  ((int *)dst)[0] = rx;
  ((int *)dst)[1] = ry;
  ((int *)dst)[2] = rz;
  ((int *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_i8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(longlong2 *)dst = __ldcs((const longlong2 *)src);
#else
  long long rx, ry;
  asm ("ld.global.cs.v2.s64 {%0,%1}, [%2];"  : "=l"(rx), "=l"(ry) : "l"(src) : "memory");
  ((long long *)dst)[0] = rx;
  ((long long *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_r2x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(__half2 *)dst = __ldcs((const __half2 *)src);
#else
  unsigned int rx;
  asm ("ld.global.cs.b32 {%0}, [%1];"  : "=r"(rx) : "l"(src) : "memory");
  ((unsigned int *)dst)[0] = rx;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_r4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float4 *)dst = __ldcs((const float4 *)src);
#else
  float rx, ry, rz, rw;
  asm ("ld.global.cs.v4.f32 {%0,%1,%2,%3}, [%4];"  : "=f"(rx), "=f"(ry), "=f"(rz), "=f"(rw) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
  ((float *)dst)[2] = rz;
  ((float *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcs_r8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcs((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cs.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

/* ldlu */
NVHPC_UTILRT_DEV_SIG __PURE int
__ldlu_i4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldlu((int *)ptr);
#else
  unsigned int ret;
  asm ("ld.global.lu.s32 %0, [%1];"  : "=r"(ret) : "l"(ptr) : "memory");
  return (int)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE long long
__ldlu_i8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldlu((long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.lu.s64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return (long long)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned long long
__ldlu_cd(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldlu((unsigned long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.lu.u64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned short
__ldlu_r2(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __half ret = __ldlu((__half *)ptr);
  return __half_as_ushort(ret);
#else
  unsigned short ret;
  asm ("ld.global.lu.b16 %0, [%1];"  : "=h"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE float
__ldlu_r4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldlu((float *)ptr);
#else
  float ret;
  asm ("ld.global.lu.f32 %0, [%1];"  : "=f"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE double
__ldlu_r8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldlu((double *)ptr);
#else
  double ret;
  asm ("ld.global.lu.f64 %0, [%1];"  : "=d"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_c4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float2 *)dst = __ldlu((const float2 *)src);
#else
  float rx, ry;
  asm ("ld.global.lu.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE cmplx2
__ldlu_c4x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  float2 r = __ldlu((const float2 *)src);
  cmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  cmplx2 ret;
  float rx, ry;
  asm ("ld.global.lu.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_c8_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldlu((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.lu.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE dcmplx2
__ldlu_c8x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  double2 r = __ldlu((const double2 *)src);
  dcmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  dcmplx2 ret;
  double rx, ry;
  asm ("ld.global.lu.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_i4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(int4 *)dst = __ldlu((const int4 *)src);
#else
  int rx, ry, rz, rw;
  asm ("ld.global.lu.v4.s32 {%0,%1,%2,%3}, [%4];"  : "=r"(rx), "=r"(ry), "=r"(rz), "=r"(rw) : "l"(src) : "memory");
  ((int *)dst)[0] = rx;
  ((int *)dst)[1] = ry;
  ((int *)dst)[2] = rz;
  ((int *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_i8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(longlong2 *)dst = __ldlu((const longlong2 *)src);
#else
  long long rx, ry;
  asm ("ld.global.lu.v2.s64 {%0,%1}, [%2];"  : "=l"(rx), "=l"(ry) : "l"(src) : "memory");
  ((long long *)dst)[0] = rx;
  ((long long *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_r2x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(__half2 *)dst = __ldlu((const __half2 *)src);
#else
  unsigned int rx;
  asm ("ld.global.lu.b32 {%0}, [%1];"  : "=r"(rx) : "l"(src) : "memory");
  ((unsigned int *)dst)[0] = rx;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_r4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float4 *)dst = __ldlu((const float4 *)src);
#else
  float rx, ry, rz, rw;
  asm ("ld.global.lu.v4.f32 {%0,%1,%2,%3}, [%4];"  : "=f"(rx), "=f"(ry), "=f"(rz), "=f"(rw) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
  ((float *)dst)[2] = rz;
  ((float *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldlu_r8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldlu((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.lu.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

/* ldcv */
NVHPC_UTILRT_DEV_SIG __PURE int
__ldcv_i4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcv((int *)ptr);
#else
  unsigned int ret;
  asm ("ld.global.cv.s32 %0, [%1];"  : "=r"(ret) : "l"(ptr) : "memory");
  return (int)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE long long
__ldcv_i8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcv((long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cv.s64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return (long long)ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned long long
__ldcv_cd(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcv((unsigned long long *)ptr);
#else
  unsigned long long ret;
  asm ("ld.global.cv.u64 %0, [%1];"  : "=l"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE unsigned short
__ldcv_r2(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __half ret = __ldcv((__half *)ptr);
  return __half_as_ushort(ret);
#else
  unsigned short ret;
  asm ("ld.global.cv.b16 %0, [%1];"  : "=h"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE float
__ldcv_r4(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcv((float *)ptr);
#else
  float ret;
  asm ("ld.global.cv.f32 %0, [%1];"  : "=f"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE double
__ldcv_r8(signed char *ptr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return __ldcv((double *)ptr);
#else
  double ret;
  asm ("ld.global.cv.f64 %0, [%1];"  : "=d"(ret) : "l"(ptr) : "memory");
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_c4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float2 *)dst = __ldcv((const float2 *)src);
#else
  float rx, ry;
  asm ("ld.global.cv.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE cmplx2
__ldcv_c4x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  float2 r = __ldcv((const float2 *)src);
  cmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  cmplx2 ret;
  float rx, ry;
  asm ("ld.global.cv.v2.f32 {%0,%1}, [%2];"  : "=f"(rx), "=f"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_c8_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcv((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cv.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE dcmplx2
__ldcv_c8x(signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  double2 r = __ldcv((const double2 *)src);
  dcmplx2 ret;
  ret.r = r.x;
  ret.i = r.y;
  return ret;
#else
  dcmplx2 ret;
  double rx, ry;
  asm ("ld.global.cv.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ret.r = rx;
  ret.i = ry;
  return ret;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_i4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(int4 *)dst = __ldcv((const int4 *)src);
#else
  int rx, ry, rz, rw;
  asm ("ld.global.cv.v4.s32 {%0,%1,%2,%3}, [%4];"  : "=r"(rx), "=r"(ry), "=r"(rz), "=r"(rw) : "l"(src) : "memory");
  ((int *)dst)[0] = rx;
  ((int *)dst)[1] = ry;
  ((int *)dst)[2] = rz;
  ((int *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_i8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(longlong2 *)dst = __ldcv((const longlong2 *)src);
#else
  long long rx, ry;
  asm ("ld.global.cv.v2.s64 {%0,%1}, [%2];"  : "=l"(rx), "=l"(ry) : "l"(src) : "memory");
  ((long long *)dst)[0] = rx;
  ((long long *)dst)[1] = ry;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_r2x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(__half2 *)dst = __ldcv((const __half2 *)src);
#else
  unsigned int rx;
  asm ("ld.global.cv.b32 {%0}, [%1];"  : "=r"(rx) : "l"(src) : "memory");
  ((unsigned int *)dst)[0] = rx;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_r4x4_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(float4 *)dst = __ldcv((const float4 *)src);
#else
  float rx, ry, rz, rw;
  asm ("ld.global.cv.v4.f32 {%0,%1,%2,%3}, [%4];"  : "=f"(rx), "=f"(ry), "=f"(rz), "=f"(rw) : "l"(src) : "memory");
  ((float *)dst)[0] = rx;
  ((float *)dst)[1] = ry;
  ((float *)dst)[2] = rz;
  ((float *)dst)[3] = rw;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__ldcv_r8x2_(signed char *dst, signed char *src)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  *(double2 *)dst = __ldcv((const double2 *)src);
#else
  double rx, ry;
  asm ("ld.global.cv.v2.f64 {%0,%1}, [%2];"  : "=d"(rx), "=d"(ry) : "l"(src) : "memory");
  ((double *)dst)[0] = rx;
  ((double *)dst)[1] = ry;
#endif
}

/* stwb */
NVHPC_UTILRT_DEV_SIG void
__stwb_i4(signed char *ptr, int value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((int *)ptr, value);
#else
  asm ("st.global.wb.s32 [%0], %1;"  :: "l"(ptr), "r"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_i8(signed char *ptr, long long value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((long long *)ptr, value);
#else
  asm ("st.global.wb.s64 [%0], %1;"  :: "l"(ptr), "l"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_cd(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((unsigned long long *)ptr, *(unsigned long long *)value);
#else
  unsigned long long vx = ((unsigned long long *)value)[0];
  asm ("st.global.wb.u64 [%0], %1;"  :: "l"(ptr), "l"(vx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r2(signed char *ptr, unsigned short value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((__half *)ptr, *(__half *)&value);
#else
  asm ("st.global.wb.b16 [%0], %1;"  :: "l"(ptr), "h"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r4(signed char *ptr, float value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((float *)ptr, value);
#else
  asm ("st.global.wb.f32 [%0], %1;"  :: "l"(ptr), "f"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r8(signed char *ptr, double value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((double *)ptr, value);
#else
  asm ("st.global.wb.f64 [%0], %1;"  :: "l"(ptr), "d"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_c4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((float2 *)ptr, *(const float2 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  asm ("st.global.wb.v2.f32 [%0], {%1,%2};"  :: "l"(ptr), "f"(vx), "f"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_c8(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.wb.v2.f64 [%0], {%1,%2};"  :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_i4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((int4 *)ptr, *(const int4 *)value);
#else
  int vx = ((int *)value)[0];
  int vy = ((int *)value)[1];
  int vz = ((int *)value)[2];
  int vw = ((int *)value)[3];
  asm ("st.global.wb.v4.s32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "r"(vx), "r"(vy), "r"(vz), "r"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_i8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((longlong2 *)ptr, *(const longlong2 *)value);
#else
  long long vx = ((long long *)value)[0];
  long long vy = ((long long *)value)[1];
  asm ("st.global.wb.v2.s64 [%0], {%1,%2};" :: "l"(ptr), "l"(vx), "l"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r2x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((__half2 *)ptr, *(const __half2 *)value);
#else
  int rx = ((int *)value)[0];
  asm ("st.global.wb.b32 [%0], %1;"  :: "l"(ptr), "r"(rx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((float4 *)ptr, *(const float4 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  float vz = ((float *)value)[2];
  float vw = ((float *)value)[3];
  asm ("st.global.wb.v4.f32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "f"(vx), "f"(vy), "f"(vz), "f"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwb_r8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwb((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.wb.v2.f64 [%0], {%1,%2};" :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

/* stcg */
NVHPC_UTILRT_DEV_SIG void
__stcg_i4(signed char *ptr, int value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((int *)ptr, value);
#else
  asm ("st.global.cg.s32 [%0], %1;"  :: "l"(ptr), "r"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_i8(signed char *ptr, long long value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((long long *)ptr, value);
#else
  asm ("st.global.cg.s64 [%0], %1;"  :: "l"(ptr), "l"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_cd(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((unsigned long long *)ptr, *(unsigned long long *)value);
#else
  unsigned long long vx = ((unsigned long long *)value)[0];
  asm ("st.global.cg.u64 [%0], %1;"  :: "l"(ptr), "l"(vx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r2(signed char *ptr, unsigned short value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((__half *)ptr, *(__half *)&value);
#else
  asm ("st.global.cg.b16 [%0], %1;"  :: "l"(ptr), "h"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r4(signed char *ptr, float value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((float *)ptr, value);
#else
  asm ("st.global.cg.f32 [%0], %1;"  :: "l"(ptr), "f"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r8(signed char *ptr, double value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((double *)ptr, value);
#else
  asm ("st.global.cg.f64 [%0], %1;"  :: "l"(ptr), "d"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_c4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((float2 *)ptr, *(const float2 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  asm ("st.global.cg.v2.f32 [%0], {%1,%2};"  :: "l"(ptr), "f"(vx), "f"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_c8(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.cg.v2.f64 [%0], {%1,%2};"  :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_i4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((int4 *)ptr, *(const int4 *)value);
#else
  int vx = ((int *)value)[0];
  int vy = ((int *)value)[1];
  int vz = ((int *)value)[2];
  int vw = ((int *)value)[3];
  asm ("st.global.cg.v4.s32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "r"(vx), "r"(vy), "r"(vz), "r"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_i8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((longlong2 *)ptr, *(const longlong2 *)value);
#else
  long long vx = ((long long *)value)[0];
  long long vy = ((long long *)value)[1];
  asm ("st.global.cg.v2.s64 [%0], {%1,%2};" :: "l"(ptr), "l"(vx), "l"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r2x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((__half2 *)ptr, *(const __half2 *)value);
#else
  int rx = ((int *)value)[0];
  asm ("st.global.cg.b32 [%0], %1;"  :: "l"(ptr), "r"(rx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((float4 *)ptr, *(const float4 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  float vz = ((float *)value)[2];
  float vw = ((float *)value)[3];
  asm ("st.global.cg.v4.f32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "f"(vx), "f"(vy), "f"(vz), "f"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcg_r8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcg((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.cg.v2.f64 [%0], {%1,%2};" :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

/* stcs */
NVHPC_UTILRT_DEV_SIG void
__stcs_i4(signed char *ptr, int value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((int *)ptr, value);
#else
  asm ("st.global.cs.s32 [%0], %1;"  :: "l"(ptr), "r"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_i8(signed char *ptr, long long value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((long long *)ptr, value);
#else
  asm ("st.global.cs.s64 [%0], %1;"  :: "l"(ptr), "l"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_cd(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((unsigned long long *)ptr, *(unsigned long long *)value);
#else
  unsigned long long vx = ((unsigned long long *)value)[0];
  asm ("st.global.cs.u64 [%0], %1;"  :: "l"(ptr), "l"(vx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r2(signed char *ptr, unsigned short value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((__half *)ptr, *(__half *)&value);
#else
  asm ("st.global.cs.b16 [%0], %1;"  :: "l"(ptr), "h"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r4(signed char *ptr, float value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((float *)ptr, value);
#else
  asm ("st.global.cs.f32 [%0], %1;"  :: "l"(ptr), "f"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r8(signed char *ptr, double value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((double *)ptr, value);
#else
  asm ("st.global.cs.f64 [%0], %1;"  :: "l"(ptr), "d"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_c4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((float2 *)ptr, *(const float2 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  asm ("st.global.cs.v2.f32 [%0], {%1,%2};"  :: "l"(ptr), "f"(vx), "f"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_c8(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.cs.v2.f64 [%0], {%1,%2};"  :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_i4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((int4 *)ptr, *(const int4 *)value);
#else
  int vx = ((int *)value)[0];
  int vy = ((int *)value)[1];
  int vz = ((int *)value)[2];
  int vw = ((int *)value)[3];
  asm ("st.global.cs.v4.s32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "r"(vx), "r"(vy), "r"(vz), "r"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_i8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((longlong2 *)ptr, *(const longlong2 *)value);
#else
  long long vx = ((long long *)value)[0];
  long long vy = ((long long *)value)[1];
  asm ("st.global.cs.v2.s64 [%0], {%1,%2};" :: "l"(ptr), "l"(vx), "l"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r2x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((__half2 *)ptr, *(const __half2 *)value);
#else
  int rx = ((int *)value)[0];
  asm ("st.global.cs.b32 [%0], %1;"  :: "l"(ptr), "r"(rx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((float4 *)ptr, *(const float4 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  float vz = ((float *)value)[2];
  float vw = ((float *)value)[3];
  asm ("st.global.cs.v4.f32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "f"(vx), "f"(vy), "f"(vz), "f"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stcs_r8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stcs((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.cs.v2.f64 [%0], {%1,%2};" :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

/* stwt */
NVHPC_UTILRT_DEV_SIG void
__stwt_i4(signed char *ptr, int value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((int *)ptr, value);
#else
  asm ("st.global.wt.s32 [%0], %1;"  :: "l"(ptr), "r"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_i8(signed char *ptr, long long value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((long long *)ptr, value);
#else
  asm ("st.global.wt.s64 [%0], %1;"  :: "l"(ptr), "l"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_cd(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((unsigned long long *)ptr, *(unsigned long long *)value);
#else
  unsigned long long vx = ((unsigned long long *)value)[0];
  asm ("st.global.wt.u64 [%0], %1;"  :: "l"(ptr), "l"(vx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r2(signed char *ptr, unsigned short value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((__half *)ptr, *(__half *)&value);
#else
  asm ("st.global.wt.b16 [%0], %1;"  :: "l"(ptr), "h"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r4(signed char *ptr, float value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((float *)ptr, value);
#else
  asm ("st.global.wt.f32 [%0], %1;"  :: "l"(ptr), "f"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r8(signed char *ptr, double value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((double *)ptr, value);
#else
  asm ("st.global.wt.f64 [%0], %1;"  :: "l"(ptr), "d"(value) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_c4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((float2 *)ptr, *(const float2 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  asm ("st.global.wt.v2.f32 [%0], {%1,%2};"  :: "l"(ptr), "f"(vx), "f"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_c8(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.wt.v2.f64 [%0], {%1,%2};"  :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_i4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((int4 *)ptr, *(const int4 *)value);
#else
  int vx = ((int *)value)[0];
  int vy = ((int *)value)[1];
  int vz = ((int *)value)[2];
  int vw = ((int *)value)[3];
  asm ("st.global.wt.v4.s32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "r"(vx), "r"(vy), "r"(vz), "r"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_i8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((longlong2 *)ptr, *(const longlong2 *)value);
#else
  long long vx = ((long long *)value)[0];
  long long vy = ((long long *)value)[1];
  asm ("st.global.wt.v2.s64 [%0], {%1,%2};" :: "l"(ptr), "l"(vx), "l"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r2x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((__half2 *)ptr, *(const __half2 *)value);
#else
  int rx = ((int *)value)[0];
  asm ("st.global.wt.b32 [%0], %1;"  :: "l"(ptr), "r"(rx) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r4x4(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((float4 *)ptr, *(const float4 *)value);
#else
  float vx = ((float *)value)[0];
  float vy = ((float *)value)[1];
  float vz = ((float *)value)[2];
  float vw = ((float *)value)[3];
  asm ("st.global.wt.v4.f32 [%0], {%1,%2,%3,%4};" :: "l"(ptr), "f"(vx), "f"(vy), "f"(vz), "f"(vw) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__stwt_r8x2(signed char *ptr, signed char *value)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  __stwt((double2 *)ptr, *(const double2 *)value);
#else
  double vx = ((double *)value)[0];
  double vy = ((double *)value)[1];
  asm ("st.global.wt.v2.f64 [%0], {%1,%2};" :: "l"(ptr), "d"(vx), "d"(vy) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG __PURE int
__nvhpc_syncthreads_count(int pred)
{
  return syncthreads_count(pred);
}

NVHPC_UTILRT_DEV_SIG void
pgf90_mvbits(signed char *from, signed char *frompos, signed char *len,
                 signed char *to, signed char *topos, signed char *szfrom,
                 signed char *szfrompos, signed char *szlen, signed char *sztopos)
{
  int ifrp, ilen, itop;

  if (4 == (*((int *)szfrompos))) 
    ifrp = *((int *)frompos);
  else
    ifrp = (int) *((long long *)frompos);

  if (4 == (*((int *)szlen))) 
    ilen = *((int *)len);
  else
    ilen = (int) *((long long *)len);

  if (4 == (*((int *)sztopos))) 
    itop = *((int *)topos);
  else
    itop = (int) *((long long *)topos);

  if (ifrp < 0 || itop < 0 || ilen <= 0)
    return;

  if (4 == (*((int *)szfrom))) {
    if ((ifrp + ilen) > 32 || (itop + ilen) > 32)
      return;

    if (ilen == 32) {
      *(int *)to = *(int *)from;
    } else {
      int ifr, ito, imsk;
      ifr = *((int *)from);
      ito = *((int *)to);
      imsk = (~(-1 << ilen)) << itop;
      *(int *)to = (ito & ~imsk) | (((ifr >> ifrp) << itop) & imsk);
    }

  } else if (8 == (*((int *)szfrom))) {
    if ((ifrp + ilen) > 64 || (itop + ilen) > 64)
      return;

    if (ilen == 64) {
      *(long long *)to = *(long long *)from;
    } else {
      long long ifr, ito, imsk;
      ifr = *((long long *)from);
      ito = *((long long *)to);
      imsk = (~(-1LL << ilen)) << itop;
      *(long long *)to = (ito & ~imsk) | (((ifr >> ifrp) << itop) & imsk);
    }
  }
}

NVHPC_UTILRT_DEV_SIG __CONST_PURE unsigned long long
__nvf_global_timer()
{
#if defined(__NVHPC_CUDA_UTILS_USE_PTX)
  unsigned long long ret;
  asm("mov.u64 %0, %globaltimer;" : "=l"(ret));
  return ret;
#else
  return cuda::ptx::get_sreg_globaltimer();
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_fence_proxy_async()
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
#else
  asm("fence.proxy.async.shared::cta;");
#endif
}

NVHPC_UTILRT_DEV_SIG __CONST_PURE bool
__nvhpc_block_elect_thread()
{
  // OpenACC compiler only maps loops to 2D threads and always uses thread (0,0,0) as a master thread.
  return (threadIdx.x == 0 && threadIdx.y == 0) ? true: false;
}

/* === CUDA pipelined asynchronous memcpy and prefetch support. */

#ifndef __NVHPC_CUDA_UTILS_USE_PTX
#include <cub/device/dispatch/dispatch_transform.cuh>
#endif

#include <cuda_pipeline_primitives.h>

/* Users need to call the accx_* functions, the compiler will rely on the __nvhpc_* variantss. */
NVHPC_UTILRT_DEV_SIG void
__nvhpc_prefetch_global(void* addr)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cub::detail::transform::prefetch(addr);
#else
  asm volatile("prefetch.global.L2 [%0];" : : "l"(__cvta_generic_to_global(addr)) : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_pipeline_memcpy_async(void* dst, const void* src, size_t size)
{
  __pipeline_memcpy_async(dst, src, size);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_pipeline_commit()
{
  __pipeline_commit();
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_pipeline_wait_prior(size_t count)
{
  __pipeline_wait_prior(count);
}

#if defined(PGI_COMPILE_BITCODE)

#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 && __CUDACC_VER_MAJOR__ < 13
#include <patched/cooperative_groups.h>
#else
#include <cooperative_groups.h>
#endif

namespace cg = cooperative_groups;

NVHPC_UTILRT_DEV_SIG void
__nvf_fastgridsync()
{
    cg::grid_group grid = cg::this_grid();
    cg::sync(grid);
}

// Version used by NGFLANG for syncthreads_grid that will call
// __nvf_fastgridsync.
NVHPC_UTILRT_DEV_SIG void
__nvhpc_grid_syncthreads_grid(signed char *gg)
{
  __nvf_fastgridsync();
}

#else // PGI_COMPILE_BITCODE

NVHPC_UTILRT_DEV_SIG void
__nvf_fastgridsync()
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_grid_syncthreads_grid(signed char *gg)
{
  assert(0); // FIXME
}

#endif // PGI_COMPILE_BITCODE

#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900

#if defined(PGI_COMPILE_BITCODE)

typedef struct __nvf_ClusterType {
  unsigned long long handle;
  unsigned int size;
  unsigned int rank;
} __nvf_ClusterType;

NVHPC_UTILRT_DEV_SIG void
__nvf_cg_syncthreads_cluster()
{
    cg::cluster_group cluster = cg::this_cluster();
    cluster.sync();
}

NVHPC_UTILRT_DEV_SIG void*
__nvf_cg_map_shared_rank(signed char *addr, int rank)
{
    cg::cluster_group cluster = cg::this_cluster();
    return (void *)(cluster.map_shared_rank(addr, rank-1));
}

/* Fortran symbol, for handling the struct arg */
NVHPC_UTILRT_DEV_SIG void
__nvf_cg_this_cluster_(signed char *cg)
{
    __nvf_ClusterType *ncl = (__nvf_ClusterType *)cg;
    cg::cluster_group cluster = cg::this_cluster();
    ncl->size = cluster.num_threads();
    ncl->rank = cluster.block_rank() + 1;

}

NVHPC_UTILRT_DEV_SIG void
__nvf_dimblocks_this_cluster_(signed char *db)
{
    cg::cluster_group cluster = cg::this_cluster();
    dim3 clusterDim = cluster.dim_blocks();
    ((int *)(db))[0] = clusterDim.x;
    ((int *)(db))[1] = clusterDim.y;
    ((int *)(db))[2] = clusterDim.z;

}

NVHPC_UTILRT_DEV_SIG void
__nvf_blockindex_this_cluster_(signed char *bi)
{
  cg::cluster_group cluster = cg::this_cluster();
  dim3 blockIndex = cluster.block_index();
  ((int *)(bi))[0] = blockIndex.x + 1;
  ((int *)(bi))[1] = blockIndex.y + 1;
  ((int *)(bi))[2] = blockIndex.z + 1;
}

NVHPC_UTILRT_DEV_SIG void
__nvf_barrier_arrive_this_cluster_()
{
  cg::cluster_group cluster = cg::this_cluster();
  cluster.barrier_arrive();
}

NVHPC_UTILRT_DEV_SIG void
__nvf_barrier_wait_this_cluster_()
{
  cg::cluster_group cluster = cg::this_cluster();
  cluster.barrier_wait();
}

#else

NVHPC_UTILRT_DEV_SIG void
__nvf_cg_syncthreads_cluster()
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void*
__nvf_cg_map_shared_rank(signed char *addr, int rank)
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvf_cg_this_cluster_(signed char *cg)
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvf_dimblocks_this_cluster_(signed char *db)
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvf_blockindex_this_cluster_(signed char *bi)
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvf_barrier_arrive_this_cluster_()
{
  assert(0); // FIXME
}

NVHPC_UTILRT_DEV_SIG void
__nvf_barrier_wait_this_cluster_()
{
  assert(0); // FIXME
}

#endif // PGI_COMPILE_BITCODE

/* === Bulk loads and stores, and associated Barriers.  1st for CUF */
/* === Still within CUDA_ARCH >= 900                                */

#ifndef __NVHPC_CUDA_UTILS_USE_PTX
#include <cuda_awbarrier_primitives.h>
#endif

/* TMA (Tensor Memory Accelerator) constants for Hopper/Blackwell architectures */
#define TMA_ALIGNMENT 16        /* Required alignment for TMA source addresses (bytes) */
#define TMA_SIZE_MULTIPLE 16    /* Required multiple for TMA transfer sizes (bytes) */
#define TMA_TENSOR_SHMEM_ALIGNMENT 128 /* Required alignment for tensor copy shared memory buffers. */

template <class P>
static __device__ __inline__ P
__nvhpc_align_up(P ptr, size_t alignment)
{
  uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
  uintptr_t aligned = (addr + (alignment - 1)) & ~(alignment - 1);
  return reinterpret_cast<P>(aligned);
}

template <class P>
static __device__ __inline__ P
__nvhpc_align_down(P ptr, size_t alignment)
{
  uintptr_t addr = reinterpret_cast<uintptr_t>(ptr);
  uintptr_t aligned = addr & ~(alignment - 1);
  return reinterpret_cast<P>(aligned);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_g2s(__mbarrier_t *barrier, const void *src, void *dst, uint32_t size)
{
  assert(__isShared(dst));
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::cp_async_bulk(cuda::ptx::space_cluster, cuda::ptx::space_global, dst, src, size, barrier);
#else
  asm("cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [%0], [%1], %2, [%3];"
    :: "r"(static_cast<unsigned int>(__cvta_generic_to_shared(dst))),
        "l"(src), "r"(size), "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier)))
     : "memory");
#endif
}

__device__ static __inline__ void
__nvhpc_tma_bulk_impl(__mbarrier_t *barrier, const void *src, void *dst, uint32_t size)
{
  __nvhpc_tma_bulk_g2s(barrier, src, dst, size);
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
#if (__CUDACC_VER_MAJOR__ >= 13) || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 9)
  cuda::ptx::mbarrier_expect_tx(cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, cuda::ptx::space_shared, barrier, size);
#else
  cuda::device::barrier_expect_tx(*(cuda::barrier<cuda::thread_scope_block> *)barrier, size);
#endif
#else
  asm("mbarrier.expect_tx.relaxed.cta.shared::cta.b64 [%0], %1;"
    :: "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))), "r"(size)
     : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldi4(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 4);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldi8(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldr2(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 2);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldr4(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 4);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldr8(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldc4(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_ldc8(__mbarrier_t *barrier, const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_impl(barrier, src, dst, size * 16);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_s2gp(const void *src, void *dst, int size)
{
  assert(__isShared(src));
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::cp_async_bulk(cuda::ptx::space_global, cuda::ptx::space_shared, dst, src, size);
#else
  asm("cp.async.bulk.global.shared::cta.bulk_group [%0], [%1], %2;"
    :: "l"(dst),
       "r"(static_cast<unsigned int>(__cvta_generic_to_shared(src))), "r"(size)
     : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_commit_group()
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::cp_async_bulk_commit_group();
#else
  asm volatile("cp.async.bulk.commit_group;"
     :
     :
     :);
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_wait_group()
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::cp_async_bulk_wait_group(cuda::ptx::n32_t<0>{});
#else
  asm volatile("cp.async.bulk.wait_group %0;"
     :
     : "n"(0)
     : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_s2g(const void *src, void *dst, int size)
{
  __nvhpc_fence_proxy_async();
  __nvhpc_tma_bulk_s2gp(src, dst, size);
  __nvhpc_tma_bulk_commit_group();
  /* Add this to ensure SMEM can be reused */
  __nvhpc_tma_bulk_wait_group();
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_sti4(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*4);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_sti8(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_str2(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*2);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_str4(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*4);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_str8(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_stc4(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*8);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_bulk_stc8(const void *src, void *dst, int size)
{
  __nvhpc_tma_bulk_s2g(src, dst, size*16);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_s2g_aligned(const void *src, void *dst, uint32_t size_in_bytes)
{
  if (__nvhpc_block_elect_thread())
    __nvhpc_tma_bulk_s2g(src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_s2g(const void *src, void *dst, uint32_t size_in_bytes)
{
  if (__nvhpc_block_elect_thread()) {
    char *aligned_dst = __nvhpc_align_up((char *)dst, TMA_ALIGNMENT);
    char *aligned_end_dst = __nvhpc_align_down((char *)dst + size_in_bytes, TMA_ALIGNMENT);
    const size_t begin_delta =
      (size_t)(aligned_dst - (char *)dst);
    const size_t end_delta =
      (size_t)((char *)dst + size_in_bytes - (char *)aligned_end_dst);
    const size_t aligned_size = (size_t)(aligned_end_dst - aligned_dst);
    memcpy(dst, src, begin_delta);
    memcpy(aligned_end_dst, (const char *)src + begin_delta + aligned_size, end_delta);
    __nvhpc_tma_bulk_s2g((const char *)src + begin_delta, aligned_dst, aligned_size);
  }
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_barrier_initp(__mbarrier_t *barrier, unsigned int count)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::mbarrier_init(barrier, count);
#else
  /* count here is the number of threads participating in the barrier */
  asm volatile("mbarrier.init.shared.b64 [%0], %1;"
    :
    : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))), "r"(count)
    : "memory");
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_barrier_init(__mbarrier_t *barrier, unsigned int count)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  cuda::ptx::mbarrier_init(barrier, count);
  cuda::ptx::fence_proxy_async(cuda::ptx::space_shared);
#else
  asm volatile("mbarrier.init.shared.b64 [%0], %1;"
    :
    : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))), "r"(count)
    : "memory");
  asm("fence.proxy.async.shared::cta;");
#endif
}

NVHPC_UTILRT_DEV_SIG __mbarrier_token_t
__nvhpc_barrier_arrive(__mbarrier_t *barrier, unsigned int tx_count)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return cuda::ptx::mbarrier_arrive_expect_tx(cuda::ptx::sem_release, cuda::ptx::scope_cta, cuda::ptx::space_shared, barrier, tx_count);
#else
  __mbarrier_token_t token; /* This function returns a token */

  /* tx_count here is the transaction count, expected # of bytes to arrive */
  asm volatile("mbarrier.arrive.expect_tx.release.cta.shared::cta.b64 %0, [%1], %2;"
      : "=l"(token)
      : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))),
        "r"(tx_count)
      : "memory");
  return token;
#endif
}

NVHPC_UTILRT_DEV_SIG __mbarrier_token_t
__nvhpc_barrier_arrive_nocnt(__mbarrier_t *barrier)
{
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return cuda::ptx::mbarrier_arrive(barrier, 1);
#else
  __mbarrier_token_t token; /* This function returns a token */
  const unsigned int tx_count = 1;

  /* tx_count here is the transaction count, expected # of bytes to arrive */
  asm volatile("mbarrier.arrive.shared::cta.b64 %0, [%1], %2;"
      : "=l"(token)
      : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))),
        "r"(tx_count)
      : "memory");
  return token;
#endif
}

NVHPC_UTILRT_DEV_SIG unsigned int
__nvhpc_barrier_try_wait_sleep(__mbarrier_t *barrier,
                          __mbarrier_token_t token, int ns) {
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return nvcuda::experimental::__awbarrier_internal::awbarrier_try_wait(barrier, token, ns) ? 1 : 0;
#else                          
  unsigned int __ready = 0;
  asm volatile("{\n\t"
                ".reg .pred p;\n\t"
                "mbarrier.try_wait.shared.b64 p, [%1], %2, %3;\n\t"
                "selp.b32 %0, 1, 0, p;\n\t"
                "}"
                : "=r"(__ready)
                : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))),
                  "l"(token),
                  "r"(static_cast<uint32_t>(ns))
                : "memory");
  return __ready;
#endif
}

NVHPC_UTILRT_DEV_SIG unsigned int
__nvhpc_barrier_try_wait(__mbarrier_t *barrier, __mbarrier_token_t token) {
  volatile unsigned int __ready = 0;
  unsigned int ns = 1000000;
  while (! __ready) {
    __ready = __nvhpc_barrier_try_wait_sleep(barrier, token, ns);
  }
  return __ready;
}

NVHPC_UTILRT_DEV_SIG unsigned int
__nvhpc_barrier_try_wait_parity_once(__mbarrier_t *barrier, uint32_t parity) {
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
  return cuda::ptx::mbarrier_try_wait_parity(cuda::ptx::sem_relaxed, cuda::ptx::scope_cta, barrier, parity);
#else
  unsigned int __ready = 0;
  asm volatile("{\n\t"
               ".reg .pred p;\n\t"
               "mbarrier.try_wait.parity.shared::cta.b64 p, [%1], %2;\n\t"
               "selp.b32 %0, 1, 0, p;\n\t"
               "}"
               : "=r"(__ready)
               : "r"(static_cast<unsigned int>(__cvta_generic_to_shared(barrier))),
                 "r"(parity)
               : "memory");
  return __ready;
#endif
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_barrier_try_wait_parity(__mbarrier_t *barrier, uint32_t parity) {
  while (!__nvhpc_barrier_try_wait_parity_once(barrier, parity));
}

/* Helper function to round up size to required multiple */
static __device__ __inline__ uint32_t
__nvhpc_round_up_multiple(uint32_t size, uint32_t multiple)
{
  return (size + multiple - 1) / multiple * multiple;
}

/* TMA compiler API: Core implementation functions with __nvhpc_ prefix, also generated from the compiler. */
NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_init_barrier(__mbarrier_t *barrier)
{
  if (__nvhpc_block_elect_thread())
    __nvhpc_barrier_init(barrier, 1);
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_g2s_aligned(__mbarrier_t *barrier, const void *src, void *dst,
                        uint32_t size_in_bytes)
{
  if (__nvhpc_block_elect_thread())
    __nvhpc_tma_bulk_g2s(barrier, src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG uint32_t
__nvhpc_tma_g2s(__mbarrier_t *barrier, const void *src, void *dst,
                uint32_t size_in_bytes)
{
  if (__nvhpc_block_elect_thread()) {
    const char *aligned_src =
      (const char *)__nvhpc_align_down(src, TMA_ALIGNMENT);
    const char *end_src = (const char *)src + size_in_bytes;
    const uint32_t new_size = __nvhpc_round_up_multiple(
      (uint32_t)(end_src - aligned_src), TMA_SIZE_MULTIPLE);
    __nvhpc_tma_bulk_g2s(barrier, aligned_src, dst, new_size);
    return new_size;
  }
  else {
    return 0;
  }
}

NVHPC_UTILRT_DEV_SIG void
__nvhpc_tma_wait_barrier_parity(__mbarrier_t *barrier, uint32_t count, uint32_t parity)
{
  if (__nvhpc_block_elect_thread())
    __nvhpc_barrier_arrive(barrier, count);
  __syncthreads();
  __nvhpc_barrier_try_wait_parity(barrier, parity);
}

template<class C>
static __device__ __inline__ void
__nvhpc_tma_tensor_g2s(__mbarrier_t *barrier, const void *tensor, const C &coords, void *dst)
{
  assert(__isShared(dst));
  if (__nvhpc_block_elect_thread()) {
    void *aligned_dst = __nvhpc_align_up(dst, TMA_TENSOR_SHMEM_ALIGNMENT);
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
    cuda::ptx::cp_async_bulk_tensor(cuda::ptx::space_shared, cuda::ptx::space_global, aligned_dst, tensor, coords, barrier);
#else
    switch (sizeof(coords)/sizeof(coords[0])) {
    case 1:
      asm("cp.async.bulk.tensor.1d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2}], [%3];"
          :
          : "r"((unsigned int)__cvta_generic_to_shared(aligned_dst)), "l"(tensor), "r"(coords[0]), "r"((unsigned int)__cvta_generic_to_shared(barrier))
          : "memory");
      break;
    case 2:
      asm("cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3}], [%4];"
          :
          : "r"((unsigned int)__cvta_generic_to_shared(aligned_dst)), "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"((unsigned int)__cvta_generic_to_shared(barrier))
          : "memory");
      break;
    case 3:
      asm("cp.async.bulk.tensor.3d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4}], [%5];"
          :
          : "r"((unsigned int)__cvta_generic_to_shared(aligned_dst)), "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"((unsigned int)__cvta_generic_to_shared(barrier))
          : "memory");
      break;
    case 4:
      asm("cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5}], [%6];"
          :
          : "r"((unsigned int)__cvta_generic_to_shared(aligned_dst)), "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"(coords[3]), "r"((unsigned int)__cvta_generic_to_shared(barrier))
          : "memory");
      break;
    case 5:
      asm("cp.async.bulk.tensor.5d.shared::cta.global.tile.mbarrier::complete_tx::bytes [%0], [%1, {%2, %3, %4, %5, %6}], [%7];"
          :
          : "r"((unsigned int)__cvta_generic_to_shared(aligned_dst)), "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"(coords[3]), "r"(coords[4]), "r"((unsigned int)__cvta_generic_to_shared(barrier))
          : "memory");
      break;
    default:
      __trap();
    }
#endif
  }
}

template<class C>
static __device__ __inline__ void
__nvhpc_tma_tensor_s2g(const void *tensor, const C &coords, const void *src)
{
  assert(__isShared(src));
  __nvhpc_fence_proxy_async();
  if (__nvhpc_block_elect_thread()) {
    const void *aligned_src = __nvhpc_align_up(src, TMA_TENSOR_SHMEM_ALIGNMENT);
#ifndef __NVHPC_CUDA_UTILS_USE_PTX
    cuda::ptx::cp_async_bulk_tensor(cuda::ptx::space_global, cuda::ptx::space_shared, tensor, coords, aligned_src);
#else
    switch (sizeof(coords)/sizeof(coords[0])) {
    case 1:
      asm("cp.async.bulk.tensor.1d.global.shared::cta.tile.bulk_group [%0, {%1}], [%2];"
          :
          : "l"(tensor), "r"(coords[0]), "r"((unsigned int)__cvta_generic_to_shared(aligned_src))
          : "memory");
      break;
    case 2:
      asm("cp.async.bulk.tensor.2d.global.shared::cta.tile.bulk_group [%0, {%1, %2}], [%3];"
          :
          : "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"((unsigned int)__cvta_generic_to_shared(aligned_src))
          : "memory");
      break;
    case 3:
      asm("cp.async.bulk.tensor.3d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3}], [%4];"
          :
          : "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"((unsigned int)__cvta_generic_to_shared(aligned_src))
          : "memory");
      break;
    case 4:
      asm("cp.async.bulk.tensor.4d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4}], [%5];"
          :
          : "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"(coords[3]), "r"((unsigned int)__cvta_generic_to_shared(aligned_src))
          : "memory");
      break;
    case 5:
      asm("cp.async.bulk.tensor.5d.global.shared::cta.tile.bulk_group [%0, {%1, %2, %3, %4, %5}], [%6];"
          :
          : "l"(tensor), "r"(coords[0]), "r"(coords[1]), "r"(coords[2]), "r"(coords[3]), "r"(coords[4]), "r"((unsigned int)__cvta_generic_to_shared(aligned_src))
          : "memory");
      break;
    default:
      __trap();
    }
#endif
    __nvhpc_tma_bulk_commit_group();
    __nvhpc_tma_bulk_wait_group();
  }
}

/* === End of Bulk loads and stores, and associated Barriers. */

#ifdef __cplusplus
extern "C" {
#endif

// OpenACC interfaces: refer to the user C header file (openacc.h) for intended
// use cases docs.

NVHPC_UTILRT_DEV_SIG void
accx_prefetch_global(void* addr)
{
  __nvhpc_prefetch_global(addr);
}

NVHPC_UTILRT_DEV_SIG void
accx_pipeline_memcpy_async(void* dst, const void* src, size_t size)
{
  __nvhpc_pipeline_memcpy_async(dst, src, size);
}

NVHPC_UTILRT_DEV_SIG void
accx_pipeline_commit()
{
  __nvhpc_pipeline_commit();
}

NVHPC_UTILRT_DEV_SIG void
accx_pipeline_wait_prior(size_t count)
{
  __nvhpc_pipeline_wait_prior(count);
}

NVHPC_UTILRT_DEV_SIG void
accx_init_barrier(void *barrier)
{
  __nvhpc_tma_init_barrier((__mbarrier_t *)barrier);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_g2s_aligned(void *barrier, const void *src, void *dst,
                     uint32_t size_in_bytes)
{
  __nvhpc_tma_g2s_aligned((__mbarrier_t *)barrier, src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG uint32_t
accx_tma_g2s(void *barrier, const void *src, void *dst,
             uint32_t size_in_bytes)
{
  return __nvhpc_tma_g2s((__mbarrier_t *)barrier, src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_s2g_aligned(const void *src, void *dst, uint32_t size_in_bytes)
{
  __nvhpc_tma_s2g_aligned(src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_s2g(const void *src, void *dst, uint32_t size_in_bytes)
{
  __nvhpc_tma_s2g(src, dst, size_in_bytes);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor1d_g2s(void *barrier, const void *tensor, int32_t c1, void *dst)
{
  const int32_t coords[] = {c1};
  __nvhpc_tma_tensor_g2s((__mbarrier_t *)barrier, tensor, coords, dst);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor2d_g2s(void *barrier, const void *tensor, int32_t c1, int32_t c2, void *dst)
{
  const int32_t coords[] = {c1, c2};
  __nvhpc_tma_tensor_g2s((__mbarrier_t *)barrier, tensor, coords, dst);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor3d_g2s(void *barrier, const void *tensor, int32_t c1, int32_t c2, int32_t c3, void *dst)
{
  const int32_t coords[] = {c1, c2, c3};
  __nvhpc_tma_tensor_g2s((__mbarrier_t *)barrier, tensor, coords, dst);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor4d_g2s(void *barrier, const void *tensor, int32_t c1, int32_t c2, int32_t c3, int32_t c4, void *dst)
{
  const int32_t coords[] = {c1, c2, c3, c4};
  __nvhpc_tma_tensor_g2s((__mbarrier_t *)barrier, tensor, coords, dst);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor5d_g2s(void *barrier, const void *tensor, int32_t c1, int32_t c2, int32_t c3, int32_t c4, int32_t c5, void *dst)
{
  const int32_t coords[] = {c1, c2, c3, c4, c5};
  __nvhpc_tma_tensor_g2s((__mbarrier_t *)barrier, tensor, coords, dst);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor1d_s2g(const void *tensor, int32_t c1, const void *src)
{
  const int32_t coords[] = {c1};
  __nvhpc_tma_tensor_s2g(tensor, coords, src);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor2d_s2g(const void *tensor, int32_t c1, int32_t c2, const void *src)
{
  const int32_t coords[] = {c1, c2};
  __nvhpc_tma_tensor_s2g(tensor, coords, src);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor3d_s2g(const void *tensor, int32_t c1, int32_t c2, int32_t c3, const void *src)
{
  const int32_t coords[] = {c1, c2, c3};
  __nvhpc_tma_tensor_s2g(tensor, coords, src);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor4d_s2g(const void *tensor, int32_t c1, int32_t c2, int32_t c3, int32_t c4, const void *src)
{
  const int32_t coords[] = {c1, c2, c3, c4};
  __nvhpc_tma_tensor_s2g(tensor, coords, src);
}

NVHPC_UTILRT_DEV_SIG void
accx_tma_tensor5d_s2g(const void *tensor, int32_t c1, int32_t c2, int32_t c3, int32_t c4, int32_t c5, const void *src)
{
  const int32_t coords[] = {c1, c2, c3, c4, c5};
  __nvhpc_tma_tensor_s2g(tensor, coords, src);
}

NVHPC_UTILRT_DEV_SIG uint32_t
accx_tma_tensor_aligned_offset(const void *smem, uint32_t elem_size)
{
  const void *aligned_smem = (const void *)__nvhpc_align_up(smem, TMA_TENSOR_SHMEM_ALIGNMENT);
  return (uint32_t)((const char *)aligned_smem - (const char *)smem) / elem_size;
}

NVHPC_UTILRT_DEV_SIG uint32_t
accx_tma_offset_from_aligned_address(const void* gmem)
{
  const char* aligned_addr = (const char*)__nvhpc_align_down(gmem, TMA_ALIGNMENT);
  return (uint32_t)((const char*)gmem - aligned_addr);
}

NVHPC_UTILRT_DEV_SIG __CONST_PURE uint32_t
accx_tma_aligned_index(const void* gmem, uint32_t elem_size)
{
  const char* base_addr = (const char *)gmem - threadIdx.x * elem_size;
  const char* aligned_addr = (const char *)__nvhpc_align_down(base_addr, TMA_ALIGNMENT);
  return (uint32_t)(base_addr - aligned_addr) / elem_size + threadIdx.x;
}

NVHPC_UTILRT_DEV_SIG void
accx_wait_barrier_parity(void *barrier, uint32_t count, uint32_t parity)
{
  __nvhpc_tma_wait_barrier_parity((__mbarrier_t *)barrier, count, parity);
}

NVHPC_UTILRT_DEV_SIG void
accx_wait_barrier(void *barrier, uint32_t count)
{
  __nvhpc_tma_wait_barrier_parity((__mbarrier_t *)barrier, count, 0);
}

#ifdef __cplusplus
}
#endif

#endif /*  __CUDA_ARCH__ */

#endif /* __NVHPC_UTILS_RUNTIME_H */
