/*
 * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 *
 * NVIDIA SOFTWARE LICENSE
 *
 * This license is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs your use of the
 * NVIDIA/CUDA C++ Library software and materials provided hereunder (“SOFTWARE”).
 *
 * This license can be accepted only by an adult of legal age of majority in the country in which the SOFTWARE is used.
 * If you are under the legal age of majority, you must ask your parent or legal guardian to consent to this license. By
 * taking delivery of the SOFTWARE, you affirm that you have reached the legal age of majority, you accept the terms of
 * this license, and you take legal and financial responsibility for the actions of your permitted users.
 *
 * You agree to use the SOFTWARE only for purposes that are permitted by (a) this license, and (b) any applicable law,
 * regulation or generally accepted practices or guidelines in the relevant jurisdictions.
 *
 * 1. LICENSE. Subject to the terms of this license, NVIDIA grants you a non-exclusive limited license to: (a) install
 * and use the SOFTWARE, and (b) distribute the SOFTWARE subject to the distribution requirements described in this
 * license. NVIDIA reserves all rights, title and interest in and to the SOFTWARE not expressly granted to you under
 * this license.
 *
 * 2. DISTRIBUTION REQUIREMENTS. These are the distribution requirements for you to exercise the distribution grant:
 * a.      The terms under which you distribute the SOFTWARE must be consistent with the terms of this license,
 * including (without limitation) terms relating to the license grant and license restrictions and protection of
 * NVIDIA’s intellectual property rights. b.      You agree to notify NVIDIA in writing of any known or suspected
 * distribution or use of the SOFTWARE not in compliance with the requirements of this license, and to enforce the terms
 * of your agreements with respect to distributed SOFTWARE.
 *
 * 3. LIMITATIONS. Your license to use the SOFTWARE is restricted as follows:
 * a.      The SOFTWARE is licensed for you to develop applications only for use in systems with NVIDIA GPUs.
 * b.      You may not reverse engineer, decompile or disassemble, or remove copyright or other proprietary notices from
 * any portion of the SOFTWARE or copies of the SOFTWARE. c.      You may not modify or create derivative works of any
 * portion of the SOFTWARE. d.      You may not bypass, disable, or circumvent any technical measure, encryption,
 * security, digital rights management or authentication mechanism in the SOFTWARE. e.      You may not use the SOFTWARE
 * in any manner that would cause it to become subject to an open source software license. As examples, licenses that
 * require as a condition of use, modification, and/or distribution that the SOFTWARE be (i) disclosed or distributed in
 * source code form; (ii) licensed for the purpose of making derivative works; or (iii) redistributable at no charge. f.
 * Unless you have an agreement with NVIDIA for this purpose, you may not use the SOFTWARE with any system or
 * application where the use or failure of the system or application can reasonably be expected to threaten or result in
 * personal injury, death, or catastrophic loss. Examples include use in avionics, navigation, military, medical, life
 * support or other life critical applications. NVIDIA does not design, test or manufacture the SOFTWARE for these
 * critical uses and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or
 * damages arising from such uses. g.      You agree to defend, indemnify and hold harmless NVIDIA and its affiliates,
 * and their respective employees, contractors, agents, officers and directors, from and against any and all claims,
 * damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited
 * to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to use
 * of the SOFTWARE outside of the scope of this Agreement, or not in compliance with its terms.
 *
 * 4. PRE-RELEASE. SOFTWARE versions identified as alpha, beta, preview, early access or otherwise as pre-release may
 * not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy,
 * availability, and reliability standards relative to commercial versions of NVIDIA software and materials. You may use
 * a pre-release SOFTWARE version at your own risk, understanding that these versions are not intended for use in
 * production or business-critical systems.
 *
 * 5. OWNERSHIP. The SOFTWARE and the related intellectual property rights therein are and will remain the sole and
 * exclusive property of NVIDIA or its licensors. The SOFTWARE is copyrighted and protected by the laws of the United
 * States and other countries, and international treaty provisions. NVIDIA may make changes to the SOFTWARE, at any time
 * without notice, but is not obligated to support or update the SOFTWARE.
 *
 * 6. COMPONENTS UNDER OTHER LICENSES. The SOFTWARE may include NVIDIA or third-party components with separate legal
 * notices or terms as may be described in proprietary notices accompanying the SOFTWARE. If and to the extent there is
 * a conflict between the terms in this license and the license terms associated with a component, the license terms
 * associated with the components control only to the extent necessary to resolve the conflict.
 *
 * 7. FEEDBACK. You may, but don’t have to, provide to NVIDIA any Feedback. “Feedback” means any suggestions, bug fixes,
 * enhancements, modifications, feature requests or other feedback regarding the SOFTWARE. For any Feedback that you
 * voluntarily provide, you hereby grant NVIDIA and its affiliates a perpetual, non-exclusive, worldwide, irrevocable
 * license to use, reproduce, modify, license, sublicense (through multiple tiers of sublicensees), and distribute
 * (through multiple tiers of distributors) the Feedback without the payment of any royalties or fees to you. NVIDIA
 * will use Feedback at its choice.
 *
 * 8. NO WARRANTIES. THE SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY EXPRESS OR IMPLIED WARRANTY OF ANY KIND INCLUDING, BUT
 * NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, OR FITNESS FOR A PARTICULAR PURPOSE. NVIDIA DOES NOT
 * WARRANT THAT THE SOFTWARE WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR
 * ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED.
 *
 * 9. LIMITATIONS OF LIABILITY. TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA AND ITS AFFILIATES SHALL NOT BE LIABLE
 * FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, PROJECT DELAYS, LOSS OF USE,
 * LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH
 * THIS LICENSE OR THE USE OR PERFORMANCE OF THE SOFTWARE, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON
 * BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION
 * OR THEORY OF LIABILITY, EVEN IF NVIDIA HAS PREVIOUSLY BEEN ADVISED OF, OR COULD REASONABLY HAVE FORESEEN, THE
 * POSSIBILITY OF SUCH DAMAGES. IN NO EVENT WILL NVIDIA’S AND ITS AFFILIATES TOTAL CUMULATIVE LIABILITY UNDER OR ARISING
 * OUT OF THIS LICENSE EXCEED US$10.00. THE NATURE OF THE LIABILITY OR THE NUMBER OF CLAIMS OR SUITS SHALL NOT ENLARGE
 * OR EXTEND THIS LIMIT.
 *
 * 10. TERMINATION. Your rights under this license will terminate automatically without notice from NVIDIA if you fail
 * to comply with any term and condition of this license or if you commence or participate in any legal proceeding
 * against NVIDIA with respect to the SOFTWARE. NVIDIA may terminate this license with advance written notice to you if
 * NVIDIA decides to no longer provide the SOFTWARE in a country or, in NVIDIA’s sole discretion, the continued use of
 * it is no longer commercially viable. Upon any termination of this license, you agree to promptly discontinue use of
 * the SOFTWARE and destroy all copies in your possession or control. Your prior distributions in accordance with this
 * license are not affected by the termination of this license. All provisions of this license will survive termination,
 * except for the license granted to you.
 *
 * 11. APPLICABLE LAW. This license will be governed in all respects by the laws of the United States and of the State
 * of Delaware as those laws are applied to contracts entered into and performed entirely within Delaware by Delaware
 * residents, without regard to the conflicts of laws principles. The United Nations Convention on Contracts for the
 * International Sale of Goods is specifically disclaimed. You agree to all terms of this Agreement in the English
 * language. The state or federal courts residing in Santa Clara County, California shall have exclusive jurisdiction
 * over any dispute or claim arising out of this license. Notwithstanding this, you agree that NVIDIA shall still be
 * allowed to apply for injunctive remedies or an equivalent type of urgent legal relief in any jurisdiction.
 *
 * 12. NO ASSIGNMENT. This license and your rights and obligations thereunder may not be assigned by you by any means or
 * operation of law without NVIDIA’s permission. Any attempted assignment not approved by NVIDIA in writing shall be
 * void and of no effect.
 *
 * 13. EXPORT. The SOFTWARE is subject to United States export laws and regulations. You agree that you will not ship,
 * transfer or export the SOFTWARE into any country, or use the SOFTWARE in any manner, prohibited by the United States
 * Bureau of Industry and Security or economic sanctions regulations administered by the U.S. Department of Treasury’s
 * Office of Foreign Assets Control (OFAC), or any applicable export laws, restrictions or regulations. These laws
 * include restrictions on destinations, end users and end use. By accepting this license, you confirm that you are not
 * a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from
 * receiving the SOFTWARE.
 *
 * 14. GOVERNMENT USE. The SOFTWARE has been developed entirely at private expense and is “commercial items” consisting
 * of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS.
 * Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the
 * restrictions in this license pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (b)(1) and (2) of the
 * Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is
 * NVIDIA, 2788 San Tomas Expressway, Santa Clara, CA 95051.
 *
 * 15. ENTIRE AGREEMENT. This license is the final, complete and exclusive agreement between the parties relating to the
 * subject matter of this license and supersedes all prior or contemporaneous understandings and agreements relating to
 * this subject matter, whether oral or written. If any court of competent jurisdiction determines that any provision of
 * this license is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect.
 * This license may only be modified in a writing signed by an authorized representative of each party.
 *
 * (v. August 20, 2021)
 */

_LIBCUDACXX_BEGIN_NAMESPACE_CUDA

namespace __detail_ap
{

template <typename _Property>
_CCCL_DEVICE void* __associate_address_space(void* __ptr, _Property __prop)
{
  if (std::is_same<_Property, access_property::shared>::value == true)
  {
    bool __b = __isShared(__ptr);
    _LIBCUDACXX_ASSERT(__b, "");
#if !defined(_CCCL_CUDACC_BELOW_11_2)
    __builtin_assume(__b);
#else // ^^^ !_CCCL_CUDACC_BELOW_11_2 ^^^ / vvv _CCCL_CUDACC_BELOW_11_2 vvv
    (void) __b;
#endif // _CCCL_CUDACC_BELOW_11_2
  }
  else if (std::is_same<_Property, access_property::global>::value == true
           || std::is_same<_Property, access_property::normal>::value == true
           || std::is_same<_Property, access_property::persisting>::value == true
           || std::is_same<_Property, access_property::streaming>::value == true
           || std::is_same<_Property, access_property>::value)
  {
    bool __b = __isGlobal(__ptr);
    _LIBCUDACXX_ASSERT(__b, "");
#if !defined(_CCCL_CUDACC_BELOW_11_2)
    __builtin_assume(__b);
#else // ^^^ !_CCCL_CUDACC_BELOW_11_2 ^^^ / vvv _CCCL_CUDACC_BELOW_11_2 vvv
    (void) __b;
#endif // _CCCL_CUDACC_BELOW_11_2
  }

  return __ptr;
}

template <typename __Prop>
_CCCL_DEVICE void* __associate_descriptor(void* __ptr, __Prop __prop)
{
  return __associate_descriptor(__ptr, static_cast<std::uint64_t>(access_property(__prop)));
}

template <>
inline _CCCL_DEVICE void* __associate_descriptor(void* __ptr, std::uint64_t __prop)
{
  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80, (return __nv_associate_access_property(__ptr, __prop);), (return __ptr;))
}

template <>
inline _CCCL_DEVICE void* __associate_descriptor(void* __ptr, access_property::shared)
{
  return __ptr;
}

template <typename _Type, typename _Property>
_CCCL_HOST_DEVICE _Type* __associate(_Type* __ptr, _Property __prop)
{
  NV_IF_ELSE_TARGET(NV_IS_DEVICE,
                    (return static_cast<_Type*>(__associate_descriptor(
                      __associate_address_space(const_cast<void*>(static_cast<const void*>(__ptr)), __prop), __prop));),
                    (return __ptr;))
}

template <typename _Property>
class __annotated_ptr_base
{
  using __error = typename _Property::__unknown_access_property_type;
};

template <>
class __annotated_ptr_base<access_property::shared>
{
protected:
  static constexpr std::uint64_t __prop = 0;

  constexpr __annotated_ptr_base() noexcept                                          = default;
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property::shared) noexcept {}
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, access_property::shared{});
  }
  _CCCL_HOST_DEVICE constexpr access_property::shared __get_property() const noexcept
  {
    return access_property::shared{};
  }
};

template <>
class __annotated_ptr_base<access_property::global>
{
protected:
  static constexpr std::uint64_t __prop = __sm_80::__interleave_normal();

  constexpr __annotated_ptr_base() noexcept                                          = default;
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property::global) noexcept {}
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, access_property::global{});
  }
  _CCCL_HOST_DEVICE constexpr access_property::global __get_property() const noexcept
  {
    return access_property::global{};
  }
};

template <>
class __annotated_ptr_base<access_property::normal>
{
protected:
  static constexpr std::uint64_t __prop = __sm_80::__interleave_normal_demote();

  constexpr __annotated_ptr_base() noexcept                                          = default;
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property::normal) noexcept {}
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, access_property::normal{});
  }
  _CCCL_HOST_DEVICE constexpr access_property::normal __get_property() const noexcept
  {
    return access_property::normal{};
  }
};

template <>
class __annotated_ptr_base<access_property::persisting>
{
protected:
  static constexpr std::uint64_t __prop = __sm_80::__interleave_persisting();

  constexpr __annotated_ptr_base() noexcept                                          = default;
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property::persisting) noexcept {}
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, access_property::persisting{});
  }
  _CCCL_HOST_DEVICE constexpr access_property::persisting __get_property() const noexcept
  {
    return access_property::persisting{};
  }
};

template <>
class __annotated_ptr_base<access_property::streaming>
{
protected:
  static constexpr std::uint64_t __prop = __sm_80::__interleave_streaming();

  constexpr __annotated_ptr_base() noexcept                                          = default;
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property::streaming) noexcept {}
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, access_property::streaming{});
  }
  _CCCL_HOST_DEVICE constexpr access_property::streaming __get_property() const noexcept
  {
    return access_property::streaming{};
  }
};

template <>
class __annotated_ptr_base<access_property>
{
protected:
  std::uint64_t __prop;

  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base() noexcept
      : __prop(access_property())
  {}
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(std::uint64_t __property) noexcept
      : __prop(__property)
  {}
  _CCCL_HOST_DEVICE constexpr __annotated_ptr_base(access_property __property) noexcept
      : __annotated_ptr_base(static_cast<std::uint64_t>(__property))
  {}
  constexpr __annotated_ptr_base(__annotated_ptr_base const&)                        = default;
  _CCCL_CONSTEXPR_CXX14 __annotated_ptr_base& operator=(const __annotated_ptr_base&) = default;
  inline _CCCL_DEVICE void* __apply_prop(void* __p) const
  {
    return __associate(__p, __prop);
  }
  _CCCL_HOST_DEVICE access_property __get_property() const noexcept
  {
    return reinterpret_cast<access_property&>(const_cast<std::uint64_t&>(__prop));
  }
};
} // namespace __detail_ap

_LIBCUDACXX_END_NAMESPACE_CUDA
