Kokkos Core Kernels Package  Version of the Day
Kokkos_HIP_Space.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 #ifndef KOKKOS_HIPSPACE_HPP
46 #define KOKKOS_HIPSPACE_HPP
47 
48 #include <Kokkos_Core_fwd.hpp>
49 
50 #if defined(KOKKOS_ENABLE_HIP)
51 
52 #include <iosfwd>
53 #include <typeinfo>
54 #include <string>
55 #include <cstddef>
56 #include <iosfwd>
57 
58 #include <Kokkos_HostSpace.hpp>
59 #include <Kokkos_Layout.hpp>
60 #include <Kokkos_ScratchSpace.hpp>
61 
62 #include <impl/Kokkos_Profiling_Interface.hpp>
63 #include <impl/Kokkos_ExecSpaceInitializer.hpp>
64 #include <impl/Kokkos_HostSharedPtr.hpp>
65 
66 #include <hip/hip_runtime_api.h>
67 /*--------------------------------------------------------------------------*/
68 
69 namespace Kokkos {
70 namespace Experimental {
73 class HIPSpace {
74  public:
76  using memory_space = HIPSpace;
77  using execution_space = Kokkos::Experimental::HIP;
78  using device_type = Kokkos::Device<execution_space, memory_space>;
79 
80  using size_type = unsigned int;
81 
82  /*--------------------------------*/
83 
84  HIPSpace();
85  HIPSpace(HIPSpace&& rhs) = default;
86  HIPSpace(const HIPSpace& rhs) = default;
87  HIPSpace& operator=(HIPSpace&& rhs) = default;
88  HIPSpace& operator=(const HIPSpace& rhs) = default;
89  ~HIPSpace() = default;
90 
92  void* allocate(const size_t arg_alloc_size) const;
93  void* allocate(const char* arg_label, const size_t arg_alloc_size,
94  const size_t arg_logical_size = 0) const;
95 
97  void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const;
98  void deallocate(const char* arg_label, void* const arg_alloc_ptr,
99  const size_t arg_alloc_size,
100  const size_t arg_logical_size = 0) const;
101 
102  private:
103  template <class, class, class, class>
104  friend class LogicalMemorySpace;
105  void* impl_allocate(const char* arg_label, const size_t arg_alloc_size,
106  const size_t arg_logical_size = 0,
107  const Kokkos::Tools::SpaceHandle =
108  Kokkos::Tools::make_space_handle(name())) const;
109  void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr,
110  const size_t arg_alloc_size,
111  const size_t arg_logical_size = 0,
112  const Kokkos::Tools::SpaceHandle =
113  Kokkos::Tools::make_space_handle(name())) const;
114 
115  public:
117  static constexpr const char* name() { return "HIP"; }
118 
119  /*--------------------------------*/
121  KOKKOS_DEPRECATED static void access_error();
122  KOKKOS_DEPRECATED static void access_error(const void* const);
123 
124  private:
125  int m_device;
126 
127  friend class Kokkos::Impl::SharedAllocationRecord<
128  Kokkos::Experimental::HIPSpace, void>;
129 };
130 
131 } // namespace Experimental
132 } // namespace Kokkos
133 
134 /*--------------------------------------------------------------------------*/
135 /*--------------------------------------------------------------------------*/
136 
137 namespace Kokkos {
138 namespace Experimental {
142 class HIPHostPinnedSpace {
143  public:
145 
146  using execution_space = HostSpace::execution_space;
147  using memory_space = HIPHostPinnedSpace;
148  using device_type = Kokkos::Device<execution_space, memory_space>;
149  using size_type = unsigned int;
150 
151  /*--------------------------------*/
152 
153  HIPHostPinnedSpace();
154  HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) = default;
155  HIPHostPinnedSpace(const HIPHostPinnedSpace& rhs) = default;
156  HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) = default;
157  HIPHostPinnedSpace& operator=(const HIPHostPinnedSpace& rhs) = default;
158  ~HIPHostPinnedSpace() = default;
159 
161  void* allocate(const size_t arg_alloc_size) const;
162  void* allocate(const char* arg_label, const size_t arg_alloc_size,
163  const size_t arg_logical_size = 0) const;
164 
166  void deallocate(void* const arg_alloc_ptr, const size_t arg_alloc_size) const;
167  void deallocate(const char* arg_label, void* const arg_alloc_ptr,
168  const size_t arg_alloc_size,
169  const size_t arg_logical_size = 0) const;
170 
171  private:
172  template <class, class, class, class>
173  friend class LogicalMemorySpace;
174  void* impl_allocate(const char* arg_label, const size_t arg_alloc_size,
175  const size_t arg_logical_size = 0,
176  const Kokkos::Tools::SpaceHandle =
177  Kokkos::Tools::make_space_handle(name())) const;
178  void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr,
179  const size_t arg_alloc_size,
180  const size_t arg_logical_size = 0,
181  const Kokkos::Tools::SpaceHandle =
182  Kokkos::Tools::make_space_handle(name())) const;
183 
184  public:
186  static constexpr const char* name() { return "HIPHostPinned"; }
187 
188  /*--------------------------------*/
189 };
190 } // namespace Experimental
191 } // namespace Kokkos
192 
193 /*--------------------------------------------------------------------------*/
194 /*--------------------------------------------------------------------------*/
195 
196 namespace Kokkos {
197 namespace Impl {
198 
199 static_assert(
200  Kokkos::Impl::MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
201  Kokkos::Experimental::HIPSpace>::assignable,
202  "");
203 
204 //----------------------------------------
205 
206 template <>
207 struct MemorySpaceAccess<Kokkos::HostSpace, Kokkos::Experimental::HIPSpace> {
208  enum : bool { assignable = false };
209  enum : bool { accessible = false };
210  enum : bool { deepcopy = true };
211 };
212 
213 template <>
214 struct MemorySpaceAccess<Kokkos::HostSpace,
215  Kokkos::Experimental::HIPHostPinnedSpace> {
216  // HostSpace::execution_space == HIPHostPinnedSpace::execution_space
217  enum : bool { assignable = true };
218  enum : bool { accessible = true };
219  enum : bool { deepcopy = true };
220 };
221 
222 //----------------------------------------
223 
224 template <>
225 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace, Kokkos::HostSpace> {
226  enum : bool { assignable = false };
227  enum : bool { accessible = false };
228  enum : bool { deepcopy = true };
229 };
230 
231 template <>
232 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
233  Kokkos::Experimental::HIPHostPinnedSpace> {
234  // HIPSpace::execution_space != HIPHostPinnedSpace::execution_space
235  enum : bool { assignable = false };
236  enum : bool { accessible = true }; // HIPSpace::execution_space
237  enum : bool { deepcopy = true };
238 };
239 
240 //----------------------------------------
241 // HIPHostPinnedSpace::execution_space == HostSpace::execution_space
242 // HIPHostPinnedSpace accessible to both HIP and Host
243 
244 template <>
245 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
247  enum : bool { assignable = false }; // Cannot access from HIP
248  enum : bool { accessible = true }; // HIPHostPinnedSpace::execution_space
249  enum : bool { deepcopy = true };
250 };
251 
252 template <>
253 struct MemorySpaceAccess<Kokkos::Experimental::HIPHostPinnedSpace,
254  Kokkos::Experimental::HIPSpace> {
255  enum : bool { assignable = false }; // Cannot access from Host
256  enum : bool { accessible = false };
257  enum : bool { deepcopy = true };
258 };
259 
260 }; // namespace Impl
261 //----------------------------------------
262 
263 } // namespace Kokkos
264 
265 /*--------------------------------------------------------------------------*/
266 /*--------------------------------------------------------------------------*/
267 
268 namespace Kokkos {
269 namespace Impl {
270 
271 void DeepCopyAsyncHIP(void* dst, const void* src, size_t n);
272 
273 template <>
274 struct DeepCopy<Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIPSpace,
275  Kokkos::Experimental::HIP> {
276  DeepCopy(void* dst, const void* src, size_t);
277  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
278  size_t);
279 };
280 
281 template <>
282 struct DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace,
283  Kokkos::Experimental::HIP> {
284  DeepCopy(void* dst, const void* src, size_t);
285  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
286  size_t);
287 };
288 
289 template <>
290 struct DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace,
291  Kokkos::Experimental::HIP> {
292  DeepCopy(void* dst, const void* src, size_t);
293  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
294  size_t);
295 };
296 
297 template <class ExecutionSpace>
298 struct DeepCopy<Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIPSpace,
299  ExecutionSpace> {
300  inline DeepCopy(void* dst, const void* src, size_t n) {
301  (void)DeepCopy<Kokkos::Experimental::HIPSpace,
302  Kokkos::Experimental::HIPSpace, Kokkos::Experimental::HIP>(
303  dst, src, n);
304  }
305 
306  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
307  size_t n) {
308  exec.fence();
309  DeepCopyAsyncHIP(dst, src, n);
310  }
311 };
312 
313 template <class ExecutionSpace>
314 struct DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace, ExecutionSpace> {
315  inline DeepCopy(void* dst, const void* src, size_t n) {
316  (void)DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace,
317  Kokkos::Experimental::HIP>(dst, src, n);
318  }
319 
320  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
321  size_t n) {
322  exec.fence();
323  DeepCopyAsyncHIP(dst, src, n);
324  }
325 };
326 
327 template <class ExecutionSpace>
328 struct DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace, ExecutionSpace> {
329  inline DeepCopy(void* dst, const void* src, size_t n) {
330  (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace,
331  Kokkos::Experimental::HIP>(dst, src, n);
332  }
333 
334  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
335  size_t n) {
336  exec.fence();
337  DeepCopyAsyncHIP(dst, src, n);
338  }
339 };
340 
341 template <>
342 struct DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace,
343  Kokkos::Experimental::HIPHostPinnedSpace,
344  Kokkos::Experimental::HIP> {
345  DeepCopy(void* dst, const void* src, size_t);
346  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
347  size_t);
348 };
349 
350 template <>
351 struct DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
352  Kokkos::Experimental::HIP> {
353  DeepCopy(void* dst, const void* src, size_t);
354  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
355  size_t);
356 };
357 
358 template <>
359 struct DeepCopy<HostSpace, Kokkos::Experimental::HIPHostPinnedSpace,
360  Kokkos::Experimental::HIP> {
361  DeepCopy(void* dst, const void* src, size_t);
362  DeepCopy(const Kokkos::Experimental::HIP&, void* dst, const void* src,
363  size_t);
364 };
365 
366 template <class ExecutionSpace>
367 struct DeepCopy<Kokkos::Experimental::HIPSpace,
368  Kokkos::Experimental::HIPHostPinnedSpace, ExecutionSpace> {
369  inline DeepCopy(void* dst, const void* src, size_t n) {
370  (void)DeepCopy<Kokkos::Experimental::HIPSpace, HostSpace,
371  Kokkos::Experimental::HIP>(dst, src, n);
372  }
373 
374  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
375  size_t n) {
376  exec.fence();
377  DeepCopyAsyncHIP(dst, src, n);
378  }
379 };
380 
381 template <class ExecutionSpace>
382 struct DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace,
383  Kokkos::Experimental::HIPSpace, ExecutionSpace> {
384  inline DeepCopy(void* dst, const void* src, size_t n) {
385  (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPSpace,
386  Kokkos::Experimental::HIP>(dst, src, n);
387  }
388 
389  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
390  size_t n) {
391  exec.fence();
392  DeepCopyAsyncHIP(dst, src, n);
393  }
394 };
395 
396 template <class ExecutionSpace>
397 struct DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace,
398  Kokkos::Experimental::HIPHostPinnedSpace, ExecutionSpace> {
399  inline DeepCopy(void* dst, const void* src, size_t n) {
400  (void)DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace,
401  Kokkos::Experimental::HIPHostPinnedSpace,
402  Kokkos::Experimental::HIP>(dst, src, n);
403  }
404 
405  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
406  size_t n) {
407  exec.fence();
408  DeepCopyAsyncHIP(dst, src, n);
409  }
410 };
411 
412 template <class ExecutionSpace>
413 struct DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
414  ExecutionSpace> {
415  inline DeepCopy(void* dst, const void* src, size_t n) {
416  (void)DeepCopy<Kokkos::Experimental::HIPHostPinnedSpace, HostSpace,
417  Kokkos::Experimental::HIP>(dst, src, n);
418  }
419 
420  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
421  size_t n) {
422  exec.fence();
423  DeepCopyAsyncHIP(dst, src, n);
424  }
425 };
426 
427 template <class ExecutionSpace>
428 struct DeepCopy<HostSpace, Kokkos::Experimental::HIPHostPinnedSpace,
429  ExecutionSpace> {
430  inline DeepCopy(void* dst, const void* src, size_t n) {
431  (void)DeepCopy<HostSpace, Kokkos::Experimental::HIPHostPinnedSpace,
432  Kokkos::Experimental::HIP>(dst, src, n);
433  }
434 
435  inline DeepCopy(const ExecutionSpace& exec, void* dst, const void* src,
436  size_t n) {
437  exec.fence();
438  DeepCopyAsyncHIP(dst, src, n);
439  }
440 };
441 } // namespace Impl
442 } // namespace Kokkos
443 
444 //----------------------------------------------------------------------------
445 //----------------------------------------------------------------------------
446 
447 namespace Kokkos {
448 namespace Impl {
449 
450 template <>
451 class SharedAllocationRecord<Kokkos::Experimental::HIPSpace, void>
452  : public HostInaccessibleSharedAllocationRecordCommon<
453  Kokkos::Experimental::HIPSpace> {
454  private:
455  friend class SharedAllocationRecordCommon<Kokkos::Experimental::HIPSpace>;
456  friend class HostInaccessibleSharedAllocationRecordCommon<
457  Kokkos::Experimental::HIPSpace>;
458  using base_t = HostInaccessibleSharedAllocationRecordCommon<
459  Kokkos::Experimental::HIPSpace>;
460  using RecordBase = SharedAllocationRecord<void, void>;
461 
462  SharedAllocationRecord(const SharedAllocationRecord&) = delete;
463  SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
464 
465 #ifdef KOKKOS_ENABLE_DEBUG
466  static RecordBase s_root_record;
467 #endif
468 
469  const Kokkos::Experimental::HIPSpace m_space;
470 
471  protected:
472  ~SharedAllocationRecord();
473 
474  SharedAllocationRecord(
475  const Kokkos::Experimental::HIPSpace& arg_space,
476  const std::string& arg_label, const size_t arg_alloc_size,
477  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
478 };
479 
480 template <>
481 class SharedAllocationRecord<Kokkos::Experimental::HIPHostPinnedSpace, void>
482  : public SharedAllocationRecordCommon<
483  Kokkos::Experimental::HIPHostPinnedSpace> {
484  private:
485  friend class SharedAllocationRecordCommon<
486  Kokkos::Experimental::HIPHostPinnedSpace>;
487  using base_t =
488  SharedAllocationRecordCommon<Kokkos::Experimental::HIPHostPinnedSpace>;
489  using RecordBase = SharedAllocationRecord<void, void>;
490 
491  SharedAllocationRecord(const SharedAllocationRecord&) = delete;
492  SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete;
493 
494 #ifdef KOKKOS_ENABLE_DEBUG
495  static RecordBase s_root_record;
496 #endif
497 
498  const Kokkos::Experimental::HIPHostPinnedSpace m_space;
499 
500  protected:
501  ~SharedAllocationRecord();
502  SharedAllocationRecord() = default;
503 
504  SharedAllocationRecord(
505  const Kokkos::Experimental::HIPHostPinnedSpace& arg_space,
506  const std::string& arg_label, const size_t arg_alloc_size,
507  const RecordBase::function_type arg_dealloc = &base_t::deallocate);
508 };
509 } // namespace Impl
510 } // namespace Kokkos
511 
512 //----------------------------------------------------------------------------
513 //----------------------------------------------------------------------------
514 
515 namespace Kokkos {
516 namespace Experimental {
517 namespace Impl {
518 class HIPInternal;
519 }
522 class HIP {
523  public:
524  //------------------------------------
526 
527 
529  using execution_space = HIP;
530  using memory_space = HIPSpace;
531  using device_type = Kokkos::Device<execution_space, memory_space>;
532 
533  using array_layout = LayoutLeft;
534  using size_type = HIPSpace::size_type;
535 
536  using scratch_memory_space = ScratchMemorySpace<HIP>;
537 
538  HIP();
539  HIP(hipStream_t stream);
540 
542  //------------------------------------
544 
545 
546  KOKKOS_INLINE_FUNCTION static int in_parallel() {
547 #if defined(__HIP_ARCH__)
548  return true;
549 #else
550  return false;
551 #endif
552  }
553 
560  static void impl_static_fence();
561 
562  void fence() const;
563 
564  hipStream_t hip_stream() const;
565 
567  static void print_configuration(std::ostream&, const bool detail = false);
568 
570  static void impl_finalize();
571 
575  struct SelectDevice {
576  int hip_device_id;
577  SelectDevice() : hip_device_id(0) {}
578  explicit SelectDevice(int id) : hip_device_id(id) {}
579  };
580 
581  int hip_device() const;
582  static hipDeviceProp_t const& hip_device_prop();
583 
584  static void impl_initialize(const SelectDevice = SelectDevice());
585 
586  static int impl_is_initialized();
587 
588  // static size_type device_arch();
589 
590  static size_type detect_device_count();
591 
592  static int concurrency();
593  static const char* name();
594 
595  inline Impl::HIPInternal* impl_internal_space_instance() const {
596  return m_space_instance.get();
597  }
598 
599  uint32_t impl_instance_id() const noexcept { return 0; }
600 
601  private:
602  Kokkos::Impl::HostSharedPtr<Impl::HIPInternal> m_space_instance;
603 };
604 } // namespace Experimental
605 namespace Tools {
606 namespace Experimental {
607 template <>
608 struct DeviceTypeTraits<Kokkos::Experimental::HIP> {
609  static constexpr DeviceType id = DeviceType::HIP;
610 };
611 } // namespace Experimental
612 } // namespace Tools
613 
614 namespace Impl {
615 
616 class HIPSpaceInitializer : public Kokkos::Impl::ExecSpaceInitializerBase {
617  public:
618  HIPSpaceInitializer() = default;
619  ~HIPSpaceInitializer() = default;
620  void initialize(const InitArguments& args) final;
621  void finalize(const bool) final;
622  void fence() final;
623  void print_configuration(std::ostream& msg, const bool detail) final;
624 };
625 
626 } // namespace Impl
627 } // namespace Kokkos
628 
629 namespace Kokkos {
630 namespace Impl {
631 
632 template <>
633 struct MemorySpaceAccess<Kokkos::Experimental::HIPSpace,
634  Kokkos::Experimental::HIP::scratch_memory_space> {
635  enum : bool { assignable = false };
636  enum : bool { accessible = true };
637  enum : bool { deepcopy = false };
638 };
639 
640 } // namespace Impl
641 } // namespace Kokkos
642 
643 #endif /* #if defined( KOKKOS_ENABLE_HIP ) */
644 #endif /* #define KOKKOS_HIPSPACE_HPP */
Memory management for host memory.
Declaration of various MemoryLayout options.
Definition: dummy.cpp:3
Access relationship between DstMemorySpace and SrcMemorySpace.