Kokkos Core Kernels Package  Version of the Day
Kokkos_DynamicView.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 #ifndef KOKKOS_DYNAMIC_VIEW_HPP
46 #define KOKKOS_DYNAMIC_VIEW_HPP
47 
48 #include <cstdio>
49 
50 #include <Kokkos_Core.hpp>
51 #include <impl/Kokkos_Error.hpp>
52 
53 namespace Kokkos {
54 namespace Experimental {
55 
56 // Simple metafunction for choosing memory space
57 // In the current implementation, if memory_space == CudaSpace,
58 // use CudaUVMSpace for the chunk 'array' allocation, which
59 // contains will contain pointers to chunks of memory allocated
60 // in CudaSpace
61 namespace Impl {
62 template <class MemSpace>
63 struct ChunkArraySpace {
64  using memory_space = MemSpace;
65 };
66 
67 #ifdef KOKKOS_ENABLE_CUDA
68 template <>
69 struct ChunkArraySpace<Kokkos::CudaSpace> {
70  using memory_space = typename Kokkos::CudaUVMSpace;
71 };
72 #endif
73 #ifdef KOKKOS_ENABLE_HIP
74 template <>
75 struct ChunkArraySpace<Kokkos::Experimental::HIPSpace> {
76  using memory_space = typename Kokkos::Experimental::HIPHostPinnedSpace;
77 };
78 #endif
79 #ifdef KOKKOS_ENABLE_SYCL
80 template <>
81 struct ChunkArraySpace<Kokkos::Experimental::SYCLDeviceUSMSpace> {
82  using memory_space = typename Kokkos::Experimental::SYCLSharedUSMSpace;
83 };
84 #endif
85 } // end namespace Impl
86 
91 template <typename DataType, typename... P>
92 class DynamicView : public Kokkos::ViewTraits<DataType, P...> {
93  public:
94  using traits = Kokkos::ViewTraits<DataType, P...>;
95 
96  private:
97  template <class, class...>
98  friend class DynamicView;
99 
100  using track_type = Kokkos::Impl::SharedAllocationTracker;
101 
102  static_assert(traits::rank == 1 && traits::rank_dynamic == 1,
103  "DynamicView must be rank-one");
104 
105  // It is assumed that the value_type is trivially copyable;
106  // when this is not the case, potential problems can occur.
107  static_assert(std::is_same<typename traits::specialize, void>::value,
108  "DynamicView only implemented for non-specialized View type");
109 
110  template <class Space, bool = Kokkos::Impl::MemorySpaceAccess<
111  Space, typename traits::memory_space>::accessible>
112  struct verify_space {
113  KOKKOS_FORCEINLINE_FUNCTION static void check() {}
114  };
115 
116  template <class Space>
117  struct verify_space<Space, false> {
118  KOKKOS_FORCEINLINE_FUNCTION static void check() {
119  Kokkos::abort(
120  "Kokkos::DynamicView ERROR: attempt to access inaccessible memory "
121  "space");
122  };
123  };
124 
125  private:
126  track_type m_track;
127  typename traits::value_type** m_chunks =
128  nullptr; // array of pointers to 'chunks' of memory
129  unsigned m_chunk_shift; // ceil(log2(m_chunk_size))
130  unsigned m_chunk_mask; // m_chunk_size - 1
131  unsigned m_chunk_max; // number of entries in the chunk array - each pointing
132  // to a chunk of extent == m_chunk_size entries
133  unsigned m_chunk_size; // 2 << (m_chunk_shift - 1)
134 
135  public:
136  //----------------------------------------------------------------------
137 
139  using array_type =
141 
143  using const_type = DynamicView<typename traits::const_data_type,
144  typename traits::device_type>;
145 
147  using non_const_type = DynamicView<typename traits::non_const_data_type,
148  typename traits::device_type>;
149 
152 
154  using uniform_device =
155  Kokkos::Device<typename traits::device_type::execution_space,
156  Kokkos::AnonymousSpace>;
157  using uniform_type = array_type;
169 
170  //----------------------------------------------------------------------
171 
172  enum { Rank = 1 };
173 
174  KOKKOS_INLINE_FUNCTION
175  size_t allocation_extent() const noexcept {
176  uintptr_t n = *reinterpret_cast<const uintptr_t*>(m_chunks + m_chunk_max);
177  return (n << m_chunk_shift);
178  }
179 
180  KOKKOS_INLINE_FUNCTION
181  size_t chunk_size() const noexcept { return m_chunk_size; }
182 
183  KOKKOS_INLINE_FUNCTION
184  size_t size() const noexcept {
185  size_t extent_0 =
186  *reinterpret_cast<const size_t*>(m_chunks + m_chunk_max + 1);
187  return extent_0;
188  }
189 
190  template <typename iType>
191  KOKKOS_INLINE_FUNCTION size_t extent(const iType& r) const {
192  return r == 0 ? size() : 1;
193  }
194 
195  template <typename iType>
196  KOKKOS_INLINE_FUNCTION size_t extent_int(const iType& r) const {
197  return r == 0 ? size() : 1;
198  }
199 
200  KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { return 0; }
201  KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { return 0; }
202  KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { return 0; }
203  KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { return 0; }
204  KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { return 0; }
205  KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { return 0; }
206  KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { return 0; }
207  KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { return 0; }
208 
209  template <typename iType>
210  KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
211  *s = 0;
212  }
213 
214  //----------------------------------------
215  // Allocation tracking properties
216 
217  KOKKOS_INLINE_FUNCTION
218  int use_count() const { return m_track.use_count(); }
219 
220  inline const std::string label() const {
221  return m_track.template get_label<typename traits::memory_space>();
222  }
223 
224  //----------------------------------------------------------------------
225  // Range span is the span which contains all members.
226 
227  using reference_type = typename traits::value_type&;
228  using pointer_type = typename traits::value_type*;
229 
230  enum {
231  reference_type_is_lvalue_reference =
232  std::is_lvalue_reference<reference_type>::value
233  };
234 
235  KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
236  return false;
237  }
238  KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return 0; }
239  KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { return 0; }
240 
241  //----------------------------------------
242 
243  template <typename I0, class... Args>
244  KOKKOS_INLINE_FUNCTION reference_type
245  operator()(const I0& i0, const Args&... /*args*/) const {
246  static_assert(Kokkos::Impl::are_integral<I0, Args...>::value,
247  "Indices must be integral type");
248 
249  DynamicView::template verify_space<
250  Kokkos::Impl::ActiveExecutionMemorySpace>::check();
251 
252  // Which chunk is being indexed.
253  const uintptr_t ic = uintptr_t(i0 >> m_chunk_shift);
254 
255  typename traits::value_type* volatile* const ch = m_chunks + ic;
256 
257  // Do bounds checking if enabled or if the chunk pointer is zero.
258  // If not bounds checking then we assume a non-zero pointer is valid.
259 
260 #if !defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
261  if (nullptr == *ch)
262 #endif
263  {
264  // Verify that allocation of the requested chunk in in progress.
265 
266  // The allocated chunk counter is m_chunks[ m_chunk_max ]
267  const uintptr_t n =
268  *reinterpret_cast<uintptr_t volatile*>(m_chunks + m_chunk_max);
269 
270  if (n <= ic) {
271  Kokkos::abort("Kokkos::DynamicView array bounds error");
272  }
273 
274  // Allocation of this chunk is in progress
275  // so wait for allocation to complete.
276  while (nullptr == *ch)
277  ;
278  }
279 
280  return (*ch)[i0 & m_chunk_mask];
281  }
282 
283  //----------------------------------------
287  template <typename IntType>
288  inline typename std::enable_if<
289  std::is_integral<IntType>::value &&
292  typename Impl::ChunkArraySpace<
293  typename traits::memory_space>::memory_space>::accessible>::type
294  resize_serial(IntType const& n) {
295  using local_value_type = typename traits::value_type;
296  using value_pointer_type = local_value_type*;
297 
298  const uintptr_t NC =
299  (n + m_chunk_mask) >>
300  m_chunk_shift; // New total number of chunks needed for resize
301 
302  if (m_chunk_max < NC) {
303  Kokkos::abort("DynamicView::resize_serial exceeded maximum size");
304  }
305 
306  // *m_chunks[m_chunk_max] stores the current number of chunks being used
307  uintptr_t* const pc = reinterpret_cast<uintptr_t*>(m_chunks + m_chunk_max);
308  std::string _label =
309  m_track.template get_label<typename traits::memory_space>();
310  if (*pc < NC) {
311  while (*pc < NC) {
312  m_chunks[*pc] = reinterpret_cast<value_pointer_type>(
313  typename traits::memory_space().allocate(
314  _label.c_str(), sizeof(local_value_type) << m_chunk_shift));
315  ++*pc;
316  }
317  } else {
318  while (NC + 1 <= *pc) {
319  --*pc;
320  typename traits::memory_space().deallocate(
321  _label.c_str(), m_chunks[*pc],
322  sizeof(local_value_type) << m_chunk_shift);
323  m_chunks[*pc] = nullptr;
324  }
325  }
326  // *m_chunks[m_chunk_max+1] stores the 'extent' requested by resize
327  *(pc + 1) = n;
328  }
329 
330  KOKKOS_INLINE_FUNCTION bool is_allocated() const {
331  if (m_chunks == nullptr) {
332  return false;
333  } else {
334  // *m_chunks[m_chunk_max] stores the current number of chunks being used
335  uintptr_t* const pc =
336  reinterpret_cast<uintptr_t*>(m_chunks + m_chunk_max);
337  return (*(pc + 1) > 0);
338  }
339  }
340 
341  //----------------------------------------------------------------------
342 
343  ~DynamicView() = default;
344  DynamicView() = default;
345  DynamicView(DynamicView&&) = default;
346  DynamicView(const DynamicView&) = default;
347  DynamicView& operator=(DynamicView&&) = default;
348  DynamicView& operator=(const DynamicView&) = default;
349 
350  template <class RT, class... RP>
351  DynamicView(const DynamicView<RT, RP...>& rhs)
352  : m_track(rhs.m_track),
353  m_chunks((typename traits::value_type**)rhs.m_chunks),
354  m_chunk_shift(rhs.m_chunk_shift),
355  m_chunk_mask(rhs.m_chunk_mask),
356  m_chunk_max(rhs.m_chunk_max),
357  m_chunk_size(rhs.m_chunk_size) {
358  using SrcTraits = typename DynamicView<RT, RP...>::traits;
359  using Mapping = Kokkos::Impl::ViewMapping<traits, SrcTraits, void>;
360  static_assert(Mapping::is_assignable,
361  "Incompatible DynamicView copy construction");
362  }
363 
364  //----------------------------------------------------------------------
365 
366  struct Destroy {
367  using local_value_type = typename traits::value_type;
368  std::string m_label;
369  local_value_type** m_chunks;
370  unsigned m_chunk_max;
371  bool m_destroy;
372  unsigned m_chunk_size;
373 
374  // Initialize or destroy array of chunk pointers.
375  // Two entries beyond the max chunks are allocation counters.
376  inline void operator()(unsigned i) const {
377  if (m_destroy && i < m_chunk_max && nullptr != m_chunks[i]) {
378  typename traits::memory_space().deallocate(
379  m_label.c_str(), m_chunks[i],
380  sizeof(local_value_type) * m_chunk_size);
381  }
382  m_chunks[i] = nullptr;
383  }
384 
385  void execute(bool arg_destroy) {
387 
388  m_destroy = arg_destroy;
389 
391  *this,
392  Range(0, m_chunk_max + 2)); // Add 2 to 'destroy' extra slots storing
393  // num_chunks and extent; previously + 1
394 
395  closure.execute();
396 
397  typename traits::execution_space().fence();
398  // Impl::ChunkArraySpace< typename traits::memory_space
399  // >::memory_space::execution_space().fence();
400  }
401 
402  void construct_shared_allocation() { execute(false); }
403 
404  void destroy_shared_allocation() { execute(true); }
405 
406  Destroy() = default;
407  Destroy(Destroy&&) = default;
408  Destroy(const Destroy&) = default;
409  Destroy& operator=(Destroy&&) = default;
410  Destroy& operator=(const Destroy&) = default;
411 
412  Destroy(std::string label, typename traits::value_type** arg_chunk,
413  const unsigned arg_chunk_max, const unsigned arg_chunk_size)
414  : m_label(label),
415  m_chunks(arg_chunk),
416  m_chunk_max(arg_chunk_max),
417  m_destroy(false),
418  m_chunk_size(arg_chunk_size) {}
419  };
420 
427  explicit inline DynamicView(const std::string& arg_label,
428  const unsigned min_chunk_size,
429  const unsigned max_extent)
430  : m_track(),
431  m_chunks(nullptr)
432  // The chunk size is guaranteed to be a power of two
433  ,
434  m_chunk_shift(Kokkos::Impl::integral_power_of_two_that_contains(
435  min_chunk_size)) // div ceil(log2(min_chunk_size))
436  ,
437  m_chunk_mask((1 << m_chunk_shift) - 1) // mod
438  ,
439  m_chunk_max((max_extent + m_chunk_mask) >>
440  m_chunk_shift) // max num pointers-to-chunks in array
441  ,
442  m_chunk_size(2 << (m_chunk_shift - 1)) {
443  using chunk_array_memory_space = typename Impl::ChunkArraySpace<
444  typename traits::memory_space>::memory_space;
445  // A functor to deallocate all of the chunks upon final destruction
446  using record_type =
447  Kokkos::Impl::SharedAllocationRecord<chunk_array_memory_space, Destroy>;
448 
449  // Allocate chunk pointers and allocation counter
450  record_type* const record =
451  record_type::allocate(chunk_array_memory_space(), arg_label,
452  (sizeof(pointer_type) * (m_chunk_max + 2)));
453  // Allocate + 2 extra slots so that *m_chunk[m_chunk_max] ==
454  // num_chunks_alloc and *m_chunk[m_chunk_max+1] == extent This must match in
455  // Destroy's execute(...) method
456 
457  m_chunks = reinterpret_cast<pointer_type*>(record->data());
458 
459  record->m_destroy = Destroy(arg_label, m_chunks, m_chunk_max, m_chunk_size);
460 
461  // Initialize to zero
462  record->m_destroy.construct_shared_allocation();
463 
464  m_track.assign_allocated_record_to_uninitialized(record);
465  }
466 };
467 
468 } // namespace Experimental
469 } // namespace Kokkos
470 
471 namespace Kokkos {
472 
473 template <class T, class... P>
474 inline typename Kokkos::Experimental::DynamicView<T, P...>::HostMirror
475 create_mirror_view(const Kokkos::Experimental::DynamicView<T, P...>& src) {
476  return src;
477 }
478 
479 template <class T, class... DP, class... SP>
480 inline void deep_copy(const View<T, DP...>& dst,
482  using dst_type = View<T, DP...>;
483  using src_type = Kokkos::Experimental::DynamicView<T, SP...>;
484 
485  using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
486  using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
487 
488  enum {
489  DstExecCanAccessSrc =
490  Kokkos::Impl::SpaceAccessibility<dst_execution_space,
491  src_memory_space>::accessible
492  };
493 
494  if (DstExecCanAccessSrc) {
495  // Copying data between views in accessible memory spaces and either
496  // non-contiguous or incompatible shape.
497  Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
498  } else {
499  Kokkos::Impl::throw_runtime_exception(
500  "deep_copy given views that would require a temporary allocation");
501  }
502 }
503 
504 template <class T, class... DP, class... SP>
505 inline void deep_copy(const Kokkos::Experimental::DynamicView<T, DP...>& dst,
506  const View<T, SP...>& src) {
507  using dst_type = Kokkos::Experimental::DynamicView<T, SP...>;
508  using src_type = View<T, DP...>;
509 
510  using dst_execution_space = typename ViewTraits<T, DP...>::execution_space;
511  using src_memory_space = typename ViewTraits<T, SP...>::memory_space;
512 
513  enum {
514  DstExecCanAccessSrc =
515  Kokkos::Impl::SpaceAccessibility<dst_execution_space,
516  src_memory_space>::accessible
517  };
518 
519  if (DstExecCanAccessSrc) {
520  // Copying data between views in accessible memory spaces and either
521  // non-contiguous or incompatible shape.
522  Kokkos::Impl::ViewRemap<dst_type, src_type>(dst, src);
523  } else {
524  Kokkos::Impl::throw_runtime_exception(
525  "deep_copy given views that would require a temporary allocation");
526  }
527 }
528 
529 namespace Impl {
530 template <class Arg0, class... DP, class... SP>
531 struct CommonSubview<Kokkos::Experimental::DynamicView<DP...>,
532  Kokkos::Experimental::DynamicView<SP...>, 1, Arg0> {
533  using DstType = Kokkos::Experimental::DynamicView<DP...>;
534  using SrcType = Kokkos::Experimental::DynamicView<SP...>;
535  using dst_subview_type = DstType;
536  using src_subview_type = SrcType;
537  dst_subview_type dst_sub;
538  src_subview_type src_sub;
539  CommonSubview(const DstType& dst, const SrcType& src, const Arg0& /*arg0*/)
540  : dst_sub(dst), src_sub(src) {}
541 };
542 
543 template <class... DP, class SrcType, class Arg0>
544 struct CommonSubview<Kokkos::Experimental::DynamicView<DP...>, SrcType, 1,
545  Arg0> {
546  using DstType = Kokkos::Experimental::DynamicView<DP...>;
547  using dst_subview_type = DstType;
548  using src_subview_type = typename Kokkos::Subview<SrcType, Arg0>;
549  dst_subview_type dst_sub;
550  src_subview_type src_sub;
551  CommonSubview(const DstType& dst, const SrcType& src, const Arg0& arg0)
552  : dst_sub(dst), src_sub(src, arg0) {}
553 };
554 
555 template <class DstType, class... SP, class Arg0>
556 struct CommonSubview<DstType, Kokkos::Experimental::DynamicView<SP...>, 1,
557  Arg0> {
558  using SrcType = Kokkos::Experimental::DynamicView<SP...>;
559  using dst_subview_type = typename Kokkos::Subview<DstType, Arg0>;
560  using src_subview_type = SrcType;
561  dst_subview_type dst_sub;
562  src_subview_type src_sub;
563  CommonSubview(const DstType& dst, const SrcType& src, const Arg0& arg0)
564  : dst_sub(dst, arg0), src_sub(src) {}
565 };
566 
567 template <class... DP, class ViewTypeB, class Layout, class ExecSpace,
568  typename iType>
569 struct ViewCopy<Kokkos::Experimental::DynamicView<DP...>, ViewTypeB, Layout,
570  ExecSpace, 1, iType> {
572  ViewTypeB b;
573 
575 
576  ViewCopy(const Kokkos::Experimental::DynamicView<DP...>& a_,
577  const ViewTypeB& b_)
578  : a(a_), b(b_) {
579  Kokkos::parallel_for("Kokkos::ViewCopy-1D", policy_type(0, b.extent(0)),
580  *this);
581  }
582 
583  KOKKOS_INLINE_FUNCTION
584  void operator()(const iType& i0) const { a(i0) = b(i0); };
585 };
586 
587 template <class... DP, class... SP, class Layout, class ExecSpace,
588  typename iType>
589 struct ViewCopy<Kokkos::Experimental::DynamicView<DP...>,
590  Kokkos::Experimental::DynamicView<SP...>, Layout, ExecSpace, 1,
591  iType> {
594 
596 
597  ViewCopy(const Kokkos::Experimental::DynamicView<DP...>& a_,
599  : a(a_), b(b_) {
600  const iType n = std::min(a.extent(0), b.extent(0));
601  Kokkos::parallel_for("Kokkos::ViewCopy-1D", policy_type(0, n), *this);
602  }
603 
604  KOKKOS_INLINE_FUNCTION
605  void operator()(const iType& i0) const { a(i0) = b(i0); };
606 };
607 
608 } // namespace Impl
609 } // namespace Kokkos
610 
611 #endif /* #ifndef KOKKOS_DYNAMIC_VIEW_HPP */
View
DynamicView(const std::string &arg_label, const unsigned min_chunk_size, const unsigned max_extent)
Allocation constructor.
DynamicView< typename traits::data_type, typename traits::device_type > array_type
Compatible view of array of scalar types.
Dynamic views are restricted to rank-one and no layout. Resize only occurs on host outside of paralle...
Can AccessSpace access MemorySpace ?
Kokkos::Device< typename traits::device_type::execution_space, Kokkos::AnonymousSpace > uniform_device
Unified types.
DynamicView< typename traits::const_data_type, typename traits::device_type > const_type
Compatible view of const data type.
Memory management for host memory.
Implementation of the ParallelFor operator that has a partial specialization for the device...
Execution policy for work over a range of an integral type.
Traits class for accessing attributes of a View.
Definition: dummy.cpp:3
std::enable_if< std::is_integral< IntType >::value &&Kokkos::Impl::MemorySpaceAccess< Kokkos::HostSpace, typename Impl::ChunkArraySpace< typename traits::memory_space >::memory_space >::accessible >::type resize_serial(IntType const &n)
Resizing in serial can grow or shrink the array size up to the maximum number of chunks.
Access relationship between DstMemorySpace and SrcMemorySpace.