Kokkos Core Kernels Package  Version of the Day
KokkosExp_MDRangePolicy.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 #ifndef KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP
46 #define KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP
47 
48 #include <initializer_list>
49 
50 #include <Kokkos_Layout.hpp>
51 #include <Kokkos_Array.hpp>
52 #include <impl/KokkosExp_Host_IterateTile.hpp>
53 #include <Kokkos_ExecPolicy.hpp>
54 #include <type_traits>
55 
56 namespace Kokkos {
57 
58 // ------------------------------------------------------------------ //
59 // Moved to Kokkos_Layout.hpp for more general accessibility
60 /*
61 enum class Iterate
62 {
63  Default, // Default for the device
64  Left, // Left indices stride fastest
65  Right, // Right indices stride fastest
66 };
67 */
68 
69 template <typename ExecSpace>
70 struct default_outer_direction {
71  using type = Iterate;
72  static constexpr Iterate value = Iterate::Right;
73 };
74 
75 template <typename ExecSpace>
76 struct default_inner_direction {
77  using type = Iterate;
78  static constexpr Iterate value = Iterate::Right;
79 };
80 
81 // Iteration Pattern
82 template <unsigned N, Iterate OuterDir = Iterate::Default,
83  Iterate InnerDir = Iterate::Default>
84 struct Rank {
85  static_assert(N != 0u, "Kokkos Error: rank 0 undefined");
86  static_assert(N != 1u,
87  "Kokkos Error: rank 1 is not a multi-dimensional range");
88  static_assert(N < 7u, "Kokkos Error: Unsupported rank...");
89 
90  using iteration_pattern = Rank<N, OuterDir, InnerDir>;
91 
92  static constexpr int rank = N;
93  static constexpr Iterate outer_direction = OuterDir;
94  static constexpr Iterate inner_direction = InnerDir;
95 };
96 
97 namespace Impl {
98 // NOTE the comparison below is encapsulated to silent warnings about pointless
99 // comparison of unsigned integer with zero
100 template <class T>
101 constexpr std::enable_if_t<!std::is_signed<T>::value, bool>
102 is_less_than_value_initialized_variable(T) {
103  return false;
104 }
105 
106 template <class T>
107 constexpr std::enable_if_t<std::is_signed<T>::value, bool>
108 is_less_than_value_initialized_variable(T arg) {
109  return arg < T{};
110 }
111 
112 // Checked narrowing conversion that calls abort if the cast changes the value
113 template <class To, class From>
114 constexpr To checked_narrow_cast(From arg) {
115  constexpr const bool is_different_signedness =
116  (std::is_signed<To>::value != std::is_signed<From>::value);
117  auto const ret = static_cast<To>(arg);
118  if (static_cast<From>(ret) != arg ||
119  (is_different_signedness &&
120  is_less_than_value_initialized_variable(arg) !=
121  is_less_than_value_initialized_variable(ret))) {
122  Kokkos::abort("unsafe narrowing conversion");
123  }
124  return ret;
125 }
126 // NOTE prefer C array U[M] to std::initalizer_list<U> so that the number of
127 // elements can be deduced (https://stackoverflow.com/q/40241370)
128 // NOTE for some unfortunate reason the policy bounds are stored as signed
129 // integer arrays (point_type which is Kokkos::Array<std::int64_t>) so we
130 // specify the index type (actual policy index_type from the traits) and check
131 // ahead of time that narrowing conversions will be safe.
132 template <class IndexType, class Array, class U, std::size_t M>
133 constexpr Array to_array_potentially_narrowing(const U (&init)[M]) {
134  using T = typename Array::value_type;
135  Array a{};
136  constexpr std::size_t N = a.size();
137  static_assert(M <= N, "");
138  auto* ptr = a.data();
139  // NOTE equivalent to
140  // std::transform(std::begin(init), std::end(init), a.data(),
141  // [](U x) { return static_cast<T>(x); });
142  // except that std::transform is not constexpr.
143  for (auto x : init) {
144  *ptr++ = checked_narrow_cast<T>(x);
145  (void)checked_narrow_cast<IndexType>(x); // see note above
146  }
147  return a;
148 }
149 
150 // NOTE Making a copy even when std::is_same<Array, Kokkos::Array<U, M>>::value
151 // is true to reduce code complexity. You may change this if you have a good
152 // reason to. Intentionally not enabling std::array at this time but this may
153 // change too.
154 template <class IndexType, class NVCC_WONT_LET_ME_CALL_YOU_Array, class U,
155  std::size_t M>
156 constexpr NVCC_WONT_LET_ME_CALL_YOU_Array to_array_potentially_narrowing(
157  Kokkos::Array<U, M> const& other) {
158  using T = typename NVCC_WONT_LET_ME_CALL_YOU_Array::value_type;
159  NVCC_WONT_LET_ME_CALL_YOU_Array a{};
160  constexpr std::size_t N = a.size();
161  static_assert(M <= N, "");
162  for (std::size_t i = 0; i < M; ++i) {
163  a[i] = checked_narrow_cast<T>(other[i]);
164  (void)checked_narrow_cast<IndexType>(other[i]); // see note above
165  }
166  return a;
167 }
168 
169 struct TileSizeProperties {
170  int max_threads;
171  int default_largest_tile_size;
172  int default_tile_size;
173  int max_total_tile_size;
174 };
175 
176 template <typename ExecutionSpace>
177 TileSizeProperties get_tile_size_properties(const ExecutionSpace&) {
178  // Host settings
179  TileSizeProperties properties;
180  properties.max_threads = std::numeric_limits<int>::max();
181  properties.default_largest_tile_size = 0;
182  properties.default_tile_size = 2;
183  properties.max_total_tile_size = std::numeric_limits<int>::max();
184  return properties;
185 }
186 
187 } // namespace Impl
188 
189 // multi-dimensional iteration pattern
190 template <typename... Properties>
191 struct MDRangePolicy : public Kokkos::Impl::PolicyTraits<Properties...> {
192  using traits = Kokkos::Impl::PolicyTraits<Properties...>;
193  using range_policy = RangePolicy<Properties...>;
194 
195  typename traits::execution_space m_space;
196 
197  using impl_range_policy =
198  RangePolicy<typename traits::execution_space,
199  typename traits::schedule_type, typename traits::index_type>;
200 
201  using execution_policy =
202  MDRangePolicy<Properties...>; // needed for is_execution_space
203  // interrogation
204 
205  template <class... OtherProperties>
206  friend struct MDRangePolicy;
207 
208  static_assert(!std::is_same<typename traits::iteration_pattern, void>::value,
209  "Kokkos Error: MD iteration pattern not defined");
210 
211  using iteration_pattern = typename traits::iteration_pattern;
212  using work_tag = typename traits::work_tag;
213  using launch_bounds = typename traits::launch_bounds;
214  using member_type = typename range_policy::member_type;
215 
216  static constexpr int rank = iteration_pattern::rank;
217 
218  using index_type = typename traits::index_type;
219  using array_index_type = std::int64_t;
220  using point_type = Kokkos::Array<array_index_type, rank>; // was index_type
221  using tile_type = Kokkos::Array<array_index_type, rank>;
222  // If point_type or tile_type is not templated on a signed integral type (if
223  // it is unsigned), then if user passes in intializer_list of
224  // runtime-determined values of signed integral type that are not const will
225  // receive a compiler error due to an invalid case for implicit conversion -
226  // "conversion from integer or unscoped enumeration type to integer type that
227  // cannot represent all values of the original, except where source is a
228  // constant expression whose value can be stored exactly in the target type"
229  // This would require the user to either pass a matching index_type parameter
230  // as template parameter to the MDRangePolicy or static_cast the individual
231  // values
232 
233  point_type m_lower = {};
234  point_type m_upper = {};
235  tile_type m_tile = {};
236  point_type m_tile_end = {};
237  index_type m_num_tiles = 1;
238  index_type m_prod_tile_dims = 1;
239  bool m_tune_tile_size = false;
240 
241  static constexpr auto outer_direction =
242  (iteration_pattern::outer_direction != Iterate::Default)
243  ? iteration_pattern::outer_direction
244  : default_outer_direction<typename traits::execution_space>::value;
245 
246  static constexpr auto inner_direction =
247  iteration_pattern::inner_direction != Iterate::Default
248  ? iteration_pattern::inner_direction
249  : default_inner_direction<typename traits::execution_space>::value;
250 
251  static constexpr auto Right = Iterate::Right;
252  static constexpr auto Left = Iterate::Left;
253 
254  KOKKOS_INLINE_FUNCTION const typename traits::execution_space& space() const {
255  return m_space;
256  }
257 
258  MDRangePolicy() = default;
259 
260  template <typename LT, std::size_t LN, typename UT, std::size_t UN,
261  typename TT = array_index_type, std::size_t TN = rank,
262  typename = std::enable_if_t<std::is_integral<LT>::value &&
263  std::is_integral<UT>::value &&
264  std::is_integral<TT>::value>>
265  MDRangePolicy(const LT (&lower)[LN], const UT (&upper)[UN],
266  const TT (&tile)[TN] = {})
267  : MDRangePolicy(
268  Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
269  lower),
270  Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
271  upper),
272  Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
273  tile)) {
274  static_assert(
275  LN == rank && UN == rank && TN <= rank,
276  "MDRangePolicy: Constructor initializer lists have wrong size");
277  }
278 
279  template <typename LT, std::size_t LN, typename UT, std::size_t UN,
280  typename TT = array_index_type, std::size_t TN = rank,
281  typename = std::enable_if_t<std::is_integral<LT>::value &&
282  std::is_integral<UT>::value &&
283  std::is_integral<TT>::value>>
284  MDRangePolicy(const typename traits::execution_space& work_space,
285  const LT (&lower)[LN], const UT (&upper)[UN],
286  const TT (&tile)[TN] = {})
287  : MDRangePolicy(
288  work_space,
289  Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
290  lower),
291  Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
292  upper),
293  Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
294  tile)) {
295  static_assert(
296  LN == rank && UN == rank && TN <= rank,
297  "MDRangePolicy: Constructor initializer lists have wrong size");
298  }
299 
300  // NOTE: Keeping these two constructor despite the templated constructors
301  // from Kokkos arrays for backwards compability to allow construction from
302  // double-braced initializer lists.
303  MDRangePolicy(point_type const& lower, point_type const& upper,
304  tile_type const& tile = tile_type{})
305  : MDRangePolicy(typename traits::execution_space(), lower, upper, tile) {}
306 
307  MDRangePolicy(const typename traits::execution_space& work_space,
308  point_type const& lower, point_type const& upper,
309  tile_type const& tile = tile_type{})
310  : m_space(work_space), m_lower(lower), m_upper(upper), m_tile(tile) {
311  init_helper(Impl::get_tile_size_properties(work_space));
312  }
313 
314  template <typename T, std::size_t NT = rank,
315  typename = std::enable_if_t<std::is_integral<T>::value>>
316  MDRangePolicy(Kokkos::Array<T, rank> const& lower,
317  Kokkos::Array<T, rank> const& upper,
319  : MDRangePolicy(typename traits::execution_space(), lower, upper, tile) {}
320 
321  template <typename T, std::size_t NT = rank,
322  typename = std::enable_if_t<std::is_integral<T>::value>>
323  MDRangePolicy(const typename traits::execution_space& work_space,
324  Kokkos::Array<T, rank> const& lower,
325  Kokkos::Array<T, rank> const& upper,
327  : MDRangePolicy(
328  work_space,
329  Impl::to_array_potentially_narrowing<index_type, decltype(m_lower)>(
330  lower),
331  Impl::to_array_potentially_narrowing<index_type, decltype(m_upper)>(
332  upper),
333  Impl::to_array_potentially_narrowing<index_type, decltype(m_tile)>(
334  tile)) {}
335 
336  template <class... OtherProperties>
337  MDRangePolicy(const MDRangePolicy<OtherProperties...> p)
338  : traits(p), // base class may contain data such as desired occupancy
339  m_space(p.m_space),
340  m_lower(p.m_lower),
341  m_upper(p.m_upper),
342  m_tile(p.m_tile),
343  m_tile_end(p.m_tile_end),
344  m_num_tiles(p.m_num_tiles),
345  m_prod_tile_dims(p.m_prod_tile_dims),
346  m_tune_tile_size(p.m_tune_tile_size) {}
347 
348  void impl_change_tile_size(const point_type& tile) {
349  m_tile = tile;
350  init_helper(Impl::get_tile_size_properties(m_space));
351  }
352  bool impl_tune_tile_size() const { return m_tune_tile_size; }
353 
354  private:
355  void init_helper(Impl::TileSizeProperties properties) {
356  m_prod_tile_dims = 1;
357  int increment = 1;
358  int rank_start = 0;
359  int rank_end = rank;
360  if (inner_direction == Iterate::Right) {
361  increment = -1;
362  rank_start = rank - 1;
363  rank_end = -1;
364  }
365  for (int i = rank_start; i != rank_end; i += increment) {
366  const index_type length = m_upper[i] - m_lower[i];
367  if (m_tile[i] <= 0) {
368  m_tune_tile_size = true;
369  if ((inner_direction == Iterate::Right && (i < rank - 1)) ||
370  (inner_direction == Iterate::Left && (i > 0))) {
371  if (m_prod_tile_dims * properties.default_tile_size <
372  static_cast<index_type>(properties.max_total_tile_size)) {
373  m_tile[i] = properties.default_tile_size;
374  } else {
375  m_tile[i] = 1;
376  }
377  } else {
378  m_tile[i] = properties.default_largest_tile_size == 0
379  ? std::max<int>(length, 1)
380  : properties.default_largest_tile_size;
381  }
382  }
383  m_tile_end[i] =
384  static_cast<index_type>((length + m_tile[i] - 1) / m_tile[i]);
385  m_num_tiles *= m_tile_end[i];
386  m_prod_tile_dims *= m_tile[i];
387  }
388  if (m_prod_tile_dims > static_cast<index_type>(properties.max_threads)) {
389  printf(" Product of tile dimensions exceed maximum limit: %d\n",
390  static_cast<int>(properties.max_threads));
391  Kokkos::abort(
392  "ExecSpace Error: MDRange tile dims exceed maximum number "
393  "of threads per block - choose smaller tile dims");
394  }
395  }
396 };
397 
398 } // namespace Kokkos
399 
400 // For backward compatibility
401 namespace Kokkos {
402 namespace Experimental {
403 using Kokkos::Iterate;
404 using Kokkos::MDRangePolicy;
405 using Kokkos::Rank;
406 } // namespace Experimental
407 } // namespace Kokkos
408 
409 #endif // KOKKOS_CORE_EXP_MD_RANGE_POLICY_HPP
Derived from the C++17 &#39;std::array&#39;. Dropping the iterator interface.
Declaration of various MemoryLayout options.
Definition: dummy.cpp:3