Kokkos Core Kernels Package  Version of the Day
Kokkos_Parallel_Reduce.hpp
1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 #ifndef KOKKOS_PARALLEL_REDUCE_HPP
46 #define KOKKOS_PARALLEL_REDUCE_HPP
47 
48 #include <Kokkos_NumericTraits.hpp>
49 #include <Kokkos_View.hpp>
50 #include <impl/Kokkos_FunctorAnalysis.hpp>
51 #include <impl/Kokkos_FunctorAdapter.hpp>
52 #include <type_traits>
53 
54 namespace Kokkos {
55 
56 template <class T, class Enable = void>
57 struct is_reducer_type {
58  enum { value = 0 };
59 };
60 
61 template <class T>
62 struct is_reducer_type<
63  T, typename std::enable_if<std::is_same<
64  typename std::remove_cv<T>::type,
65  typename std::remove_cv<typename T::reducer>::type>::value>::type> {
66  enum { value = 1 };
67 };
68 
69 template <class Scalar, class Space>
70 struct Sum {
71  public:
72  // Required
73  using reducer = Sum<Scalar, Space>;
74  using value_type = typename std::remove_cv<Scalar>::type;
75 
76  using result_view_type = Kokkos::View<value_type, Space>;
77 
78  private:
79  result_view_type value;
80  bool references_scalar_v;
81 
82  public:
83  KOKKOS_INLINE_FUNCTION
84  Sum(value_type& value_) : value(&value_), references_scalar_v(true) {}
85 
86  KOKKOS_INLINE_FUNCTION
87  Sum(const result_view_type& value_)
88  : value(value_), references_scalar_v(false) {}
89 
90  // Required
91  KOKKOS_INLINE_FUNCTION
92  void join(value_type& dest, const value_type& src) const { dest += src; }
93 
94  KOKKOS_INLINE_FUNCTION
95  void join(volatile value_type& dest, const volatile value_type& src) const {
96  dest += src;
97  }
98 
99  KOKKOS_INLINE_FUNCTION
100  void init(value_type& val) const {
101  val = reduction_identity<value_type>::sum();
102  }
103 
104  KOKKOS_INLINE_FUNCTION
105  value_type& reference() const { return *value.data(); }
106 
107  KOKKOS_INLINE_FUNCTION
108  result_view_type view() const { return value; }
109 
110  KOKKOS_INLINE_FUNCTION
111  bool references_scalar() const { return references_scalar_v; }
112 };
113 
114 template <class Scalar, class Space>
115 struct Prod {
116  public:
117  // Required
118  using reducer = Prod<Scalar, Space>;
119  using value_type = typename std::remove_cv<Scalar>::type;
120 
121  using result_view_type = Kokkos::View<value_type, Space>;
122 
123  private:
124  result_view_type value;
125  bool references_scalar_v;
126 
127  public:
128  KOKKOS_INLINE_FUNCTION
129  Prod(value_type& value_) : value(&value_), references_scalar_v(true) {}
130 
131  KOKKOS_INLINE_FUNCTION
132  Prod(const result_view_type& value_)
133  : value(value_), references_scalar_v(false) {}
134 
135  // Required
136  KOKKOS_INLINE_FUNCTION
137  void join(value_type& dest, const value_type& src) const { dest *= src; }
138 
139  KOKKOS_INLINE_FUNCTION
140  void join(volatile value_type& dest, const volatile value_type& src) const {
141  dest *= src;
142  }
143 
144  KOKKOS_INLINE_FUNCTION
145  void init(value_type& val) const {
146  val = reduction_identity<value_type>::prod();
147  }
148 
149  KOKKOS_INLINE_FUNCTION
150  value_type& reference() const { return *value.data(); }
151 
152  KOKKOS_INLINE_FUNCTION
153  result_view_type view() const { return value; }
154 
155  KOKKOS_INLINE_FUNCTION
156  bool references_scalar() const { return references_scalar_v; }
157 };
158 
159 template <class Scalar, class Space>
160 struct Min {
161  public:
162  // Required
163  using reducer = Min<Scalar, Space>;
164  using value_type = typename std::remove_cv<Scalar>::type;
165 
166  using result_view_type = Kokkos::View<value_type, Space>;
167 
168  private:
169  result_view_type value;
170  bool references_scalar_v;
171 
172  public:
173  KOKKOS_INLINE_FUNCTION
174  Min(value_type& value_) : value(&value_), references_scalar_v(true) {}
175 
176  KOKKOS_INLINE_FUNCTION
177  Min(const result_view_type& value_)
178  : value(value_), references_scalar_v(false) {}
179 
180  // Required
181  KOKKOS_INLINE_FUNCTION
182  void join(value_type& dest, const value_type& src) const {
183  if (src < dest) dest = src;
184  }
185 
186  KOKKOS_INLINE_FUNCTION
187  void join(volatile value_type& dest, const volatile value_type& src) const {
188  if (src < dest) dest = src;
189  }
190 
191  KOKKOS_INLINE_FUNCTION
192  void init(value_type& val) const {
193  val = reduction_identity<value_type>::min();
194  }
195 
196  KOKKOS_INLINE_FUNCTION
197  value_type& reference() const { return *value.data(); }
198 
199  KOKKOS_INLINE_FUNCTION
200  result_view_type view() const { return value; }
201 
202  KOKKOS_INLINE_FUNCTION
203  bool references_scalar() const { return references_scalar_v; }
204 };
205 
206 template <class Scalar, class Space>
207 struct Max {
208  public:
209  // Required
210  using reducer = Max<Scalar, Space>;
211  using value_type = typename std::remove_cv<Scalar>::type;
212 
213  using result_view_type = Kokkos::View<value_type, Space>;
214 
215  private:
216  result_view_type value;
217  bool references_scalar_v;
218 
219  public:
220  KOKKOS_INLINE_FUNCTION
221  Max(value_type& value_) : value(&value_), references_scalar_v(true) {}
222 
223  KOKKOS_INLINE_FUNCTION
224  Max(const result_view_type& value_)
225  : value(value_), references_scalar_v(false) {}
226 
227  // Required
228  KOKKOS_INLINE_FUNCTION
229  void join(value_type& dest, const value_type& src) const {
230  if (src > dest) dest = src;
231  }
232 
233  KOKKOS_INLINE_FUNCTION
234  void join(volatile value_type& dest, const volatile value_type& src) const {
235  if (src > dest) dest = src;
236  }
237 
238  // Required
239  KOKKOS_INLINE_FUNCTION
240  void init(value_type& val) const {
241  val = reduction_identity<value_type>::max();
242  }
243 
244  KOKKOS_INLINE_FUNCTION
245  value_type& reference() const { return *value.data(); }
246 
247  KOKKOS_INLINE_FUNCTION
248  result_view_type view() const { return value; }
249 
250  KOKKOS_INLINE_FUNCTION
251  bool references_scalar() const { return references_scalar_v; }
252 };
253 
254 template <class Scalar, class Space>
255 struct LAnd {
256  public:
257  // Required
258  using reducer = LAnd<Scalar, Space>;
259  using value_type = typename std::remove_cv<Scalar>::type;
260 
261  using result_view_type = Kokkos::View<value_type, Space>;
262 
263  private:
264  result_view_type value;
265  bool references_scalar_v;
266 
267  public:
268  KOKKOS_INLINE_FUNCTION
269  LAnd(value_type& value_) : value(&value_), references_scalar_v(true) {}
270 
271  KOKKOS_INLINE_FUNCTION
272  LAnd(const result_view_type& value_)
273  : value(value_), references_scalar_v(false) {}
274 
275  KOKKOS_INLINE_FUNCTION
276  void join(value_type& dest, const value_type& src) const {
277  dest = dest && src;
278  }
279 
280  KOKKOS_INLINE_FUNCTION
281  void join(volatile value_type& dest, const volatile value_type& src) const {
282  dest = dest && src;
283  }
284 
285  KOKKOS_INLINE_FUNCTION
286  void init(value_type& val) const {
287  val = reduction_identity<value_type>::land();
288  }
289 
290  KOKKOS_INLINE_FUNCTION
291  value_type& reference() const { return *value.data(); }
292 
293  KOKKOS_INLINE_FUNCTION
294  result_view_type view() const { return value; }
295 
296  KOKKOS_INLINE_FUNCTION
297  bool references_scalar() const { return references_scalar_v; }
298 };
299 
300 template <class Scalar, class Space>
301 struct LOr {
302  public:
303  // Required
304  using reducer = LOr<Scalar, Space>;
305  using value_type = typename std::remove_cv<Scalar>::type;
306 
307  using result_view_type = Kokkos::View<value_type, Space>;
308 
309  private:
310  result_view_type value;
311  bool references_scalar_v;
312 
313  public:
314  KOKKOS_INLINE_FUNCTION
315  LOr(value_type& value_) : value(&value_), references_scalar_v(true) {}
316 
317  KOKKOS_INLINE_FUNCTION
318  LOr(const result_view_type& value_)
319  : value(value_), references_scalar_v(false) {}
320 
321  // Required
322  KOKKOS_INLINE_FUNCTION
323  void join(value_type& dest, const value_type& src) const {
324  dest = dest || src;
325  }
326 
327  KOKKOS_INLINE_FUNCTION
328  void join(volatile value_type& dest, const volatile value_type& src) const {
329  dest = dest || src;
330  }
331 
332  KOKKOS_INLINE_FUNCTION
333  void init(value_type& val) const {
334  val = reduction_identity<value_type>::lor();
335  }
336 
337  KOKKOS_INLINE_FUNCTION
338  value_type& reference() const { return *value.data(); }
339 
340  KOKKOS_INLINE_FUNCTION
341  result_view_type view() const { return value; }
342 
343  KOKKOS_INLINE_FUNCTION
344  bool references_scalar() const { return references_scalar_v; }
345 };
346 
347 template <class Scalar, class Space>
348 struct BAnd {
349  public:
350  // Required
351  using reducer = BAnd<Scalar, Space>;
352  using value_type = typename std::remove_cv<Scalar>::type;
353 
354  using result_view_type = Kokkos::View<value_type, Space>;
355 
356  private:
357  result_view_type value;
358  bool references_scalar_v;
359 
360  public:
361  KOKKOS_INLINE_FUNCTION
362  BAnd(value_type& value_) : value(&value_), references_scalar_v(true) {}
363 
364  KOKKOS_INLINE_FUNCTION
365  BAnd(const result_view_type& value_)
366  : value(value_), references_scalar_v(false) {}
367 
368  // Required
369  KOKKOS_INLINE_FUNCTION
370  void join(value_type& dest, const value_type& src) const {
371  dest = dest & src;
372  }
373 
374  KOKKOS_INLINE_FUNCTION
375  void join(volatile value_type& dest, const volatile value_type& src) const {
376  dest = dest & src;
377  }
378 
379  KOKKOS_INLINE_FUNCTION
380  void init(value_type& val) const {
381  val = reduction_identity<value_type>::band();
382  }
383 
384  KOKKOS_INLINE_FUNCTION
385  value_type& reference() const { return *value.data(); }
386 
387  KOKKOS_INLINE_FUNCTION
388  result_view_type view() const { return value; }
389 
390  KOKKOS_INLINE_FUNCTION
391  bool references_scalar() const { return references_scalar_v; }
392 };
393 
394 template <class Scalar, class Space>
395 struct BOr {
396  public:
397  // Required
398  using reducer = BOr<Scalar, Space>;
399  using value_type = typename std::remove_cv<Scalar>::type;
400 
401  using result_view_type = Kokkos::View<value_type, Space>;
402 
403  private:
404  result_view_type value;
405  bool references_scalar_v;
406 
407  public:
408  KOKKOS_INLINE_FUNCTION
409  BOr(value_type& value_) : value(&value_), references_scalar_v(true) {}
410 
411  KOKKOS_INLINE_FUNCTION
412  BOr(const result_view_type& value_)
413  : value(value_), references_scalar_v(false) {}
414 
415  // Required
416  KOKKOS_INLINE_FUNCTION
417  void join(value_type& dest, const value_type& src) const {
418  dest = dest | src;
419  }
420 
421  KOKKOS_INLINE_FUNCTION
422  void join(volatile value_type& dest, const volatile value_type& src) const {
423  dest = dest | src;
424  }
425 
426  KOKKOS_INLINE_FUNCTION
427  void init(value_type& val) const {
428  val = reduction_identity<value_type>::bor();
429  }
430 
431  KOKKOS_INLINE_FUNCTION
432  value_type& reference() const { return *value.data(); }
433 
434  KOKKOS_INLINE_FUNCTION
435  result_view_type view() const { return value; }
436 
437  KOKKOS_INLINE_FUNCTION
438  bool references_scalar() const { return references_scalar_v; }
439 };
440 
441 template <class Scalar, class Index>
442 struct ValLocScalar {
443  Scalar val;
444  Index loc;
445 
446  KOKKOS_INLINE_FUNCTION
447  void operator=(const ValLocScalar& rhs) {
448  val = rhs.val;
449  loc = rhs.loc;
450  }
451 
452  KOKKOS_INLINE_FUNCTION
453  void operator=(const volatile ValLocScalar& rhs) volatile {
454  val = rhs.val;
455  loc = rhs.loc;
456  }
457 };
458 
459 template <class Scalar, class Index, class Space>
460 struct MinLoc {
461  private:
462  using scalar_type = typename std::remove_cv<Scalar>::type;
463  using index_type = typename std::remove_cv<Index>::type;
464 
465  public:
466  // Required
467  using reducer = MinLoc<Scalar, Index, Space>;
468  using value_type = ValLocScalar<scalar_type, index_type>;
469 
470  using result_view_type = Kokkos::View<value_type, Space>;
471 
472  private:
473  result_view_type value;
474  bool references_scalar_v;
475 
476  public:
477  KOKKOS_INLINE_FUNCTION
478  MinLoc(value_type& value_) : value(&value_), references_scalar_v(true) {}
479 
480  KOKKOS_INLINE_FUNCTION
481  MinLoc(const result_view_type& value_)
482  : value(value_), references_scalar_v(false) {}
483 
484  // Required
485  KOKKOS_INLINE_FUNCTION
486  void join(value_type& dest, const value_type& src) const {
487  if (src.val < dest.val) dest = src;
488  }
489 
490  KOKKOS_INLINE_FUNCTION
491  void join(volatile value_type& dest, const volatile value_type& src) const {
492  if (src.val < dest.val) dest = src;
493  }
494 
495  KOKKOS_INLINE_FUNCTION
496  void init(value_type& val) const {
497  val.val = reduction_identity<scalar_type>::min();
498  val.loc = reduction_identity<index_type>::min();
499  }
500 
501  KOKKOS_INLINE_FUNCTION
502  value_type& reference() const { return *value.data(); }
503 
504  KOKKOS_INLINE_FUNCTION
505  result_view_type view() const { return value; }
506 
507  KOKKOS_INLINE_FUNCTION
508  bool references_scalar() const { return references_scalar_v; }
509 };
510 
511 template <class Scalar, class Index, class Space>
512 struct MaxLoc {
513  private:
514  using scalar_type = typename std::remove_cv<Scalar>::type;
515  using index_type = typename std::remove_cv<Index>::type;
516 
517  public:
518  // Required
519  using reducer = MaxLoc<Scalar, Index, Space>;
520  using value_type = ValLocScalar<scalar_type, index_type>;
521 
522  using result_view_type = Kokkos::View<value_type, Space>;
523 
524  private:
525  result_view_type value;
526  bool references_scalar_v;
527 
528  public:
529  KOKKOS_INLINE_FUNCTION
530  MaxLoc(value_type& value_) : value(&value_), references_scalar_v(true) {}
531 
532  KOKKOS_INLINE_FUNCTION
533  MaxLoc(const result_view_type& value_)
534  : value(value_), references_scalar_v(false) {}
535 
536  // Required
537  KOKKOS_INLINE_FUNCTION
538  void join(value_type& dest, const value_type& src) const {
539  if (src.val > dest.val) dest = src;
540  }
541 
542  KOKKOS_INLINE_FUNCTION
543  void join(volatile value_type& dest, const volatile value_type& src) const {
544  if (src.val > dest.val) dest = src;
545  }
546 
547  KOKKOS_INLINE_FUNCTION
548  void init(value_type& val) const {
549  val.val = reduction_identity<scalar_type>::max();
550  val.loc = reduction_identity<index_type>::min();
551  }
552 
553  KOKKOS_INLINE_FUNCTION
554  value_type& reference() const { return *value.data(); }
555 
556  KOKKOS_INLINE_FUNCTION
557  result_view_type view() const { return value; }
558 
559  KOKKOS_INLINE_FUNCTION
560  bool references_scalar() const { return references_scalar_v; }
561 };
562 
563 template <class Scalar>
564 struct MinMaxScalar {
565  Scalar min_val, max_val;
566 
567  KOKKOS_INLINE_FUNCTION
568  void operator=(const MinMaxScalar& rhs) {
569  min_val = rhs.min_val;
570  max_val = rhs.max_val;
571  }
572 
573  KOKKOS_INLINE_FUNCTION
574  void operator=(const volatile MinMaxScalar& rhs) volatile {
575  min_val = rhs.min_val;
576  max_val = rhs.max_val;
577  }
578 };
579 
580 template <class Scalar, class Space>
581 struct MinMax {
582  private:
583  using scalar_type = typename std::remove_cv<Scalar>::type;
584 
585  public:
586  // Required
587  using reducer = MinMax<Scalar, Space>;
588  using value_type = MinMaxScalar<scalar_type>;
589 
590  using result_view_type = Kokkos::View<value_type, Space>;
591 
592  private:
593  result_view_type value;
594  bool references_scalar_v;
595 
596  public:
597  KOKKOS_INLINE_FUNCTION
598  MinMax(value_type& value_) : value(&value_), references_scalar_v(true) {}
599 
600  KOKKOS_INLINE_FUNCTION
601  MinMax(const result_view_type& value_)
602  : value(value_), references_scalar_v(false) {}
603 
604  // Required
605  KOKKOS_INLINE_FUNCTION
606  void join(value_type& dest, const value_type& src) const {
607  if (src.min_val < dest.min_val) {
608  dest.min_val = src.min_val;
609  }
610  if (src.max_val > dest.max_val) {
611  dest.max_val = src.max_val;
612  }
613  }
614 
615  KOKKOS_INLINE_FUNCTION
616  void join(volatile value_type& dest, const volatile value_type& src) const {
617  if (src.min_val < dest.min_val) {
618  dest.min_val = src.min_val;
619  }
620  if (src.max_val > dest.max_val) {
621  dest.max_val = src.max_val;
622  }
623  }
624 
625  KOKKOS_INLINE_FUNCTION
626  void init(value_type& val) const {
627  val.max_val = reduction_identity<scalar_type>::max();
628  val.min_val = reduction_identity<scalar_type>::min();
629  }
630 
631  KOKKOS_INLINE_FUNCTION
632  value_type& reference() const { return *value.data(); }
633 
634  KOKKOS_INLINE_FUNCTION
635  result_view_type view() const { return value; }
636 
637  KOKKOS_INLINE_FUNCTION
638  bool references_scalar() const { return references_scalar_v; }
639 };
640 
641 template <class Scalar, class Index>
642 struct MinMaxLocScalar {
643  Scalar min_val, max_val;
644  Index min_loc, max_loc;
645 
646  KOKKOS_INLINE_FUNCTION
647  void operator=(const MinMaxLocScalar& rhs) {
648  min_val = rhs.min_val;
649  min_loc = rhs.min_loc;
650  max_val = rhs.max_val;
651  max_loc = rhs.max_loc;
652  }
653 
654  KOKKOS_INLINE_FUNCTION
655  void operator=(const volatile MinMaxLocScalar& rhs) volatile {
656  min_val = rhs.min_val;
657  min_loc = rhs.min_loc;
658  max_val = rhs.max_val;
659  max_loc = rhs.max_loc;
660  }
661 };
662 
663 template <class Scalar, class Index, class Space>
664 struct MinMaxLoc {
665  private:
666  using scalar_type = typename std::remove_cv<Scalar>::type;
667  using index_type = typename std::remove_cv<Index>::type;
668 
669  public:
670  // Required
671  using reducer = MinMaxLoc<Scalar, Index, Space>;
672  using value_type = MinMaxLocScalar<scalar_type, index_type>;
673 
674  using result_view_type = Kokkos::View<value_type, Space>;
675 
676  private:
677  result_view_type value;
678  bool references_scalar_v;
679 
680  public:
681  KOKKOS_INLINE_FUNCTION
682  MinMaxLoc(value_type& value_) : value(&value_), references_scalar_v(true) {}
683 
684  KOKKOS_INLINE_FUNCTION
685  MinMaxLoc(const result_view_type& value_)
686  : value(value_), references_scalar_v(false) {}
687 
688  // Required
689  KOKKOS_INLINE_FUNCTION
690  void join(value_type& dest, const value_type& src) const {
691  if (src.min_val < dest.min_val) {
692  dest.min_val = src.min_val;
693  dest.min_loc = src.min_loc;
694  }
695  if (src.max_val > dest.max_val) {
696  dest.max_val = src.max_val;
697  dest.max_loc = src.max_loc;
698  }
699  }
700 
701  KOKKOS_INLINE_FUNCTION
702  void join(volatile value_type& dest, const volatile value_type& src) const {
703  if (src.min_val < dest.min_val) {
704  dest.min_val = src.min_val;
705  dest.min_loc = src.min_loc;
706  }
707  if (src.max_val > dest.max_val) {
708  dest.max_val = src.max_val;
709  dest.max_loc = src.max_loc;
710  }
711  }
712 
713  KOKKOS_INLINE_FUNCTION
714  void init(value_type& val) const {
715  val.max_val = reduction_identity<scalar_type>::max();
716  val.min_val = reduction_identity<scalar_type>::min();
717  val.max_loc = reduction_identity<index_type>::min();
718  val.min_loc = reduction_identity<index_type>::min();
719  }
720 
721  KOKKOS_INLINE_FUNCTION
722  value_type& reference() const { return *value.data(); }
723 
724  KOKKOS_INLINE_FUNCTION
725  result_view_type view() const { return value; }
726 
727  KOKKOS_INLINE_FUNCTION
728  bool references_scalar() const { return references_scalar_v; }
729 };
730 } // namespace Kokkos
731 namespace Kokkos {
732 namespace Impl {
733 
734 template <class T, class ReturnType, class ValueTraits>
735 struct ParallelReduceReturnValue;
736 
737 template <class ReturnType, class FunctorType>
738 struct ParallelReduceReturnValue<
739  typename std::enable_if<Kokkos::is_view<ReturnType>::value>::type,
740  ReturnType, FunctorType> {
741  using return_type = ReturnType;
742  using reducer_type = InvalidType;
743 
744  using value_type_scalar = typename return_type::value_type;
745  using value_type_array = typename return_type::value_type* const;
746 
747  using value_type = std::conditional_t<return_type::rank == 0,
748  value_type_scalar, value_type_array>;
749 
750  static return_type& return_value(ReturnType& return_val, const FunctorType&) {
751  return return_val;
752  }
753 };
754 
755 template <class ReturnType, class FunctorType>
756 struct ParallelReduceReturnValue<
757  typename std::enable_if<!Kokkos::is_view<ReturnType>::value &&
758  (!std::is_array<ReturnType>::value &&
759  !std::is_pointer<ReturnType>::value) &&
760  !Kokkos::is_reducer_type<ReturnType>::value>::type,
761  ReturnType, FunctorType> {
762  using return_type =
764 
765  using reducer_type = InvalidType;
766 
767  using value_type = typename return_type::value_type;
768 
769  static return_type return_value(ReturnType& return_val, const FunctorType&) {
770  return return_type(&return_val);
771  }
772 };
773 
774 template <class ReturnType, class FunctorType>
775 struct ParallelReduceReturnValue<
776  typename std::enable_if<(std::is_array<ReturnType>::value ||
777  std::is_pointer<ReturnType>::value)>::type,
778  ReturnType, FunctorType> {
780  Kokkos::HostSpace, Kokkos::MemoryUnmanaged>;
781 
782  using reducer_type = InvalidType;
783 
784  using value_type = typename return_type::value_type[];
785 
786  static return_type return_value(ReturnType& return_val,
787  const FunctorType& functor) {
788  if (std::is_array<ReturnType>::value)
789  return return_type(return_val);
790  else
791  return return_type(return_val, functor.value_count);
792  }
793 };
794 
795 template <class ReturnType, class FunctorType>
796 struct ParallelReduceReturnValue<
797  typename std::enable_if<Kokkos::is_reducer_type<ReturnType>::value>::type,
798  ReturnType, FunctorType> {
799  using return_type = ReturnType;
800  using reducer_type = ReturnType;
801  using value_type = typename return_type::value_type;
802 
803  static return_type return_value(ReturnType& return_val, const FunctorType&) {
804  return return_val;
805  }
806 };
807 
808 template <class T, class ReturnType, class FunctorType>
809 struct ParallelReducePolicyType;
810 
811 template <class PolicyType, class FunctorType>
812 struct ParallelReducePolicyType<
813  typename std::enable_if<
814  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type,
815  PolicyType, FunctorType> {
816  using policy_type = PolicyType;
817  static PolicyType policy(const PolicyType& policy_) { return policy_; }
818 };
819 
820 template <class PolicyType, class FunctorType>
821 struct ParallelReducePolicyType<
822  typename std::enable_if<std::is_integral<PolicyType>::value>::type,
823  PolicyType, FunctorType> {
824  using execution_space =
825  typename Impl::FunctorPolicyExecutionSpace<FunctorType,
826  void>::execution_space;
827 
828  using policy_type = Kokkos::RangePolicy<execution_space>;
829 
830  static policy_type policy(const PolicyType& policy_) {
831  return policy_type(0, policy_);
832  }
833 };
834 
835 template <class FunctorType, class ExecPolicy, class ValueType,
836  class ExecutionSpace>
837 struct ParallelReduceFunctorType {
838  using functor_type = FunctorType;
839  static const functor_type& functor(const functor_type& functor) {
840  return functor;
841  }
842 };
843 
844 template <class PolicyType, class FunctorType, class ReturnType>
845 struct ParallelReduceAdaptor {
846  using return_value_adapter =
847  Impl::ParallelReduceReturnValue<void, ReturnType, FunctorType>;
848 #ifdef KOKKOS_IMPL_NEED_FUNCTOR_WRAPPER
849  using functor_adaptor =
850  Impl::ParallelReduceFunctorType<FunctorType, PolicyType,
851  typename return_value_adapter::value_type,
852  typename PolicyType::execution_space>;
853 #endif
854  static inline void execute(const std::string& label, const PolicyType& policy,
855  const FunctorType& functor,
856  ReturnType& return_value) {
857  uint64_t kpID = 0;
858 
859  PolicyType inner_policy = policy;
860  Kokkos::Tools::Impl::begin_parallel_reduce<
861  typename return_value_adapter::reducer_type>(inner_policy, functor,
862  label, kpID);
863 
864  Kokkos::Impl::shared_allocation_tracking_disable();
865 #ifdef KOKKOS_IMPL_NEED_FUNCTOR_WRAPPER
866  Impl::ParallelReduce<typename functor_adaptor::functor_type, PolicyType,
867  typename return_value_adapter::reducer_type>
868  closure(functor_adaptor::functor(functor), inner_policy,
869  return_value_adapter::return_value(return_value, functor));
870 #else
871  Impl::ParallelReduce<FunctorType, PolicyType,
872  typename return_value_adapter::reducer_type>
873  closure(functor, inner_policy,
874  return_value_adapter::return_value(return_value, functor));
875 #endif
876  Kokkos::Impl::shared_allocation_tracking_enable();
877  closure.execute();
878 
879  Kokkos::Tools::Impl::end_parallel_reduce<
880  typename return_value_adapter::reducer_type>(inner_policy, functor,
881  label, kpID);
882  }
883 };
884 } // namespace Impl
885 
886 //----------------------------------------------------------------------------
887 
899 // Parallel Reduce Blocking behavior
900 
901 namespace Impl {
902 template <typename T>
903 struct ReducerHasTestReferenceFunction {
904  template <typename E>
905  static std::true_type test_func(decltype(&E::references_scalar));
906  template <typename E>
907  static std::false_type test_func(...);
908 
909  enum {
910  value = std::is_same<std::true_type, decltype(test_func<T>(nullptr))>::value
911  };
912 };
913 
914 template <class ExecutionSpace, class Arg>
915 constexpr std::enable_if_t<
916  // constraints only necessary because SFINAE lacks subsumption
917  !ReducerHasTestReferenceFunction<Arg>::value &&
918  !Kokkos::is_view<Arg>::value,
919  // return type:
920  bool>
921 parallel_reduce_needs_fence(ExecutionSpace const&, Arg const&) {
922  return true;
923 }
924 
925 template <class ExecutionSpace, class Reducer>
926 constexpr std::enable_if_t<
927  // equivalent to:
928  // (requires (Reducer const& r) {
929  // { reducer.references_scalar() } -> std::convertible_to<bool>;
930  // })
931  ReducerHasTestReferenceFunction<Reducer>::value,
932  // return type:
933  bool>
934 parallel_reduce_needs_fence(ExecutionSpace const&, Reducer const& reducer) {
935  return reducer.references_scalar();
936 }
937 
938 template <class ExecutionSpace, class ViewLike>
939 constexpr std::enable_if_t<
940  // requires Kokkos::ViewLike<ViewLike>
941  Kokkos::is_view<ViewLike>::value,
942  // return type:
943  bool>
944 parallel_reduce_needs_fence(ExecutionSpace const&, ViewLike const&) {
945  return false;
946 }
947 
948 template <class ExecutionSpace, class... Args>
949 struct ParallelReduceFence {
950  template <class... ArgsDeduced>
951  static void fence(const ExecutionSpace& ex, ArgsDeduced&&... args) {
952  if (Impl::parallel_reduce_needs_fence(ex, (ArgsDeduced &&) args...)) {
953  ex.fence();
954  }
955  }
956 };
957 
958 } // namespace Impl
959 
1000 // ReturnValue is scalar or array: take by reference
1001 
1002 template <class PolicyType, class FunctorType, class ReturnType>
1003 inline typename std::enable_if<
1004  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type
1005 parallel_reduce(const std::string& label, const PolicyType& policy,
1006  const FunctorType& functor, ReturnType& return_value) {
1007  Impl::ParallelReduceAdaptor<PolicyType, FunctorType, ReturnType>::execute(
1008  label, policy, functor, return_value);
1009  Impl::ParallelReduceFence<typename PolicyType::execution_space,
1010  ReturnType>::fence(policy.space(), return_value);
1011 }
1012 
1013 template <class PolicyType, class FunctorType, class ReturnType>
1014 inline typename std::enable_if<
1015  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type
1016 parallel_reduce(const PolicyType& policy, const FunctorType& functor,
1017  ReturnType& return_value) {
1018  Impl::ParallelReduceAdaptor<PolicyType, FunctorType, ReturnType>::execute(
1019  "", policy, functor, return_value);
1020  Impl::ParallelReduceFence<typename PolicyType::execution_space,
1021  ReturnType>::fence(policy.space(), return_value);
1022 }
1023 
1024 template <class FunctorType, class ReturnType>
1025 inline void parallel_reduce(const size_t& policy, const FunctorType& functor,
1026  ReturnType& return_value) {
1027  using policy_type =
1028  typename Impl::ParallelReducePolicyType<void, size_t,
1029  FunctorType>::policy_type;
1030  Impl::ParallelReduceAdaptor<policy_type, FunctorType, ReturnType>::execute(
1031  "", policy_type(0, policy), functor, return_value);
1032  Impl::ParallelReduceFence<typename policy_type::execution_space, ReturnType>::
1033  fence(typename policy_type::execution_space(), return_value);
1034 }
1035 
1036 template <class FunctorType, class ReturnType>
1037 inline void parallel_reduce(const std::string& label, const size_t& policy,
1038  const FunctorType& functor,
1039  ReturnType& return_value) {
1040  using policy_type =
1041  typename Impl::ParallelReducePolicyType<void, size_t,
1042  FunctorType>::policy_type;
1043  Impl::ParallelReduceAdaptor<policy_type, FunctorType, ReturnType>::execute(
1044  label, policy_type(0, policy), functor, return_value);
1045  Impl::ParallelReduceFence<typename policy_type::execution_space, ReturnType>::
1046  fence(typename policy_type::execution_space(), return_value);
1047 }
1048 
1049 // ReturnValue as View or Reducer: take by copy to allow for inline construction
1050 
1051 template <class PolicyType, class FunctorType, class ReturnType>
1052 inline typename std::enable_if<
1053  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type
1054 parallel_reduce(const std::string& label, const PolicyType& policy,
1055  const FunctorType& functor, const ReturnType& return_value) {
1056  ReturnType return_value_impl = return_value;
1057  Impl::ParallelReduceAdaptor<PolicyType, FunctorType, ReturnType>::execute(
1058  label, policy, functor, return_value_impl);
1059  Impl::ParallelReduceFence<typename PolicyType::execution_space,
1060  ReturnType>::fence(policy.space(), return_value);
1061 }
1062 
1063 template <class PolicyType, class FunctorType, class ReturnType>
1064 inline typename std::enable_if<
1065  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type
1066 parallel_reduce(const PolicyType& policy, const FunctorType& functor,
1067  const ReturnType& return_value) {
1068  ReturnType return_value_impl = return_value;
1069  Impl::ParallelReduceAdaptor<PolicyType, FunctorType, ReturnType>::execute(
1070  "", policy, functor, return_value_impl);
1071  Impl::ParallelReduceFence<typename PolicyType::execution_space,
1072  ReturnType>::fence(policy.space(), return_value);
1073 }
1074 
1075 template <class FunctorType, class ReturnType>
1076 inline void parallel_reduce(const size_t& policy, const FunctorType& functor,
1077  const ReturnType& return_value) {
1078  using policy_type =
1079  typename Impl::ParallelReducePolicyType<void, size_t,
1080  FunctorType>::policy_type;
1081  ReturnType return_value_impl = return_value;
1082  Impl::ParallelReduceAdaptor<policy_type, FunctorType, ReturnType>::execute(
1083  "", policy_type(0, policy), functor, return_value_impl);
1084  Impl::ParallelReduceFence<typename policy_type::execution_space, ReturnType>::
1085  fence(typename policy_type::execution_space(), return_value);
1086 }
1087 
1088 template <class FunctorType, class ReturnType>
1089 inline void parallel_reduce(const std::string& label, const size_t& policy,
1090  const FunctorType& functor,
1091  const ReturnType& return_value) {
1092  using policy_type =
1093  typename Impl::ParallelReducePolicyType<void, size_t,
1094  FunctorType>::policy_type;
1095  ReturnType return_value_impl = return_value;
1096  Impl::ParallelReduceAdaptor<policy_type, FunctorType, ReturnType>::execute(
1097  label, policy_type(0, policy), functor, return_value_impl);
1098  Impl::ParallelReduceFence<typename policy_type::execution_space, ReturnType>::
1099  fence(typename policy_type::execution_space(), return_value);
1100 }
1101 
1102 // No Return Argument
1103 
1104 template <class PolicyType, class FunctorType>
1105 inline void parallel_reduce(
1106  const std::string& label, const PolicyType& policy,
1107  const FunctorType& functor,
1108  typename std::enable_if<
1109  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type* =
1110  nullptr) {
1111  using ValueTraits = Kokkos::Impl::FunctorValueTraits<FunctorType, void>;
1112  using value_type = std::conditional_t<(ValueTraits::StaticValueSize != 0),
1113  typename ValueTraits::value_type,
1114  typename ValueTraits::pointer_type>;
1115 
1116  static_assert(
1117  Impl::FunctorAnalysis<Impl::FunctorPatternInterface::REDUCE, PolicyType,
1118  FunctorType>::has_final_member_function,
1119  "Calling parallel_reduce without either return value or final function.");
1120 
1121  using result_view_type =
1123  result_view_type result_view;
1124 
1125  Impl::ParallelReduceAdaptor<PolicyType, FunctorType,
1126  result_view_type>::execute(label, policy, functor,
1127  result_view);
1128 }
1129 
1130 template <class PolicyType, class FunctorType>
1131 inline void parallel_reduce(
1132  const PolicyType& policy, const FunctorType& functor,
1133  typename std::enable_if<
1134  Kokkos::Impl::is_execution_policy<PolicyType>::value>::type* =
1135  nullptr) {
1136  using ValueTraits = Kokkos::Impl::FunctorValueTraits<FunctorType, void>;
1137  using value_type = std::conditional_t<(ValueTraits::StaticValueSize != 0),
1138  typename ValueTraits::value_type,
1139  typename ValueTraits::pointer_type>;
1140 
1141  static_assert(
1142  Impl::FunctorAnalysis<Impl::FunctorPatternInterface::REDUCE, PolicyType,
1143  FunctorType>::has_final_member_function,
1144  "Calling parallel_reduce without either return value or final function.");
1145 
1146  using result_view_type =
1148  result_view_type result_view;
1149 
1150  Impl::ParallelReduceAdaptor<PolicyType, FunctorType,
1151  result_view_type>::execute("", policy, functor,
1152  result_view);
1153 }
1154 
1155 template <class FunctorType>
1156 inline void parallel_reduce(const size_t& policy, const FunctorType& functor) {
1157  using policy_type =
1158  typename Impl::ParallelReducePolicyType<void, size_t,
1159  FunctorType>::policy_type;
1160  using ValueTraits = Kokkos::Impl::FunctorValueTraits<FunctorType, void>;
1161  using value_type = std::conditional_t<(ValueTraits::StaticValueSize != 0),
1162  typename ValueTraits::value_type,
1163  typename ValueTraits::pointer_type>;
1164 
1165  static_assert(
1166  Impl::FunctorAnalysis<Impl::FunctorPatternInterface::REDUCE,
1167  RangePolicy<>,
1168  FunctorType>::has_final_member_function,
1169  "Calling parallel_reduce without either return value or final function.");
1170 
1171  using result_view_type =
1173  result_view_type result_view;
1174 
1175  Impl::ParallelReduceAdaptor<policy_type, FunctorType,
1176  result_view_type>::execute("",
1177  policy_type(0, policy),
1178  functor, result_view);
1179 }
1180 
1181 template <class FunctorType>
1182 inline void parallel_reduce(const std::string& label, const size_t& policy,
1183  const FunctorType& functor) {
1184  using policy_type =
1185  typename Impl::ParallelReducePolicyType<void, size_t,
1186  FunctorType>::policy_type;
1187  using ValueTraits = Kokkos::Impl::FunctorValueTraits<FunctorType, void>;
1188  using value_type = std::conditional_t<(ValueTraits::StaticValueSize != 0),
1189  typename ValueTraits::value_type,
1190  typename ValueTraits::pointer_type>;
1191 
1192  static_assert(
1193  Impl::FunctorAnalysis<Impl::FunctorPatternInterface::REDUCE,
1194  RangePolicy<>,
1195  FunctorType>::has_final_member_function,
1196  "Calling parallel_reduce without either return value or final function.");
1197 
1198  using result_view_type =
1200  result_view_type result_view;
1201 
1202  Impl::ParallelReduceAdaptor<policy_type, FunctorType,
1203  result_view_type>::execute(label,
1204  policy_type(0, policy),
1205  functor, result_view);
1206 }
1207 
1208 } // namespace Kokkos
1209 
1210 #endif // KOKKOS_PARALLEL_REDUCE_HPP
Memory management for host memory.
ReturnType
Execution policy for work over a range of an integral type.
Definition: dummy.cpp:3