FM / ATen

/usr/local/lib64/python3.6/site-packages/torch/include/ATen/

/usr/local/lib64/python3.6/site-packages/torch/include/ATen

Name	Size	Mode	Actions
core/	-	0755	rm
cpu/	-	0755	rm
cuda/	-	0755	rm
cudnn/	-	0755	rm
detail/	-	0755	rm
hip/	-	0755	rm
native/	-	0755	rm
quantized/	-	0755	rm
AccumulateType.h	4438	0644	edit dl rm
ArrayRef.h	44	0644	edit dl rm
ATen.h	998	0644	edit dl rm
autocast_mode.h	6716	0644	edit dl rm
Backend.h	43	0644	edit dl rm
Backtrace.h	46	0644	edit dl rm
BatchedFallback.h	965	0644	edit dl rm
BatchedTensorImpl.h	5383	0644	edit dl rm
CompositeExplicitAutogradFunctions.h	1622	0644	edit dl rm
CompositeExplicitAutogradFunctions_inl.h	54075	0644	edit dl rm
CompositeImplicitAutogradFunctions.h	1622	0644	edit dl rm
CompositeImplicitAutogradFunctions_inl.h	142082	0644	edit dl rm
Config.h	734	0644	edit dl rm
Context.h	12767	0644	edit dl rm
cpp_custom_type_hack.h	5326	0644	edit dl rm
CPUApplyUtils.h	12582	0644	edit dl rm
CPUFixedAllocator.h	830	0644	edit dl rm
CPUFunctions.h	1600	0644	edit dl rm
CPUFunctions_inl.h	171924	0644	edit dl rm
CPUGeneratorImpl.h	1431	0644	edit dl rm
CUDAFunctions.h	1601	0644	edit dl rm
CUDAFunctions_inl.h	185696	0644	edit dl rm
CUDAGeneratorImpl.h	4695	0644	edit dl rm
Device.h	42	0644	edit dl rm
DeviceGuard.h	1134	0644	edit dl rm
Dimname.h	31	0644	edit dl rm
DimVector.h	46	0644	edit dl rm
Dispatch.h	52137	0644	edit dl rm
div_rtn.h	204	0644	edit dl rm
DLConvertor.h	576	0644	edit dl rm
dlpack.h	5244	0644	edit dl rm
DynamicLibrary.h	369	0644	edit dl rm
ExpandUtils.h	14506	0644	edit dl rm
Formatting.h	34	0644	edit dl rm
Functions.h	846326	0644	edit dl rm
Generator.h	46	0644	edit dl rm
InferSize.h	2143	0644	edit dl rm
InitialTensorOptions.h	445	0644	edit dl rm
Layout.h	42	0644	edit dl rm
MapAllocator.h	2999	0644	edit dl rm
MatrixRef.h	3016	0644	edit dl rm
MemoryOverlap.h	1117	0644	edit dl rm
MetaFunctions.h	1601	0644	edit dl rm
MetaFunctions_inl.h	84006	0644	edit dl rm
NamedTensor.h	35	0644	edit dl rm
NamedTensorUtils.h	5747	0644	edit dl rm
NativeFunctions.h	354651	0644	edit dl rm
NativeMetaFunctions.h	35445	0644	edit dl rm
NumericUtils.h	2787	0644	edit dl rm
OpaqueTensorImpl.h	6080	0644	edit dl rm
Operators.h	1707199	0644	edit dl rm
OpMathType.h	460	0644	edit dl rm
Parallel.h	4875	0644	edit dl rm
ParallelNative.h	2443	0644	edit dl rm
ParallelNativeTBB.h	2934	0644	edit dl rm
ParallelOpenMP.h	3049	0644	edit dl rm
PTThreadPool.h	394	0644	edit dl rm
record_function.h	24044	0644	edit dl rm
RedispatchFunctions.h	1112886	0644	edit dl rm
RegistrationDeclarations.h	545777	0644	edit dl rm
SavedTensorHooks.h	328	0644	edit dl rm
Scalar.h	44	0644	edit dl rm
ScalarOps.h	2272	0644	edit dl rm
ScalarType.h	129	0644	edit dl rm
SequenceNumber.h	373	0644	edit dl rm
SmallVector.h	47	0644	edit dl rm
SparseCsrTensorImpl.h	2045	0644	edit dl rm
SparseCsrTensorUtils.h	523	0644	edit dl rm
SparseTensorImpl.h	12417	0644	edit dl rm
SparseTensorUtils.h	4219	0644	edit dl rm
Storage.h	43	0644	edit dl rm
Tensor.h	48	0644	edit dl rm
TensorAccessor.h	51	0644	edit dl rm
TensorGeometry.h	1855	0644	edit dl rm
TensorIndexing.h	21923	0644	edit dl rm
TensorIterator.h	29962	0644	edit dl rm
TensorIteratorInternal.h	1862	0644	edit dl rm
TensorMeta.h	2917	0644	edit dl rm
TensorNames.h	2519	0644	edit dl rm
TensorOperators.h	3275	0644	edit dl rm
TensorOptions.h	49	0644	edit dl rm
TensorUtils.h	5687	0644	edit dl rm
ThreadLocalState.h	3289	0644	edit dl rm
TracerMode.h	5576	0644	edit dl rm
TypeDefault.h	680	0644	edit dl rm
Utils.h	5993	0644	edit dl rm
Version.h	340	0644	edit dl rm
VmapMode.h	952	0644	edit dl rm
VmapTransforms.h	7654	0644	edit dl rm
WrapDimUtils.h	3438	0644	edit dl rm
WrapDimUtilsMulti.h	768	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/ATen/Parallel.h (4875B)

#pragma once
#include <ATen/Config.h>
#include <ATen/core/ivalue.h>
#include <c10/macros/Macros.h>

namespace at {

inline int64_t divup(int64_t x, int64_t y) {
  return (x + y - 1) / y;
}

// Called during new thread initialization
TORCH_API void init_num_threads();

// Sets the number of threads to be used in parallel region
TORCH_API void set_num_threads(int);

// Returns the maximum number of threads that may be used in a parallel region
TORCH_API int get_num_threads();

// Returns the current thread number (starting from 0)
// in the current parallel region, or 0 in the sequential region
TORCH_API int get_thread_num();

// Checks whether the code runs in parallel region
TORCH_API bool in_parallel_region();

namespace internal {

// Initialise num_threads lazily at first parallel call
inline TORCH_API void lazy_init_num_threads() {
  thread_local bool init = false;
  if (C10_UNLIKELY(!init)) {
    at::init_num_threads();
    init = true;
  }
}

TORCH_API void set_thread_num(int);

class TORCH_API ThreadIdGuard {
public:
  ThreadIdGuard(int new_id):
    old_id_(at::get_thread_num()) {
    set_thread_num(new_id);
  }

~ThreadIdGuard() {
    set_thread_num(old_id_);
  }

private:
  int old_id_;
};

}  // namespace internal

/*
parallel_for

begin: index at which to start applying user function

end: index at which to stop applying user function

grain_size: number of elements per chunk. impacts the degree of parallelization

f: user function applied in parallel to the chunks, signature:
  void f(int64_t begin, int64_t end)

Warning: parallel_for does NOT copy thread local
states from the current thread to the worker threads.
This means for example that Tensor operations CANNOT be used in the
body of your function, only data pointers.
*/
template <class F>
inline void parallel_for(
    const int64_t begin,
    const int64_t end,
    const int64_t grain_size,
    const F& f);

/*
parallel_reduce

begin: index at which to start applying reduction

end: index at which to stop applying reduction

grain_size: number of elements per chunk. impacts number of elements in
intermediate results tensor and degree of parallelization.

ident: identity for binary combination function sf. sf(ident, x) needs to return
x.

f: function for reduction over a chunk. f needs to be of signature scalar_t
f(int64_t partial_begin, int64_t partial_end, scalar_t identifiy)

sf: function to combine two partial results. sf needs to be of signature
scalar_t sf(scalar_t x, scalar_t y)

For example, you might have a tensor of 10000 entires and want to sum together
all the elements. Parallel_reduce with a grain_size of 2500 will then allocate
an intermediate result tensor with 4 elements. Then it will execute the function
"f" you provide and pass the beginning and end index of these chunks, so
0-2499, 2500-4999, etc. and the combination identity. It will then write out
the result from each of these chunks into the intermediate result tensor. After
that it'll reduce the partial results from each chunk into a single number using
the combination function sf and the identity ident. For a total summation this
would be "+" and 0 respectively. This is similar to tbb's approach [1], where
you need to provide a function to accumulate a subrange, a function to combine
two partial results and an identity.

Warning: parallel_reduce does NOT copy thread local
states from the current thread to the worker threads.
This means for example that Tensor operations CANNOT be used in the
body of your function, only data pointers.

[1] https://software.intel.com/en-us/node/506154
*/
template <class scalar_t, class F, class SF>
inline scalar_t parallel_reduce(
    const int64_t begin,
    const int64_t end,
    const int64_t grain_size,
    const scalar_t ident,
    const F& f,
    const SF& sf);

// Returns a detailed string describing parallelization settings
TORCH_API std::string get_parallel_info();

// Sets number of threads used for inter-op parallelism
TORCH_API void set_num_interop_threads(int);

// Returns the number of threads used for inter-op parallelism
TORCH_API int get_num_interop_threads();

// Launches inter-op parallel task
TORCH_API void launch(std::function<void()> func);
namespace internal {
void launch_no_thread_state(std::function<void()> fn);
} // namespace internal

// Launches intra-op parallel task
TORCH_API void intraop_launch(std::function<void()> func);

// Launches intra-op parallel task, returns a future
TORCH_API c10::intrusive_ptr<c10::ivalue::Future> intraop_launch_future(
    std::function<void()> func);

// Returns number of intra-op threads used by default
TORCH_API int intraop_default_num_threads();

} // namespace at

#if AT_PARALLEL_OPENMP
#include <ATen/ParallelOpenMP.h>
#elif AT_PARALLEL_NATIVE
#include <ATen/ParallelNative.h>
#elif AT_PARALLEL_NATIVE_TBB
#include <ATen/ParallelNativeTBB.h>
#endif

cmd: