AT_DISPATCH_FLOATING_TYPES(input.scalar_type(), "log_sigmoid_cpu", [&] {
  using Vec = Vectorized<scalar_t>;
  scalar_t* output_data = output.data_ptr<scalar_t>();
  scalar_t* buffer_data = buffer.data_ptr<scalar_t>();
  const scalar_t* input_data = input.const_data_ptr<scalar_t>();
  parallel_for(0, input.numel(), 1, [&] (int64_t begin, int64_t end) {
    // ...
    Vec min_vec = vec::minimum(data_vec, Vec(scalar_t(0)));
    // ...
  });
});

template <typename scalar_t>
void add_kernel(const scalar_t* a, const scalar_t* b, scalar_t* out, int64_t n) {
  for (int64_t i = 0; i < n; ++i) out[i] = a[i] + b[i];
}

  // Implemented in aten/src/ATen/templates/TensorMethods.cpp
  template <typename T>
  const T* const_data_ptr() const;

  template <typename T>
  T* mutable_data_ptr() const;
  // ...
  template <typename T>
  T* data_ptr() const;

template <typename T>
T* TensorBase::mutable_data_ptr() const {
  check_type(*this, c10::CppTypeToScalarType<T>());
  return this->unsafeGetTensorImpl()->mutable_data_ptr_impl<T>();
}

template <typename T>
T* TensorBase::data_ptr() const {
  return this->mutable_data_ptr<T>();
}

#define DEFINE_CAST(T, name)                                                \
   template TORCH_API const T* TensorBase::const_data_ptr<T>() const;       \
   template TORCH_API const T* TensorBase::const_data_ptr<const T>() const; \
   template TORCH_API T* TensorBase::mutable_data_ptr() const;              \
   template TORCH_API T* TensorBase::data_ptr() const;

 AT_FORALL_SCALAR_TYPES_WITH_COMPLEX(DEFINE_CAST)
 AT_FORALL_QINT_TYPES(DEFINE_CAST)
 DEFINE_CAST(uint16_t, UInt16)
 DEFINE_CAST(uint32_t, UInt32)
 DEFINE_CAST(uint64_t, UInt64)
 #undef DEFINE_CAST

#include <algorithm>
#include <list>
int main() {
  std::list<int> l{3, 1, 2};
  std::sort(l.begin(), l.end());
}

  template <
      typename ItTy,
      typename = std::enable_if_t<std::is_convertible_v<
          typename std::iterator_traits<ItTy>::iterator_category,
          std::input_iterator_tag>>>
  SmallVector(ItTy S, ItTy E) : SmallVectorImpl<T>(N) {

template <typename scalar_t, int width>
struct alignas(16) _f16Vec {
  // ...
  using Converter = _typeConvert<scalar_t>;
  using T1 = typename Converter::hip_type;
  using T2 = typename Converter::packed_hip_type;
  T1 data[width];

template <c10::ScalarType N>
using ScalarTypeToCPPTypeT = typename ScalarTypeToCPPType<N>::type;

  intrusive_ptr& operator=(intrusive_ptr&& rhs) & noexcept {
    return this->template operator= <TTarget, NullType>(std::move(rhs));
  }

template <class F>
inline void parallel_for(
    const int64_t begin,
    const int64_t end,
    const int64_t grain_size,
    const F& f);

template <typename func_t>
void cpu_kernel(TensorIteratorBase& iter, func_t&& op, int64_t grain_size = at::internal::GRAIN_SIZE, bool check_dynamic_casting = true) {
  using traits = function_traits<func_t>;
  // this could be extended to work with void return types
  TORCH_INTERNAL_ASSERT(iter.ninputs() == traits::arity);
  TORCH_INTERNAL_ASSERT(iter.noutputs() == 1);
  // ...

template <class T, class = std::ostream&>
struct Streamable : std::false_type {};

template <class T>
struct Streamable<T, decltype(std::declval<std::ostream&>() << T{})>
    : std::true_type {};

// Generic container constructor (anything with .data() and .size())
template <typename Container>
ArrayRef(const Container&) -> ArrayRef<
    std::remove_pointer_t<decltype(std::declval<Container>().data())>>;

template <typename... Args>
inline auto str(const Args&... args) {
  return detail::_str_wrapper<
      typename detail::CanonicalizeStrTypes<Args>::type...>::call(args...);
}

template <typename T>
auto TensorBase::register_hook(T&& hook) const -> TensorBase::hook_return_void_t<T> {
  // Return the grad argument in case of a hook with void return type to have an
  // std::function with Tensor return type
  static_assert(std::is_same_v<decltype(hook(TensorBase())), void>,
                "Expected hook to return void");
  return _register_hook([fn=std::forward<T>(hook)](const TensorBase& grad) {
    fn(grad);
    return TensorBase();
  });
}

/// Deduction guides for ArrayRef to support CTAD with inherited constructors
/// These mirror the constructors inherited from HeaderOnlyArrayRef
// Single element constructor
template <typename T>
ArrayRef(const T&) -> ArrayRef<T>;

// Pointer and length constructor
template <typename T>
ArrayRef(const T*, size_t) -> ArrayRef<T>;
// ...
// std::vector constructor
template <typename T, typename A>
ArrayRef(const std::vector<T, A>&) -> ArrayRef<T>;

/// Storage for the SmallVector elements.  This is specialized for the N=0 case
/// to avoid allocating unnecessary storage.
template <typename T, unsigned N>
struct SmallVectorStorage {
  alignas(T) char InlineElts[N * sizeof(T)];
};
// ...
template <
    typename T,
    unsigned N = CalculateSmallVectorDefaultInlinedElements<T>::value>
class /* LLVM_GSL_OWNER */ SmallVector : public SmallVectorImpl<T>,
                                         SmallVectorStorage<T, N> {

constexpr size_t kDimVectorStaticSize = C10_SIZES_AND_STRIDES_MAX_INLINE_SIZE;

/// A container for sizes or strides
using DimVector = SmallVector<int64_t, kDimVectorStaticSize>;

  /// Construct a HeaderOnlyArrayRef from a C array.
  template <size_t N>
  /* implicit */ constexpr HeaderOnlyArrayRef(const T (&Arr)[N])
      : Data(Arr), Length(N) {}

namespace impl {

// These are used to map ScalarTypes to C++ types.

template <c10::ScalarType N>
struct ScalarTypeToCPPType;

#define SPECIALIZE_ScalarTypeToCPPType(cpp_type, scalar_type)                \
  template <>                                                                \
  struct ScalarTypeToCPPType<c10::ScalarType::scalar_type> {                 \
    using type = cpp_type;                                                   \
    /* ... */                                                                \
    static type t;                                                           \
  };

AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(SPECIALIZE_ScalarTypeToCPPType)

#undef SPECIALIZE_ScalarTypeToCPPType

template <c10::ScalarType N>
using ScalarTypeToCPPTypeT = typename ScalarTypeToCPPType<N>::type;

} // namespace impl

// Map from C++ type to ScalarType enum
template <typename T>
struct CppTypeToScalarType;

#define SPECIALIZE_CppTypeToScalarType(cpp_type, scalar_type)                  \
  template <>                                                                  \
  struct CppTypeToScalarType<cpp_type>                                         \
      : std::                                                                  \
            integral_constant<c10::ScalarType, c10::ScalarType::scalar_type> { \
  };

AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(SPECIALIZE_CppTypeToScalarType)

template <int BLOCK_Y_SIZE>
__global__ void cp_gather_indexer_k_quant_cache_kernel(
    const char* __restrict__ kv_cache,  // [num_blocks, block_size,
                                        // cache_stride]
    char* __restrict__ dst_k,           // [num_tokens, head_dim]
    // ...

// Macro to dispatch the kernel based on the data amount.
#define CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(BLOCK_Y_SIZE)                    \
  vllm::cp_gather_indexer_k_quant_cache_kernel<BLOCK_Y_SIZE>                  \
      <<<dim3((num_tokens + BLOCK_Y_SIZE - 1) / BLOCK_Y_SIZE,                 \
              (head_dim + 8 * vec_size - 1) / (8 * vec_size)),                \
         dim3(8, BLOCK_Y_SIZE), 0, stream>>>(                                 \
          reinterpret_cast<char*>(kv_cache.data_ptr()),                       \
          /* ... */                                                           \
          quant_block_size);
// ...
  if (num_tokens < 32) {
    CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(1);
  } else if (num_tokens < 64) {
    CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(2);
  } else if (num_tokens < 128) {
    CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(4);
  } else if (num_tokens < 256) {
    CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(8);
  } else if (num_tokens < 512) {
    CALL_CP_GATHER_INDEXER_K_QUANT_CACHE(16);
  // ...

template <typename scalar_t, int width, bool HasWeight>
__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
fused_add_rms_norm_kernel(
    scalar_t* __restrict__ input,  // [..., hidden_size]
    const int64_t input_stride,
    scalar_t* __restrict__ residual,      // [..., hidden_size]
    const scalar_t* __restrict__ weight,  // [hidden_size], null if !HasWeight
    const float epsilon, const int num_tokens, const int hidden_size,
    const int64_t residual_stride) {
  // Sanity checks on our vector struct and type-punned pointer arithmetic
  static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
  static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
  // ...

#define LAUNCH_FUSED_ADD_RMS_NORM(width, has_weight)                       \
  VLLM_STABLE_DISPATCH_FLOATING_TYPES(                                     \
      input.scalar_type(), "fused_add_rms_norm_kernel", [&] {              \
        if (has_weight) {                                                  \
          vllm::fused_add_rms_norm_kernel<scalar_t, width, true>           \
              <<<grid, block, 0, stream>>>(                                \
                  input.mutable_data_ptr<scalar_t>(), input_stride,        \
                  residual.mutable_data_ptr<scalar_t>(),                   \
                  weight->const_data_ptr<scalar_t>(), epsilon, num_tokens, \
                  hidden_size, residual_stride);                           \
        } else {                                                           \
          vllm::fused_add_rms_norm_kernel<scalar_t, width, false>          \
              <<<grid, block, 0, stream>>>(/* ... */);                     \
        }                                                                  \
      });
// ...
    if (ptrs_are_aligned && offsets_are_multiple_of_vector_width &&
        !batch_invariant_launch) {
      LAUNCH_FUSED_ADD_RMS_NORM(8, true);
    } else {
      LAUNCH_FUSED_ADD_RMS_NORM(0, true);
    }

template <>                                    // 空的模板参数列表：这是特化
struct ScalarTypeToCPPType<c10::ScalarType::Float> {
  using type = float;
};

template <typename torch_type>
struct _typeConvert {
  static constexpr bool exists = false;
};

template <>
struct _typeConvert<float> {
  static constexpr bool exists = true;
  using hip_type = float;
  using packed_hip_type = float2;
  using packed_hip_type4 = float4;  // For 128-bit vectorization

  __device__ static __forceinline__ float convert(hip_type x) { return x; }
  // ...
};

 #define DEFINE_ITEM(T, name)      \
   template <>                     \
   TORCH_API T Tensor::item() const { \
     return item().to##name();     \
   }

 AT_FORALL_SCALAR_TYPES_WITH_COMPLEX(DEFINE_ITEM)

template <typename... Args>
struct _str_wrapper final {
  static std::string call(const Args&... args) {
    std::ostringstream ss;
    _str(ss, args...);
    return ss.str();
  }
};

// Specializations for already-a-string types.
template <>
struct _str_wrapper<std::string> final {
  // return by reference to avoid the binary size of a string copy
  static const std::string& call(const std::string& str) {
    return str;
  }
};
template <>
struct _str_wrapper<const char*> final {
  static const char* call(const char* str) {
    return str;
  }
};

template <
    typename T,
    bool = (std::is_trivially_copy_constructible_v<T>) &&
        (std::is_trivially_move_constructible_v<T>) &&
        std::is_trivially_destructible_v<T>>
class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> {
  // ...
  static void destroy_range(T* S, T* E) {
    while (S != E) {
      --E;
      E->~T();
    }
  }
// ...
/// SmallVectorTemplateBase<TriviallyCopyable = true> - This is where we put
/// method implementations that are designed to work with trivially copyable
/// T's. This allows using memcpy in place of copy/move construction and
/// skipping destruction.
template <typename T>
class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> {

/// We need the storage to be properly aligned even for small-size of 0 so that
/// the pointer math in \a SmallVectorTemplateCommon::getFirstEl() is
/// well-defined.
template <typename T>
struct alignas(T) SmallVectorStorage<T, 0> {};

template <typename T>
struct CanonicalizeStrTypes {
  using type = const T&;
};

template <size_t N>
// NOLINTNEXTLINE(*c-arrays*)
struct CanonicalizeStrTypes<char[N]> {
  using type = const char*;
};

template <typename T>
inline std::ostream& _str(std::ostream& ss, const T& t) {
  if constexpr (std::is_enum_v<T> && !Streamable<T>::value) {
    return _str(ss, static_cast<typename std::underlying_type<T>::type>(t));
  } else {
    ss << t;
    return ss;
  }
}

template <typename T>
inline std::ostream& _str(std::ostream& ss, const std::optional<T>& t) {
  if (t.has_value()) {
    return _str(ss, t.value());
  }
  ss << "std::nullopt";
  return ss;
}

template <class... Args>          // Args 是一个"类型参数包"：零个或多个类型
static intrusive_ptr make(Args&&... args) {   // args 是"函数参数包"
  return intrusive_ptr(new TTarget(std::forward<Args>(args)...));  // 展开
}

inline std::ostream& _str(std::ostream& ss) {
  return ss;
}
// ... 上面 4.3 节的单参数版本 ...
template <typename T, typename... Args>
inline std::ostream& _str(std::ostream& ss, const T& t, const Args&... args) {
  return _str(_str(ss, t), args...);
}

namespace c10::detail {
template <typename... Args>
auto torchCheckMsgImpl(const char* /*msg*/, const Args&... args) {
  return ::c10::str(args...);
}
inline C10_API const char* torchCheckMsgImpl(const char* msg) {
  return msg;
}
// If there is just 1 user-provided C-string argument, use it.
inline C10_API const char* torchCheckMsgImpl(
    const char* /*msg*/,
    const char* args) {
  return args;
}
} // namespace c10::detail

constexpr uint16_t NumScalarTypes =
    static_cast<uint16_t>(ScalarType::NumOptions);

inline constexpr bool should_include_kernel_dtype(
    const char* /*kernel_tag_str*/,
    at::ScalarType /*scalar_type*/
) {
  return true;
}

  template <typename T>
  ArrayRef<T> generic_sizes() {
    static_assert(
        std::is_same_v<T, int64_t> || std::is_same_v<T, c10::SymInt>,
        "Only supports int64_t and c10::SymInt.");

    if constexpr (std::is_same_v<T, int64_t>) {
      return sizes();
    } else {
      return sym_sizes();
    }
  }

  __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
    if constexpr (width % 2 == 0) {
#pragma unroll
      for (int i = 0; i < width; i += 2) {
        if constexpr (std::is_same_v<T2, float2>) {
          data[i] += other.data[i];
          data[i + 1] += other.data[i + 1];
        } else {
          T2 temp{data[i], data[i + 1]};
          temp += T2{other.data[i], other.data[i + 1]};
          data[i] = temp.x;
          // ...

#define AT_PRIVATE_CHECK_SELECTIVE_BUILD(enum_type)   \
  do {                                                \
    if constexpr (!at::should_include_kernel_dtype(   \
                      at_dispatch_name, enum_type)) { \
      TORCH_CHECK(                                    \
          false,                                      \
          "dtype '",                                  \
          toString(enum_type),                        \
          "' not selected for kernel tag ",           \
          at_dispatch_name);                          \
    }                                                 \
  } while (0)

  template <typename A>
  /* implicit */ HeaderOnlyArrayRef(const std::vector<T, A>& Vec)
      : Data(Vec.data()), Length(Vec.size()) {
    static_assert(
        !std::is_same_v<T, bool>,
        "HeaderOnlyArrayRef<bool> cannot be constructed from a std::vector<bool> bitfield.");
  }

  template <
      typename Container,
      typename U = decltype(std::declval<Container>().data()),
      // NOLINTNEXTLINE(modernize-use-constraints)
      typename = std::enable_if_t<
          (std::is_same_v<U, T*> || std::is_same_v<U, T const*>)>>
  /* implicit */ HeaderOnlyArrayRef(const Container& container)
      : Data(container.data()), Length(container.size()) {}

template <typename scalar_t, int width, bool HasWeight>
__global__ std::enable_if_t<(width > 0) && _typeConvert<scalar_t>::exists>
fused_add_rms_norm_kernel(/* ... */)
// ...
template <typename scalar_t, int width, bool HasWeight>
__global__ std::enable_if_t<(width == 0) || !_typeConvert<scalar_t>::exists>
fused_add_rms_norm_kernel(/* ... */)

template <typename func_t,
    std::enable_if_t<!std::is_void_v<typename function_traits<func_t>::result_type>>* = nullptr>
inline void
execute_op(char* C10_RESTRICT data[], const int64_t* strides, int64_t i, int64_t n, func_t&& op) {
// ...
template <typename func_t,
    std::enable_if_t<std::is_void_v<typename function_traits<func_t>::result_type>>* = nullptr>
inline void
execute_op(char* C10_RESTRICT data[], const int64_t* strides, int64_t i, int64_t n, func_t&& op) {

template <typename Container>
  requires std::is_same_v<decltype(std::declval<Container>().data()), const T*>
HeaderOnlyArrayRef(const Container& container);

template <typename C, typename T>
concept ContiguousContainerOf = requires(const C& c) {
  { c.data() } -> std::convertible_to<const T*>;
  { c.size() } -> std::convertible_to<size_t>;
};

switch (x.scalar_type()) {
  case ScalarType::Float:  kernel<float>(...);  break;
  case ScalarType::Double: kernel<double>(...); break;
  default: TORCH_CHECK(false, "not implemented for ", toString(x.scalar_type()));
}

#define AT_DISPATCH_CASE_FLOATING_TYPES(...)            \
  AT_DISPATCH_CASE(at::ScalarType::Double, __VA_ARGS__) \
  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__)

#define AT_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
  AT_DISPATCH_SWITCH(TYPE, NAME, AT_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))

#define AT_DISPATCH_SWITCH(TYPE, NAME, ...) \
  THO_DISPATCH_SWITCH_TMPL(                 \
      RECORD_KERNEL_FUNCTION_DTYPE,         \
      TORCH_CHECK_NOT_IMPLEMENTED,          \
      TYPE,                                 \
      NAME,                                 \
      __VA_ARGS__)

#define THO_DISPATCH_SWITCH_TMPL(                                           \
    PRELUDE, CHECK_NOT_IMPLEMENTED, TYPE, NAME, ...)                        \
  [&] {                                                                     \
    const auto& the_type = TYPE;                                            \
    constexpr const char* at_dispatch_name = NAME;                          \
    /* don't use TYPE again in case it is an expensive or side-effect op */ \
    torch::headeronly::ScalarType _st = ::detail::scalar_type(the_type);    \
    PRELUDE(at_dispatch_name, _st);                                         \
    C10_DIAGNOSTIC_PUSH_AND_IGNORED_IF_DEFINED("-Wswitch-enum")             \
    switch (_st) {                                                          \
      __VA_ARGS__                                                           \
      default:                                                              \
        CHECK_NOT_IMPLEMENTED(                                              \
            false,                                                          \
            '"',                                                            \
            at_dispatch_name,                                               \
            "\" not implemented for '",                                     \
            torch::headeronly::toString(_st),                               \
            "'");                                                           \
    }                                                                       \
    C10_DIAGNOSTIC_POP()                                                    \
  }()

#define AT_PRIVATE_CASE_TYPE_USING_HINT(enum_type, HINT, ...) \
  THO_PRIVATE_CASE_TYPE_USING_HINT_TMPL(                      \
      AT_PRIVATE_CHECK_SELECTIVE_BUILD, enum_type, HINT, __VA_ARGS__)

#define AT_DISPATCH_CASE(enum_type, ...) \
  AT_PRIVATE_CASE_TYPE_USING_HINT(enum_type, scalar_t, __VA_ARGS__)

#define THO_PRIVATE_CASE_TYPE_USING_HINT_TMPL(PRELUDE, enum_type, HINT, ...) \
  case enum_type: {                                                          \
    PRELUDE(enum_type);                                                      \
    using HINT [[maybe_unused]] =                                            \
        torch::headeronly::impl::ScalarTypeToCPPTypeT<enum_type>;            \
    return __VA_ARGS__();                                                    \
  }

[&] {
  const auto& the_type = input.scalar_type();
  constexpr const char* at_dispatch_name = "log_sigmoid_cpu";
  torch::headeronly::ScalarType _st = ::detail::scalar_type(the_type);
  switch (_st) {
    case at::ScalarType::Double: {
      using scalar_t [[maybe_unused]] = torch::headeronly::impl::ScalarTypeToCPPTypeT<at::ScalarType::Double>;  // double
      return [&] {
        using Vec = Vectorized<scalar_t>;
        scalar_t* output_data = output.data_ptr<scalar_t>();
        // ... 整个 lambda 体，第一份
      }();
    }
    case at::ScalarType::Float: {
      using scalar_t [[maybe_unused]] = torch::headeronly::impl::ScalarTypeToCPPTypeT<at::ScalarType::Float>;   // float
      return [&] {
        using Vec = Vectorized<scalar_t>;
        scalar_t* output_data = output.data_ptr<scalar_t>();
        // ... 整个 lambda 体，第二份
      }();
    }
    default:
      TORCH_CHECK_NOT_IMPLEMENTED(false, '"', at_dispatch_name, "\" not implemented for '", torch::headeronly::toString(_st), "'");
  }
}()

template <typename T>
T* TensorBase::mutable_data_ptr() const {
  check_type(*this, c10::CppTypeToScalarType<T>());
  return this->unsafeGetTensorImpl()->mutable_data_ptr_impl<T>();
}

/*
 * Adapted from
 * https://github.com/pytorch/pytorch/blob/v2.0.1/aten/src/ATen/Dispatch.h
 */
#pragma once

#include <torch/all.h>

// Need a special dispatch case macro since we will nest the FP8 dispatch.
// Instead of the usual 'scalar_t', this names the dispatched type 'fp8_t'.
#define AT_DISPATCH_FP8_CASE(enum_type, ...) \
  AT_PRIVATE_CASE_TYPE_USING_HINT(enum_type, fp8_t, __VA_ARGS__)

#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...)         \
  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
  AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)  \
  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)

#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
  AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))

//  AT_DISPATCH_V2(
//    self.scalar_type(),
//    "_local_scalar_dense_cpu",
//    AT_WRAP([&] {
//      scalar_t value = *self.data_ptr<scalar_t>();
//      r = Scalar(value);
//    }),
//    AT_EXPAND(AT_ALL_TYPES),
//    AT_EXPAND(AT_COMPLEX_TYPES),
//    kComplexHalf,
//    kHalf,
//  )

#define AT_DISPATCH_V2(TYPE, NAME, BODY, ...) \
  THO_DISPATCH_V2_TMPL(                       \
      AT_DISPATCH_SWITCH,                     \
      AT_DISPATCH_CASE,                       \
      TYPE,                                   \
      NAME,                                   \
      AT_WRAP(BODY),                          \
      __VA_ARGS__)

#define THO_AP_VAR_TMPL(C, N, T, ...) \
  AT_EXPAND(                          \
      AT_CONCAT(THO_AP, AT_NUM_ARGS(__VA_ARGS__))(C, AT_WRAP(N), __VA_ARGS__))
// ...
#define THO_AP1(C, N, _1) C(_1, N)
#define THO_AP2(C, N, _1, _2) C(_1, N) C(_2, N)
// ... 到 THO_AP60

/// ArrayRef - Represent a constant reference to an array (0 or more elements
/// consecutively in memory), i.e. a start pointer and a length.  It allows
/// various APIs to take consecutive elements easily and conveniently.
///
/// This class does not own the underlying data, it is expected to be used in
/// situations where the data resides in some other buffer, whose lifetime
/// extends past that of the ArrayRef. For this reason, it is not in general
/// safe to store an ArrayRef.
///
/// This is intended to be trivially copyable, so it should be passed by
/// value.

 protected:
  /// The start of the array, in an external buffer.
  const T* Data;

  /// The number of elements.
  size_type Length;

  /// Construct a HeaderOnlyArrayRef from a single element.
  constexpr HeaderOnlyArrayRef(const T& OneElt) : Data(&OneElt), Length(1) {}

  /// Construct a HeaderOnlyArrayRef from a pointer and length.
  constexpr HeaderOnlyArrayRef(const T* data, size_t length)
      : Data(data), Length(length) {}
  // ... 通用容器（5.4 节的 enable_if 版本）、std::vector、std::array<T, N>、C 数组 T[N] ...

  /// Construct a HeaderOnlyArrayRef from a std::initializer_list.
  /* implicit */ constexpr HeaderOnlyArrayRef(
      const std::initializer_list<T>& Vec)
      : Data(
            std::begin(Vec) == std::end(Vec) ? static_cast<T*>(nullptr)
                                             : std::begin(Vec)),
        Length(Vec.size()) {}

  /**
   * Return a reference to the sizes of this tensor.  This reference remains
   * valid as long as the tensor is live and not resized.
   */
  IntArrayRef sizes() const {
    if (C10_UNLIKELY(matches_policy(SizesStridesPolicy::CustomSizes))) {
      return sizes_custom();
    }
    return sizes_and_strides_.sizes_arrayref();
  }

  /// Disallow accidental assignment from a temporary.
  ///
  /// The declaration here is extra complicated so that "arrayRef = {}"
  /// continues to select the move assignment operator.
  template <typename U>
  // NOLINTNEXTLINE(modernize-use-constraints)
  std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(
      // NOLINTNEXTLINE(cppcoreguidelines-missing-std-forward)
      U&& Temporary) = delete;

  template <typename U>
  std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(
      std::initializer_list<U>) = delete;

Tensor empty_cpu(
    IntArrayRef size,
    std::optional<ScalarType> dtype_opt,
    std::optional<Layout> layout_opt,
    std::optional<Device> device_opt,
    std::optional<bool> pin_memory_opt,
    std::optional<c10::MemoryFormat> memory_format_opt) {

namespace c10 {

#if !defined(FBCODE_CAFFE2) && !defined(C10_NODEPRECATED)
// NOLINTNEXTLINE(misc-unused-using-decls)
using std::bad_optional_access;
// NOLINTNEXTLINE(misc-unused-using-decls)
using std::make_optional;
// NOLINTNEXTLINE(misc-unused-using-decls)
using std::nullopt;
// NOLINTNEXTLINE(misc-unused-using-decls)
using std::nullopt_t;
// NOLINTNEXTLINE(misc-unused-using-decls)
using std::optional;
#endif
// ... 两个标了 [[deprecated]] 的 value_or_else ...

Tensor empty_cpu(IntArrayRef size, std::optional<ScalarType> dtype_opt, ...)

struct __lambda_at_line_74 {
  // 捕获的变量成为成员：[&] 时是引用，[=] 时是拷贝
  Tensor& x_c; Tensor& out; double& alpha; double& beta;
  void operator()(int64_t begin, int64_t end) const { /* 函数体 */ }
};

  // Convert label in [A-Za-z] to subscript in [0, TOTAL_LABELS)
  auto label_to_subscript = [=](unsigned char label) -> uint8_t {
    return std::isupper(label) ? label - 'A' : label - 'a' + NUM_OF_LETTERS;
  };

auto exp_vec = [](const auto& v) { /* ... */ };   // aten/src/ATen/native/cpu/FlashAttentionKernel.cpp

template <int n>
struct ForcedUnroll {
  template <typename Func, typename... Args>
  C10_ALWAYS_INLINE void operator()(const Func& f, Args... args) const {
    ForcedUnroll<n - 1>{}(f, args...);
    f(std::integral_constant<int, n - 1>{}, args...);
  }
};

template <>
struct ForcedUnroll<1> {
  template <typename Func, typename... Args>
  C10_ALWAYS_INLINE void operator()(const Func& f, Args... args) const {
    f(std::integral_constant<int, 0>{}, args...);
  }
};

  c10::ForcedUnroll<IntegerLog2(kF16RegistersPerIteration)>{}([&offset, &x](auto idx) {
    offset /= 2;
    for (const auto i : c10::irange(offset)) {
      x[i] = x[i] + x[offset + i];
    }
  });

template <class F>
inline void parallel_for(int64_t begin, int64_t end, int64_t grain_size, const F& f);   // 模板参数

TORCH_API void invoke_parallel(int64_t begin, int64_t end, int64_t grain_size,
                               const std::function<void(int64_t, int64_t)>& f);          // 类型擦除

std::function<void()> make_task(const Tensor& x) {
  double scale = compute_scale(x);
  return [&] { use(x, scale); };   // 返回后 scale 已销毁，x 也可能已销毁
}

template <class F>
inline void invoke_parallel(
    int64_t begin,
    int64_t end,
    int64_t grain_size,
    const F& f) {
  std::atomic_flag err_flag = ATOMIC_FLAG_INIT;
  std::exception_ptr eptr;

#pragma omp parallel
  {
    // ...
    if (begin_tid < end) {
      try {
        internal::ThreadIdGuard tid_guard(tid);
        f(begin_tid, std::min(end, chunk_size + begin_tid));
      } catch (...) {
        if (!err_flag.test_and_set()) {
          eptr = std::current_exception();
        }
      }
    }
  }
  if (eptr) {
    std::rethrow_exception(eptr);
  }
}

at::Tensor scale_shift_cpu(const at::Tensor& x, double alpha, double beta) {
  TORCH_CHECK(x.is_floating_point(), "expected floating point tensor");
  auto x_c = x.contiguous();
  auto out = at::empty_like(x_c);
  AT_DISPATCH_FLOATING_TYPES(x_c.scalar_type(), "scale_shift_cpu", [&] {
    const scalar_t* in = x_c.data_ptr<scalar_t>();
    scalar_t* o = out.data_ptr<scalar_t>();
    at::parallel_for(0, x_c.numel(), 4096, [&](int64_t begin, int64_t end) {
      for (int64_t i = begin; i < end; ++i)
        o[i] = static_cast<scalar_t>(alpha) * in[i] + static_cast<scalar_t>(beta);
    });
  });
  return out;
}

// minic10/core/ScalarType.h
#pragma once
#include <cstddef>
#include <cstdint>
#include <type_traits>

namespace minic10 {

// 一张表：(C++ 类型, 枚举名)。下面所有映射都由它生成，加 dtype 只改这里。
// 对照 torch/headeronly/core/ScalarType.h 的 AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS。
#define MINI_FORALL_SCALAR_TYPES(_) \
  _(float, Float)                   \
  _(double, Double)                 \
  _(int64_t, Long)

enum class ScalarType : int8_t {
#define MINI_DEFINE_ENUM(_, name) name,
  MINI_FORALL_SCALAR_TYPES(MINI_DEFINE_ENUM)
#undef MINI_DEFINE_ENUM
  Undefined,
  NumOptions
};

// 运行期值 -> 编译期类型：只声明主模板，不定义。
// 对没有映射的枚举值（Undefined）使用它会直接编译失败，而不是得到一个错的类型。
template <ScalarType N>
struct ScalarTypeToCPPType;

#define MINI_SPECIALIZE_ScalarTypeToCPPType(cpp_type, scalar_type) \
  template <>                                                       \
  struct ScalarTypeToCPPType<ScalarType::scalar_type> {             \
    using type = cpp_type;                                          \
  };
MINI_FORALL_SCALAR_TYPES(MINI_SPECIALIZE_ScalarTypeToCPPType)
#undef MINI_SPECIALIZE_ScalarTypeToCPPType

template <ScalarType N>
using ScalarTypeToCPPTypeT = typename ScalarTypeToCPPType<N>::type;

// 反向：编译期类型 -> 运行期值。继承 integral_constant，因此 CppTypeToScalarType<float>()
// 可以隐式转换成 ScalarType::Float，也可以用 ::value 取。
template <typename T>
struct CppTypeToScalarType;

#define MINI_SPECIALIZE_CppTypeToScalarType(cpp_type, scalar_type)          \
  template <>                                                                \
  struct CppTypeToScalarType<cpp_type>                                       \
      : std::integral_constant<ScalarType, ScalarType::scalar_type> {};
MINI_FORALL_SCALAR_TYPES(MINI_SPECIALIZE_CppTypeToScalarType)
#undef MINI_SPECIALIZE_CppTypeToScalarType

// 运行期查表的两个函数也从同一张表生成
inline constexpr size_t itemsize(ScalarType t) {
  switch (t) {
#define MINI_CASE(cpp_type, name) \
  case ScalarType::name:          \
    return sizeof(cpp_type);
    MINI_FORALL_SCALAR_TYPES(MINI_CASE)
#undef MINI_CASE
    default:
      return 0;
  }
}

inline constexpr const char* toString(ScalarType t) {
  switch (t) {
#define MINI_CASE(_, name) \
  case ScalarType::name:   \
    return #name;
    MINI_FORALL_SCALAR_TYPES(MINI_CASE)
#undef MINI_CASE
    default:
      return "UNKNOWN_SCALAR";
  }
}

// 编译期自检：两张映射互为逆
static_assert(CppTypeToScalarType<ScalarTypeToCPPTypeT<ScalarType::Float>>::value == ScalarType::Float);
static_assert(std::is_same_v<ScalarTypeToCPPTypeT<CppTypeToScalarType<double>::value>, double>);
static_assert(itemsize(ScalarType::Long) == 8);

}  // namespace minic10

// minic10/core/Dispatch.h
#pragma once
#include <stdexcept>
#include <string>
#include "minic10/core/ScalarType.h"

// 对照 torch/headeronly/core/Dispatch.h 的 THO_PRIVATE_CASE_TYPE_USING_HINT_TMPL：
// 一个 case 分支 = 把运行期枚举值 enum_type 映射成编译期类型别名 HINT，然后调用 lambda。
#define MINI_PRIVATE_CASE_TYPE_USING_HINT(enum_type, HINT, ...)              \
  case enum_type: {                                                          \
    using HINT [[maybe_unused]] = ::minic10::ScalarTypeToCPPTypeT<enum_type>; \
    return __VA_ARGS__();                                                    \
  }

// scalar_t 这个名字就是在这里定下来的
#define MINI_DISPATCH_CASE(enum_type, ...) \
  MINI_PRIVATE_CASE_TYPE_USING_HINT(enum_type, scalar_t, __VA_ARGS__)

// 对照 THO_DISPATCH_SWITCH_TMPL：整个 switch 包在一个立即调用的 lambda 里，
// 所以宏可以出现在表达式位置，case 里的 return 也只是从这个 lambda 返回。
#define MINI_DISPATCH_SWITCH(TYPE, NAME, ...)                                  \
  [&] {                                                                        \
    const auto& the_type = TYPE;                                               \
    constexpr const char* mini_dispatch_name = NAME;                           \
    ::minic10::ScalarType _st = the_type;                                      \
    switch (_st) {                                                             \
      __VA_ARGS__                                                              \
      default:                                                                 \
        throw std::runtime_error(std::string("\"") + mini_dispatch_name +      \
                                 "\" not implemented for '" +                  \
                                 ::minic10::toString(_st) + "'");              \
    }                                                                          \
  }()

#define MINI_DISPATCH_CASE_FLOATING_TYPES(...)                    \
  MINI_DISPATCH_CASE(::minic10::ScalarType::Double, __VA_ARGS__) \
  MINI_DISPATCH_CASE(::minic10::ScalarType::Float, __VA_ARGS__)

#define MINI_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
  MINI_DISPATCH_SWITCH(TYPE, NAME, MINI_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))

#define MINI_DISPATCH_CASE_ALL_TYPES(...)           \
  MINI_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)    \
  MINI_DISPATCH_CASE(::minic10::ScalarType::Long, __VA_ARGS__)

#define MINI_DISPATCH_ALL_TYPES(TYPE, NAME, ...) \
  MINI_DISPATCH_SWITCH(TYPE, NAME, MINI_DISPATCH_CASE_ALL_TYPES(__VA_ARGS__))

// minic10/util/ArrayRef.h
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <stdexcept>
#include <type_traits>
#include <vector>

namespace minic10 {

// ArrayRef<T>：指针 + 长度的只读视图。不拥有数据，不能存起来，按值传递。
// 对照 c10/util/ArrayRef.h + torch/headeronly/util/HeaderOnlyArrayRef.h。
template <typename T>
class ArrayRef final {
  const T* data_ = nullptr;
  size_t length_ = 0;

 public:
  using value_type = T;
  using iterator = const T*;
  using const_iterator = const T*;
  using size_type = size_t;

  constexpr ArrayRef() = default;
  // 单个元素：&one 指向调用方的对象
  constexpr ArrayRef(const T& one) : data_(&one), length_(1) {}
  constexpr ArrayRef(const T* data, size_t length) : data_(data), length_(length) {}
  constexpr ArrayRef(const T* begin, const T* end) : data_(begin), length_(end - begin) {}

  // 从 std::vector 隐式构造：vector<bool> 是位域，没有 data()，用 static_assert 挡掉
  template <typename A>
  /* implicit */ ArrayRef(const std::vector<T, A>& vec) : data_(vec.data()), length_(vec.size()) {
    static_assert(!std::is_same_v<T, bool>, "ArrayRef<bool> cannot be constructed from std::vector<bool>");
  }
  // 非类型模板参数 N：std::array<T, N> 和 C 数组 T[N] 的长度是类型的一部分
  template <size_t N>
  /* implicit */ constexpr ArrayRef(const std::array<T, N>& arr) : data_(arr.data()), length_(N) {}
  template <size_t N>
  /* implicit */ constexpr ArrayRef(const T (&arr)[N]) : data_(arr), length_(N) {}
  // 让 f({2, 3}) 能直接调用 f(IntArrayRef)
  /* implicit */ constexpr ArrayRef(const std::initializer_list<T>& il)
      : data_(il.begin() == il.end() ? nullptr : il.begin()), length_(il.size()) {}

  constexpr iterator begin() const { return data_; }
  constexpr iterator end() const { return data_ + length_; }
  constexpr bool empty() const { return length_ == 0; }
  constexpr const T* data() const { return data_; }
  constexpr size_t size() const { return length_; }
  constexpr const T& operator[](size_t i) const { return data_[i]; }

  const T& front() const {
    if (empty()) throw std::out_of_range("ArrayRef: front() of empty list");
    return data_[0];
  }
  const T& back() const {
    if (empty()) throw std::out_of_range("ArrayRef: back() of empty list");
    return data_[length_ - 1];
  }
  const T& at(size_t i) const {
    if (i >= length_) throw std::out_of_range("ArrayRef: index out of range");
    return data_[i];
  }
  ArrayRef<T> slice(size_t n, size_t m) const {
    if (n + m > length_) throw std::out_of_range("ArrayRef: invalid slice");
    return ArrayRef<T>(data_ + n, m);
  }
  ArrayRef<T> slice(size_t n) const { return slice(n, length_ - n); }

  constexpr bool equals(ArrayRef rhs) const {
    if (length_ != rhs.length_) return false;
    for (size_t i = 0; i < length_; ++i)
      if (!(data_[i] == rhs.data_[i])) return false;
    return true;
  }
  // 需要拥有一份时，显式拷出来
  std::vector<T> vec() const { return std::vector<T>(begin(), end()); }

  // 禁止从临时对象赋值：ref = int64_t{5} 或 ref = {int64_t{1}, int64_t{2}} 会立刻悬垂。
  // 写成模板 + enable_if 是为了让 "ref = {}" 仍然选到默认的移动赋值（照抄 c10 的写法）。
  template <typename U>
  std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(U&&) = delete;
  template <typename U>
  std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(std::initializer_list<U>) = delete;
};

template <typename T>
inline bool operator==(ArrayRef<T> a, ArrayRef<T> b) { return a.equals(b); }
template <typename T>
inline bool operator!=(ArrayRef<T> a, ArrayRef<T> b) { return !a.equals(b); }

using IntArrayRef = ArrayRef<int64_t>;

}  // namespace minic10

#include "minic10/util/ArrayRef.h"
// ...
struct TensorImpl : intrusive_ptr_target {
  // 构造函数参数从 std::vector<int64_t> 改成 IntArrayRef，内部用 .vec() 拷一份拥有
  TensorImpl(intrusive_ptr<StorageImpl> storage, IntArrayRef sizes,
             ScalarType dtype, DispatchKey key)
      : storage_(std::move(storage)), sizes_(sizes.vec()), dtype_(dtype), key_(key) {
    // ... strides_ 计算不变 ...
  }
  // 返回视图而不是 const std::vector&：调用方看不到、也不依赖内部容器是什么
  IntArrayRef sizes() const noexcept { return sizes_; }
  IntArrayRef strides() const noexcept { return strides_; }
  // ...
 private:
  intrusive_ptr<StorageImpl> storage_;
  std::vector<int64_t> sizes_;     // 内部仍用 vector 拥有；换成 SmallVector<int64_t, 5> 时对外接口不变
  std::vector<int64_t> strides_;
  // ...
};

class Tensor {
  intrusive_ptr<TensorImpl> impl_;
 public:
  // ...
  IntArrayRef sizes() const { return impl_->sizes(); }
  IntArrayRef strides() const { return impl_->strides(); }
  int64_t dim() const { return static_cast<int64_t>(impl_->sizes().size()); }
  ScalarType dtype() const { return impl_->dtype(); }
  int64_t numel() const { return impl_->numel(); }

  // 成员函数模板：每个 T 一份实例。调用时 T 无法从实参推导，必须显式写 data_ptr<float>()
  template <typename T>
  T* data_ptr() const {
    if (CppTypeToScalarType<T>::value != impl_->dtype()) {
      throw std::runtime_error(std::string("expected scalar type ") +
                               toString(CppTypeToScalarType<T>::value) + " but found " +
                               toString(impl_->dtype()));
    }
    return static_cast<T*>(impl_->data());
  }
  // ...
};

// 参数从 std::vector<int64_t> 改成 IntArrayRef：调用方传 {2, 3}、std::vector、std::array 都行
inline Tensor empty(IntArrayRef sizes, ScalarType dtype) {
  int64_t numel = 1;
  for (auto s : sizes) numel *= s;
  auto storage = make_intrusive<StorageImpl>(numel * itemsize(dtype), GetCPUAllocator());
  return Tensor(make_intrusive<TensorImpl>(std::move(storage), sizes, dtype, DispatchKey::CPU));
}

// minic10/ops/ops.h
#pragma once
#include "minic10/core/Tensor.h"

namespace minic10 {
Tensor add(const Tensor& a, const Tensor& b);
Tensor mul(const Tensor& a, const Tensor& b);
}  // namespace minic10

// minic10/ops/add.cpp
#include <stdexcept>
#include <string>
#include "minic10/core/Dispatch.h"
#include "minic10/core/Tensor.h"
#include "minic10/ops/ops.h"

namespace minic10 {
namespace {

// 函数模板：scalar_t 是编译期参数。它本身不是代码，只是生成代码的配方。
template <typename scalar_t>
void add_kernel(const scalar_t* a, const scalar_t* b, scalar_t* out, int64_t n) {
  for (int64_t i = 0; i < n; ++i) {
    out[i] = a[i] + b[i];
  }
}

void check_binary_inputs(const Tensor& a, const Tensor& b, const char* name) {
  if (!a.defined() || !b.defined()) {
    throw std::runtime_error(std::string(name) + ": undefined tensor");
  }
  if (a.dtype() != b.dtype()) {
    throw std::runtime_error(std::string(name) + ": dtype mismatch: " +
                             toString(a.dtype()) + " vs " + toString(b.dtype()));
  }
  if (a.sizes() != b.sizes()) {   // IntArrayRef 的 operator==：逐元素比较
    throw std::runtime_error(std::string(name) + ": shape mismatch");
  }
}

}  // namespace

Tensor add(const Tensor& a, const Tensor& b) {
  check_binary_inputs(a, b, "add");
  Tensor out = empty(a.sizes(), a.dtype());
  // 运行期的 a.dtype() 在这里变成编译期的 scalar_t；lambda 体被实例化两次（double、float）
  MINI_DISPATCH_FLOATING_TYPES(a.dtype(), "add_cpu", [&] {
    add_kernel<scalar_t>(a.data_ptr<scalar_t>(), b.data_ptr<scalar_t>(),
                         out.data_ptr<scalar_t>(), a.numel());
  });
  return out;
}

}  // namespace minic10

// minic10/ops/mul.cpp
#include <stdexcept>
#include <string>
#include "minic10/core/Dispatch.h"
#include "minic10/core/Tensor.h"
#include "minic10/ops/ops.h"

namespace minic10 {

Tensor mul(const Tensor& a, const Tensor& b) {
  if (a.dtype() != b.dtype() || a.sizes() != b.sizes()) {
    throw std::runtime_error("mul: dtype or shape mismatch");
  }
  Tensor out = empty(a.sizes(), a.dtype());
  // 这次 kernel 直接写在 lambda 里，并且多分发一个 Long：lambda 体被实例化三次
  MINI_DISPATCH_ALL_TYPES(a.dtype(), "mul_cpu", [&] {
    const scalar_t* pa = a.data_ptr<scalar_t>();
    const scalar_t* pb = b.data_ptr<scalar_t>();
    scalar_t* po = out.data_ptr<scalar_t>();
    for (int64_t i = 0; i < a.numel(); ++i) {
      po[i] = pa[i] * pb[i];
    }
  });
  return out;
}

}  // namespace minic10

// main.cpp
#include <cstdio>
#include <exception>
#include "minic10/core/Tensor.h"
#include "minic10/ops/ops.h"

using namespace minic10;

template <typename T>
Tensor arange_like(IntArrayRef sizes) {
  Tensor t = empty(sizes, CppTypeToScalarType<T>::value);   // 类型 -> 枚举
  T* p = t.data_ptr<T>();
  for (int64_t i = 0; i < t.numel(); ++i) p[i] = static_cast<T>(i);
  return t;
}

template <typename T>
void print(const char* tag, const Tensor& t) {
  std::printf("%s [%s, dim=%lld]:", tag, toString(t.dtype()), (long long)t.dim());
  const T* p = t.data_ptr<T>();
  for (int64_t i = 0; i < t.numel(); ++i) std::printf(" %g", static_cast<double>(p[i]));
  std::printf("\n");
}

int main() {
  Tensor f = arange_like<float>({2, 3});
  Tensor d = arange_like<double>({2, 3});
  Tensor l = arange_like<int64_t>({6});

  print<float>("add(f, f)", add(f, f));
  print<double>("add(d, d)", add(d, d));
  print<int64_t>("mul(l, l)", mul(l, l));

  try {
    add(l, l);   // Long 没有进 FLOATING_TYPES 的 case 列表 -> default 分支
  } catch (const std::exception& e) {
    std::printf("add(l, l) threw: %s\n", e.what());
  }
  try {
    f.data_ptr<double>();   // 编译期类型 double 与运行期 dtype Float 不一致
  } catch (const std::exception& e) {
    std::printf("f.data_ptr<double>() threw: %s\n", e.what());
  }
  return 0;
}