build_ncclx.sh

#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

set -x

function do_cmake_build() {
  local source_dir="$1"
  local extra_flags="$2"
  cmake -G Ninja \
    -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \
    -DCMAKE_INSTALL_PREFIX="$CMAKE_PREFIX_PATH" \
    -DCMAKE_MODULE_PATH="$CMAKE_PREFIX_PATH" \
    -DCMAKE_INSTALL_DIR="$CMAKE_PREFIX_PATH" \
    -DBIN_INSTALL_DIR="$CMAKE_PREFIX_PATH/bin" \
    -DLIB_INSTALL_DIR="$CMAKE_PREFIX_PATH/$LIB_SUFFIX" \
    -DINCLUDE_INSTALL_DIR="$CMAKE_PREFIX_PATH/include" \
    -DCMAKE_INSTALL_INCLUDEDIR="$CMAKE_PREFIX_PATH/include" \
    -DCMAKE_INSTALL_LIBDIR="$CMAKE_PREFIX_PATH/$LIB_SUFFIX" \
    -DBUILD_SHARED_LIBS=OFF \
    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
    -DCMAKE_CXX_STANDARD=20 \
    -DCMAKE_POLICY_VERSION_MINIMUM=3.22 \
    $extra_flags \
    -S "${source_dir}"
  ninja -j$(nproc)
  ninja install
}

function clean_third_party {
  local library_name="$1"
  if [ "$CLEAN_THIRD_PARTY" == 1 ]; then
    rm -rf "${CONDA_PREFIX}"/include/"${library_name}"*/
    rm -rf "${CONDA_PREFIX}"/include/"${library_name}"*.h
  fi
}

function build_fb_oss_library() {
  local repo_url="$1"
  local repo_tag="$2"
  local library_name="$3"
  local extra_flags="$4"

  clean_third_party "$library_name"

  if [ ! -e "$library_name" ]; then
    git clone --depth 1 -b "$repo_tag" "$repo_url" "$library_name"
  fi

  local source_dir="../${library_name}/${library_name}"
  if [ -f ${library_name}/CMakeLists.txt ]; then
    source_dir="../${library_name}"
  fi
  if [ -f ${library_name}/build/cmake/CMakeLists.txt ]; then
    source_dir="../${library_name}/build/cmake"
  fi
  if [ -f ${library_name}/cmake_unofficial/CMakeLists.txt ]; then
    source_dir="../${library_name}/cmake_unofficial"
  fi

  export LDFLAGS="-Wl,--allow-shlib-undefined"
  rm -rf build-output
  mkdir -p build-output
  pushd build-output
  do_cmake_build "$source_dir" "$extra_flags"
  popd
}

function build_automake_library() {
  local repo_url="$1"
  local repo_tag="$2"
  local library_name="$3"
  local extra_flags="$4"

  clean_third_party "$library_name"

  if [ ! -e "$library_name" ]; then
    git clone --depth 1 -b "$repo_tag" "$repo_url" "$library_name"
  fi

  export LDFLAGS="-Wl,--allow-shlib-undefined"
  pushd "$library_name"
  ./configure --prefix="$CMAKE_PREFIX_PATH" --disable-pie

  make -j$(nproc)
  make install
  popd
}

function build_boost() {
  local repo_url="https://github.com/boostorg/boost.git"
  local repo_tag="boost-1.82.0"
  local library_name="boost"
  local extra_flags=""

  # clean up existing boost
  clean_third_party "$library_name"

  if [ ! -e "$library_name" ]; then
    git clone -j 10 --recurse-submodules --depth 1 -b "$repo_tag" "$repo_url" "$library_name"
  fi

  export LDFLAGS="-Wl,--allow-shlib-undefined"
  pushd "$library_name"
  ./bootstrap.sh --prefix="$CMAKE_PREFIX_PATH" --libdir="$CMAKE_PREFIX_PATH/$LIB_SUFFIX" --without-libraries=python
  ./b2 -q cxxflags=-fPIC cflags=-fPIC install
  popd
}

function build_openssl() {
  local repo_url="https://github.com/openssl/openssl.git"
  local repo_tag="openssl-3.5.1"
  local library_name="openssl"
  local extra_flags=""

  # clean up existing boost
  clean_third_party "$library_name"

  if [ ! -e "$library_name" ]; then
    git clone -j 10 --recurse-submodules --depth 1 -b "$repo_tag" "$repo_url" "$library_name"
  fi

  pushd "$library_name"
  ./config no-shared --prefix="$CMAKE_PREFIX_PATH" --openssldir="$CMAKE_PREFIX_PATH" --libdir=lib

  make -j$(nproc)
  make install
  popd
}

function build_third_party {
  # build third-party libraries
  if [ "$CLEAN_THIRD_PARTY" == 1 ]; then
    rm -f "${CONDA_PREFIX}"/*.cmake 2>/dev/null || true
  fi
  local third_party_tag="v2026.01.19.00"

  mkdir -p /tmp/third-party
  pushd /tmp/third-party
  if [[ -z "${USE_SYSTEM_LIBS}" ]]; then
    build_fb_oss_library "https://github.com/fmtlib/fmt.git" "11.2.0" fmt "-DFMT_INSTALL=ON -DFMT_TEST=OFF -DFMT_DOC=OFF"
    build_fb_oss_library "https://github.com/fmtlib/fmt.git" "11.2.0" fmt "-DFMT_INSTALL=ON -DFMT_TEST=OFF -DFMT_DOC=OFF -DBUILD_SHARED_LIBS=ON"
    build_fb_oss_library "https://github.com/madler/zlib.git" "v1.2.13" zlib "-DZLIB_BUILD_TESTING=OFF"
    build_boost
    build_openssl
    build_fb_oss_library "https://github.com/Cyan4973/xxHash.git" "v0.8.0" xxhash
    # we need both static and dynamic gflags since thrift generator can't
    # statically link against glog.
    build_fb_oss_library "https://github.com/gflags/gflags.git" "v2.2.2" gflags
    build_fb_oss_library "https://github.com/gflags/gflags.git" "v2.2.2" gflags "-DBUILD_SHARED_LIBS=ON"
    # we need both static and dynamic glog since thrift generator can't
    # statically link against glog.
    build_fb_oss_library "https://github.com/google/glog.git" "v0.4.0" glog
    build_fb_oss_library "https://github.com/google/glog.git" "v0.4.0" glog "-DBUILD_SHARED_LIBS=ON"
    build_fb_oss_library "https://github.com/facebook/zstd.git" "v1.5.6" zstd
    build_automake_library "https://github.com/jedisct1/libsodium.git" "1.0.20-RELEASE" sodium
    build_fb_oss_library "https://github.com/fastfloat/fast_float.git" "v8.0.2" fast_float "-DFASTFLOAT_INSTALL=ON"
    build_fb_oss_library "https://github.com/libevent/libevent.git" "release-2.1.12-stable" event
    build_fb_oss_library "https://github.com/google/double-conversion.git" "v3.3.1" double-conversion
    build_fb_oss_library "https://github.com/facebook/folly.git" "$third_party_tag" folly "-DUSE_STATIC_DEPS_ON_UNIX=ON -DOPENSSL_USE_STATIC_LIBS=ON"
  else
    if [[ -z "${NCCL_SKIP_CONDA_INSTALL}" ]]; then
      DEPS=(
        cmake
        ninja
        jemalloc
        gtest
        boost
        double-conversion
        libevent
        conda-forge::libsodium
        libunwind
        snappy
        conda-forge::fast_float
        libdwarf-dev
        gflags
        glog==0.4.0
        xxhash
        zstd
        conda-forge::zlib
        conda-forge::libopenssl-static
        fmt
      )
      conda install "${DEPS[@]}" --yes
      build_fb_oss_library "https://github.com/facebook/folly.git" "$third_party_tag" folly
    fi
  fi

  if [[ -z "${NCCL_FEEDSTOCK_BUILD}" ]]; then
    build_fb_oss_library "https://github.com/facebookincubator/fizz.git" "$third_party_tag" fizz "-DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF"
    build_fb_oss_library "https://github.com/facebook/mvfst" "$third_party_tag" quic
    build_fb_oss_library "https://github.com/facebook/wangle.git" "$third_party_tag" wangle "-DBUILD_TESTS=OFF"
    build_fb_oss_library "https://github.com/facebook/fbthrift.git" "$third_party_tag" thrift
  fi
  popd
}

function build_comms_tracing_service {
  local include_prefix="comms/analyzer/if"
  local base_dir="${PWD}"
  local build_dir=/tmp/build/comms_tracing_service

  mkdir -p "$build_dir"
  pushd "$build_dir"
  # set up the directory structure
  mkdir -p "$include_prefix"
  cp -r "${base_dir}/${include_prefix}"/* "$include_prefix"
  mv "$include_prefix"/CMakeLists.txt .

  if [[ -z "${NCCL_FEEDSTOCK_BUILD}" ]]; then
    # Reuse thrift cmake build tree (source build path)
    cp -r /tmp/third-party/thrift/build .
  else
    # fbthrift installed via conda — cmake finds it via CMAKE_PREFIX_PATH
    mkdir -p build
  fi

  # build the thrift service library
  cd build
  do_cmake_build ..

  popd
}

if [ -z "$DEV_SIGNATURE" ]; then
    is_git=$(git rev-parse --is-inside-work-tree)
    if [ $is_git ]; then
        DEV_SIGNATURE="git-"$(git rev-parse --short HEAD)
    else
        echo "Cannot detect source repository hash. Skip"
        DEV_SIGNATURE=""
    fi
fi

set -e

export CMAKE_PREFIX_PATH="$CONDA_PREFIX"
export LIB_PREFIX="lib64"

BUILDDIR=${BUILDDIR:="${PWD}/build/ncclx"}
CUDA_HOME=${CUDA_HOME:="/usr/local/cuda"}
NVCC_ARCH=${NVCC_ARCH:="a100,h100"}

# Add b200 support if CUDA 12.8+ is available
CUDA_VERSION=$("${CUDA_HOME}/bin/nvcc" --version | grep -oP 'release \K[0-9]+\.[0-9]+')
CUDA_MAJOR=$(echo "$CUDA_VERSION" | cut -d. -f1)
CUDA_MINOR=$(echo "$CUDA_VERSION" | cut -d. -f2)
if [[ "$CUDA_MAJOR" -gt 12 ]] || [[ "$CUDA_MAJOR" -eq 12 && "$CUDA_MINOR" -ge 8 ]]; then
    NVCC_ARCH="${NVCC_ARCH},b200"
fi
NCCL_FP8=${NCCL_FP8:=1}
CLEAN_BUILD=${CLEAN_BUILD:=0}
LIB_SUFFIX=${LIB_SUFFIX:-lib}
CONDA_INCLUDE_DIR="${CONDA_PREFIX}/include"
CONDA_LIB_DIR="${CONDA_PREFIX}/lib"
NCCL_HOOK_LIBS=${NCCL_HOOK_LIBS:=0}
NCCL_HOME=${NCCL_HOME:="${PWD}/comms/ncclx/stable"}
BASE_DIR=${BASE_DIR:="${PWD}"}
CUDARTLIB=cudart_static
THIRD_PARTY_LDFLAGS=""

if [[ -z "${NCCL_BUILD_SKIP_DEPS}" ]]; then
  echo "Building dependencies"
  build_third_party
  build_comms_tracing_service
fi

# Generate nccl_cvars files (these are no longer checked into the repo)
# The files are generated by extractcvars.py which reads nccl_cvars.yaml and nccl_cvars.cc.in
echo "Generating nccl_cvars files..."
CVARS_DIR="$BASE_DIR/comms/utils/cvars"

# Validate that the required source files exist
if [ ! -f "$CVARS_DIR/extractcvars.py" ]; then
  echo "ERROR: extractcvars.py not found at $CVARS_DIR/extractcvars.py"
  exit 1
fi
if [ ! -f "$CVARS_DIR/nccl_cvars.yaml" ]; then
  echo "ERROR: nccl_cvars.yaml not found at $CVARS_DIR/nccl_cvars.yaml"
  exit 1
fi
if [ ! -f "$CVARS_DIR/nccl_cvars.cc.in" ]; then
  echo "ERROR: nccl_cvars.cc.in not found at $CVARS_DIR/nccl_cvars.cc.in"
  exit 1
fi

# Install pyyaml if not already installed (required by extractcvars.py)
if [[ -z "${NCCL_SKIP_CONDA_INSTALL}" ]]; then
  conda install pyyaml --yes
fi

# Run the extractcvars.py script directly to generate the files
export NCCL_CVARS_OUTPUT_DIR="$CVARS_DIR"
python3 "$CVARS_DIR/extractcvars.py"

# Verify the files were generated
if [ ! -f "$CVARS_DIR/nccl_cvars.h" ] || [ ! -f "$CVARS_DIR/nccl_cvars.cc" ]; then
  echo "ERROR: Failed to generate nccl_cvars files"
  exit 1
fi
echo "Successfully generated nccl_cvars files in $CVARS_DIR"

# set up the third-party ldflags
export PKG_CONFIG_PATH="${CONDA_LIB_DIR}"/pkgconfig
THRIFT_SERVICE_LDFLAGS=(
  "-l:libcomms_tracing_service.a"
  "-Wl,--start-group"
  "-l:libasync.a"
  "-l:libconcurrency.a"
  "-l:libthrift-core.a"
  "-l:libthriftanyrep.a"
  "-l:libthriftcpp2.a"
)

# libthrift_dynamic_value.a and libthrift_path.a are only available in fbthrift main branch
# (used by feedstock build), not in the pinned v2026.01.19.00 tag (used by OSS build)
if [[ -n "${NCCL_FEEDSTOCK_BUILD}" ]]; then
  THRIFT_SERVICE_LDFLAGS+=(
    "-l:libthrift_dynamic_value.a"
    "-l:libthrift_path.a"
  )
fi

THRIFT_SERVICE_LDFLAGS+=(
  "-l:libthriftmetadata.a"
  "-l:libthriftprotocol.a"
  "-l:libthrifttype.a"
  "-l:libthrifttyperep.a"
  "-l:librpcmetadata.a"
  "-l:libruntime.a"
  "-l:libserverdbginfo.a"
  "-l:libtransport.a"
  "-l:libcommon.a"
  "-Wl,--end-group"
  "-l:libwangle.a"
  "-l:libfizz.a"
  "-l:libxxhash.a"
)
THIRD_PARTY_LDFLAGS+="${THRIFT_SERVICE_LDFLAGS[*]} "
THIRD_PARTY_LDFLAGS+="$(pkg-config --libs --static libfolly) "
if [[ -z "${USE_SYSTEM_LIBS}" ]]; then
  THIRD_PARTY_LDFLAGS+="-l:libglog.a -l:libgflags.a -l:libboost_context.a -l:libfmt.a -l:libssl.a -l:libcrypto.a"
else
  THIRD_PARTY_LDFLAGS+="-lglog -lgflags -lboost_context -lfmt -lssl -lcrypto"
fi

echo "$THIRD_PARTY_LDFLAGS"

if [[ -z "${NVCC_GENCODE-}" ]]; then
    IFS=',' read -ra arch_array <<< "$NVCC_ARCH"
    arch_gencode=""
    for arch in "${arch_array[@]}"
    do
        case "$arch" in
        "p100")
        arch_gencode="$arch_gencode -gencode=arch=compute_60,code=sm_60"
            ;;
        "v100")
        arch_gencode="$arch_gencode -gencode=arch=compute_70,code=sm_70"
            ;;
        "a100")
        arch_gencode="$arch_gencode -gencode=arch=compute_80,code=sm_80"
        ;;
        "h100")
            arch_gencode="$arch_gencode -gencode=arch=compute_90,code=sm_90"
        ;;
        "b200")
            arch_gencode="$arch_gencode -gencode=arch=compute_100a,code=sm_100a"
        ;;
        "b300")
            arch_gencode="$arch_gencode -gencode=arch=compute_103a,code=sm_103a"
        ;;
        esac
    done
    NVCC_GENCODE=$arch_gencode
fi

if [ "$CLEAN_BUILD" == 1 ]; then
    rm -rf "$BUILDDIR"
fi

mkdir -p "$BUILDDIR"

# Pre-generate nccl_git_version.h before the make build starts.
# The Makefile has a rule for this, but it fails with re-cc/distcc because
# the generated header is not available on the remote compilation machine.
GIT_VERSION_DIR="$BUILDDIR/obj/include"
mkdir -p "$GIT_VERSION_DIR"
if [ -f "${NCCL_HOME}/src/misc/generate_git_version.py" ]; then
  python3 "${NCCL_HOME}/src/misc/generate_git_version.py" "$GIT_VERSION_DIR/nccl_git_version.h"
else
  cat > "$GIT_VERSION_DIR/nccl_git_version.h" <<HEADER
/* Auto-generated fallback. */
#ifndef NCCL_GIT_VERSION_H
#define NCCL_GIT_VERSION_H
#ifndef NCCL_GIT_BRANCH
#define NCCL_GIT_BRANCH "unknown"
#endif
#ifndef NCCL_GIT_COMMIT_HASH
#define NCCL_GIT_COMMIT_HASH "${DEV_SIGNATURE:-unknown}"
#endif
#endif /* NCCL_GIT_VERSION_H */
HEADER
fi

pushd "${NCCL_HOME}"

function build_nccl {
  make VERBOSE=1 -j$(nproc) \
    src.build \
    BUILDDIR="$BUILDDIR" \
    NVCC_GENCODE="$NVCC_GENCODE" \
    CUDA_HOME="$CUDA_HOME" \
    NCCL_HOME="$NCCL_HOME" \
    NCCL_SUFFIX="x-${DEV_SIGNATURE}" \
    NCCL_FP8="$NCCL_FP8" \
    BASE_DIR="$BASE_DIR" \
    CONDA_INCLUDE_DIR="$CONDA_INCLUDE_DIR" \
    CONDA_LIB_DIR="$CONDA_LIB_DIR" \
    THIRD_PARTY_LDFLAGS="$THIRD_PARTY_LDFLAGS" \
    CUDARTLIB="$CUDARTLIB" \
    ENABLE_PIPES="${ENABLE_PIPES:-0}"
}

function build_and_install_nccl {
make VERBOSE=1 -j$(nproc) \
    src.install \
    BUILDDIR="$BUILDDIR" \
    NVCC_GENCODE="$NVCC_GENCODE" \
    CUDA_HOME="$CUDA_HOME" \
    NCCL_HOME="$NCCL_HOME" \
    NCCL_SUFFIX="x-${DEV_SIGNATURE}" \
    NCCL_FP8="$NCCL_FP8" \
    BASE_DIR="$BASE_DIR" \
    CONDA_INCLUDE_DIR="$CONDA_INCLUDE_DIR" \
    CONDA_LIB_DIR="$CONDA_LIB_DIR" \
    THIRD_PARTY_LDFLAGS="$THIRD_PARTY_LDFLAGS" \
    CUDARTLIB="$CUDARTLIB" \
    ENABLE_PIPES="${ENABLE_PIPES:-0}"
}

if [[ -z "${NCCL_BUILD_AND_INSTALL}" ]]; then
  build_nccl
else
  build_and_install_nccl
fi

# sanity check
if [ -n "${NCCL_RUN_SANITY_CHECK}" ]; then
    if [ -f examples/Makefile ]; then
        pushd examples
        export NCCL_DEBUG=WARN
        export LD_LIBRARY_PATH=$BUILDDIR/lib

        make all \
          NVCC_GENCODE="$NVCC_GENCODE" \
          CUDA_HOME="$CUDA_HOME" \
          NCCL_HOME="$CONDA_PREFIX"

        set +e

        TIMEOUT=10s
        timeout $TIMEOUT "$BUILDDIR"/examples/HelloWorld
        if [ "$?" == "124" ]; then
            echo "Program TIMEOUT in ${TIMEOUT}. Terminate."
        fi
        popd
    else
        echo "Skipping sanity check: examples/Makefile not found"
    fi
fi

popd