From a1e27ca22fa8a1393360a12da1484961e9fff580 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Fri, 25 Jun 2021 05:02:25 +0900 Subject: [PATCH 1/4] 1. changed oneCCL patch to remove unnecessary files during installation. 2. Replaced PYTORCH_INCLUDE_DIRS and PYTORCH_LIBRARY_DIRS with PYTORCH_INSTALL_DIR --- CMakeLists.txt | 4 +- .../Fix_the_link_path_issue_in_oneCCL.patch | 24 ----- patches/oneCCL.patch | 100 ++++++++++++++++++ setup.py | 4 +- src/CMakeLists.txt | 4 +- 5 files changed, 106 insertions(+), 30 deletions(-) delete mode 100644 patches/Fix_the_link_path_issue_in_oneCCL.patch create mode 100644 patches/oneCCL.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index f632614..64d1154 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,8 +28,8 @@ if(Patch_FOUND) else() message(FATAL_ERROR, "Did not find patch") endif() -set (patch_file "${CMAKE_CURRENT_SOURCE_DIR}/patches/Fix_the_link_path_issue_in_oneCCL.patch") -execute_process(COMMAND ${Patch_EXECUTABLE} -p1 -i ${patch_file} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +set (patch_file "${CMAKE_CURRENT_SOURCE_DIR}/patches/oneCCL.patch") +execute_process(COMMAND ${Patch_EXECUTABLE} -p1 -i ${patch_file} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party/oneCCL) # Find OneCCL Lib set(DEPENDS_LIB) diff --git a/patches/Fix_the_link_path_issue_in_oneCCL.patch b/patches/Fix_the_link_path_issue_in_oneCCL.patch deleted file mode 100644 index 38f9caf..0000000 --- a/patches/Fix_the_link_path_issue_in_oneCCL.patch +++ /dev/null @@ -1,24 +0,0 @@ -Index: torch-ccl/third_party/oneCCL/src/CMakeLists.txt -IDEA additional info: -Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP -<+>UTF-8 -=================================================================== ---- torch-ccl/third_party/oneCCL/src/CMakeLists.txt (revision 15dafd4094016c50a571b4e891aa344db090e7f2) -+++ torch-ccl/third_party/oneCCL/src/CMakeLists.txt (date 1608299636844) -@@ -232,7 +232,7 @@ - add_library(ccl SHARED $) - target_include_directories(ccl PUBLIC ${CCL_INC_DIRS}) - # link with release_mt libmpi.so for oneAPI Base toolkit --set_target_properties(ccl PROPERTIES LINK_FLAGS "-Wl,-rpath,../../../../mpi/latest/lib/release_mt/") -+set_target_properties(ccl PROPERTIES LINK_FLAGS "-Wl,-rpath,${CMAKE_INSTALL_RPATH}") - target_link_libraries(ccl PUBLIC dl pthread ${EXTERNAL_LIBS} ${COMPUTE_RUNTIME_TARGET_NAME} fabric mpi) - if (NOT LIB_SO_VERSION AND NOT LIB_MAJOR_VERSION) - set_target_properties(ccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CCL_BUILD_DIR}) -@@ -240,6 +240,7 @@ - set_target_properties(ccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CCL_BUILD_DIR} VERSION ${LIB_SO_VERSION} SOVERSION ${LIB_MAJOR_VERSION}) - endif() - install(TARGETS ccl LIBRARY DESTINATION ${CCL_INSTALL_LIB}) -+ - install(FILES - "../cmake/FindOpenCL.cmake" - "../cmake/FindL0.cmake" diff --git a/patches/oneCCL.patch b/patches/oneCCL.patch new file mode 100644 index 0000000..84b2755 --- /dev/null +++ b/patches/oneCCL.patch @@ -0,0 +1,100 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e4dce00..fa3f90c 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -253,9 +253,9 @@ configure_file(cmake/vars.sh.in ${CMAKE_CURRENT_BINARY_DIR}/vars.sh @ONLY) + configure_file(cmake/setvars.sh.in ${CMAKE_CURRENT_BINARY_DIR}/setvars.sh @ONLY) + configure_file(cmake/ccl ${CMAKE_CURRENT_BINARY_DIR}/ccl @ONLY) + configure_file(third-party-programs.txt ${CMAKE_CURRENT_BINARY_DIR}/third-party-programs.txt COPYONLY) +-install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/vars.sh DESTINATION ${CCL_INSTALL_ENV}) +-install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/setvars.sh DESTINATION ${CCL_INSTALL_ENV}) +-install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ccl DESTINATION ${CCL_INSTALL_MODULE}) ++#install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/vars.sh DESTINATION ${CCL_INSTALL_ENV}) ++#install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/setvars.sh DESTINATION ${CCL_INSTALL_ENV}) ++#install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/ccl DESTINATION ${CCL_INSTALL_MODULE}) + install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/third-party-programs.txt DESTINATION ${CCL_INSTALL_LICENSE}) + install(PROGRAMS ${PROJECT_SOURCE_DIR}/LICENSE DESTINATION ${CCL_INSTALL_LICENSE}) + +@@ -274,14 +274,14 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}/include) + #include other CMakeLists + + add_subdirectory(src) +-add_subdirectory(examples/benchmark) +-add_subdirectory(examples/common) +-add_subdirectory(examples/cpu) ++#add_subdirectory(examples/benchmark) ++#add_subdirectory(examples/common) ++#add_subdirectory(examples/cpu) + if (CCL_ENABLE_SYCL) + add_subdirectory(examples/sycl) + endif() + +-add_subdirectory(tests/functional) ++#add_subdirectory(tests/functional) + + if (BUILD_UT) + #add_subdirectory(tests/unit) +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index d5a25c0..a15695c 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -240,20 +240,21 @@ else() + set_target_properties(ccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CCL_BUILD_DIR} VERSION ${LIB_SO_VERSION} SOVERSION ${LIB_MAJOR_VERSION}) + endif() + install(TARGETS ccl LIBRARY DESTINATION ${CCL_INSTALL_LIB}) +-install(FILES +- "../cmake/FindOpenCL.cmake" +- "../cmake/FindL0.cmake" +- "../cmake/FindSYCL.cmake" +- "../cmake/FindIntelSYCL.cmake" +- "../cmake/FindComputeCpp.cmake" +- DESTINATION ${CCL_INSTALL_LIB}) ++ ++#install(FILES ++# "../cmake/FindOpenCL.cmake" ++# "../cmake/FindL0.cmake" ++# "../cmake/FindSYCL.cmake" ++# "../cmake/FindIntelSYCL.cmake" ++# "../cmake/FindComputeCpp.cmake" ++# DESTINATION ${CCL_INSTALL_LIB}) + + + #static library + add_library(ccl-static STATIC $) + set_target_properties(ccl-static PROPERTIES OUTPUT_NAME ccl) + set_target_properties(ccl-static PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CCL_BUILD_DIR}) +-install(TARGETS ccl-static ARCHIVE DESTINATION ${CCL_INSTALL_LIB} OPTIONAL) ++#install(TARGETS ccl-static ARCHIVE DESTINATION ${CCL_INSTALL_LIB} OPTIONAL) + + if(MULTI_GPU_SUPPORT) + message("Turn on L0 multi-gpu unit tests") +@@ -266,8 +267,8 @@ if(MULTI_GPU_SUPPORT) + endif(MULTI_GPU_SUPPORT) + + #headers installation +-install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/ +- DESTINATION ${CCL_INSTALL_INCLUDE} FILES_MATCHING REGEX ".*\\.(h|hpp)$") ++#install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/ ++# DESTINATION ${CCL_INSTALL_INCLUDE} FILES_MATCHING REGEX ".*\\.(h|hpp)$") + + #mpi & ofi rt + file(GLOB mpi_bins "${PROJECT_SOURCE_DIR}/mpi/bin/*") +@@ -276,14 +277,14 @@ install(PROGRAMS ${mpi_bins} DESTINATION ${CCL_INSTALL_BIN}) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/ofi/lib/ + DESTINATION ${CCL_INSTALL_LIB}) + +-install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/include/ +- DESTINATION ${CCL_INSTALL_INCLUDE}) ++#install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/include/ ++# DESTINATION ${CCL_INSTALL_INCLUDE}) + + install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/lib/ + DESTINATION ${CCL_INSTALL_LIB}) + +-install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/etc/ +- DESTINATION ${CCL_INSTALL_ETC}) ++#install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/etc/ ++# DESTINATION ${CCL_INSTALL_ETC}) + + install(DIRECTORY ${PROJECT_SOURCE_DIR}/mpi/licensing/ + DESTINATION ${CCL_INSTALL_LICENSE}/mpi/) diff --git a/setup.py b/setup.py index dad90b9..da97622 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def build_cmake(self, extension: CMakeExtension): build_dir.mkdir(parents=True, exist_ok=True) install_dir = TORCH_CCL_PATH + torch_include_dir = CMakeExtension.convert_cmake_dirs(include_paths()) # Now that the necessary directories are created, build my_env = os.environ.copy() @@ -88,8 +89,7 @@ def build_cmake(self, extension: CMakeExtension): build_options = { # The value cannot be easily obtained in CMakeLists.txt. 'PYTHON_INCLUDE_DIRS': str(distutils.sysconfig.get_python_inc()), - 'PYTORCH_INCLUDE_DIRS': CMakeExtension.convert_cmake_dirs(include_paths()), - 'PYTORCH_LIBRARY_DIRS': CMakeExtension.convert_cmake_dirs(library_paths()), + 'PYTORCH_INSTALL_DIR': os.path.join(torch_include_dir, '..'), } extension.generate(build_options, my_env, build_dir, install_dir) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76ce7db..75f471b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ target_compile_options(torch_ccl PUBLIC -Wall -Wno-sign-compare -Wno-unused-function) -target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INCLUDE_DIRS} ${IPEX_INCLUDE_DIRS} ./) +target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INSTALL_DIR}/include ${IPEX_INCLUDE_DIRS} ./) get_target_property(mpi_lib mpi IMPORTED_LOCATION) get_filename_component(MPI_LIBRARY_DIRS ${mpi_lib} DIRECTORY) @@ -15,7 +15,7 @@ get_target_property(fabric_lib fabric IMPORTED_LOCATION) get_filename_component(FABRIC_LIBRARY_DIRS ${fabric_lib} DIRECTORY) target_link_directories(torch_ccl PUBLIC ${MPI_LIBRARY_DIRS} ${FABRIC_LIBRARY_DIRS}) -target_link_directories(torch_ccl PUBLIC ${PYTORCH_LIBRARY_DIRS}) +target_link_directories(torch_ccl PUBLIC ${PYTORCH_INSTALL_DIR}/lib) target_link_directories(torch_ccl PUBLIC ${IPEX_LIBRARY_DIRS}) target_link_libraries(torch_ccl PUBLIC ${DEPENDS_LIB}) target_link_libraries(torch_ccl PUBLIC libtorch_python.so) From 6113dfa43dde8029f82999f54a130574adcfb104 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Fri, 25 Jun 2021 05:28:32 +0900 Subject: [PATCH 2/4] changed oneCCL patch to remove unnecessary files during installation. --- setup.py | 3 ++- src/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index da97622..69f57db 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,8 @@ def build_cmake(self, extension: CMakeExtension): build_options = { # The value cannot be easily obtained in CMakeLists.txt. 'PYTHON_INCLUDE_DIRS': str(distutils.sysconfig.get_python_inc()), - 'PYTORCH_INSTALL_DIR': os.path.join(torch_include_dir, '..'), + 'PYTORCH_INCLUDE_DIRS': CMakeExtension.convert_cmake_dirs(include_paths()), + 'PYTORCH_LIBRARY_DIRS': CMakeExtension.convert_cmake_dirs(library_paths()), } extension.generate(build_options, my_env, build_dir, install_dir) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 75f471b..34bbe0c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ target_compile_options(torch_ccl PUBLIC -Wall -Wno-sign-compare -Wno-unused-function) -target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INSTALL_DIR}/include ${IPEX_INCLUDE_DIRS} ./) + target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INCLUDE_DIRS} ${IPEX_INCLUDE_DIRS} ./) get_target_property(mpi_lib mpi IMPORTED_LOCATION) get_filename_component(MPI_LIBRARY_DIRS ${mpi_lib} DIRECTORY) @@ -15,7 +15,7 @@ get_target_property(fabric_lib fabric IMPORTED_LOCATION) get_filename_component(FABRIC_LIBRARY_DIRS ${fabric_lib} DIRECTORY) target_link_directories(torch_ccl PUBLIC ${MPI_LIBRARY_DIRS} ${FABRIC_LIBRARY_DIRS}) -target_link_directories(torch_ccl PUBLIC ${PYTORCH_INSTALL_DIR}/lib) +target_link_directories(torch_ccl PUBLIC ${PYTORCH_LIBRARY_DIRS}) target_link_directories(torch_ccl PUBLIC ${IPEX_LIBRARY_DIRS}) target_link_libraries(torch_ccl PUBLIC ${DEPENDS_LIB}) target_link_libraries(torch_ccl PUBLIC libtorch_python.so) From a5d8ef99a24a287e72643f52db5ea1d6ad5cdf50 Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Fri, 25 Jun 2021 05:30:37 +0900 Subject: [PATCH 3/4] changed oneCCL patch to remove unnecessary files during installation. --- setup.py | 1 - src/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 69f57db..dad90b9 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ def build_cmake(self, extension: CMakeExtension): build_dir.mkdir(parents=True, exist_ok=True) install_dir = TORCH_CCL_PATH - torch_include_dir = CMakeExtension.convert_cmake_dirs(include_paths()) # Now that the necessary directories are created, build my_env = os.environ.copy() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 34bbe0c..76ce7db 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ target_compile_options(torch_ccl PUBLIC -Wall -Wno-sign-compare -Wno-unused-function) - target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INCLUDE_DIRS} ${IPEX_INCLUDE_DIRS} ./) +target_include_directories(torch_ccl PUBLIC ${ONECCL_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} ${PYTORCH_INCLUDE_DIRS} ${IPEX_INCLUDE_DIRS} ./) get_target_property(mpi_lib mpi IMPORTED_LOCATION) get_filename_component(MPI_LIBRARY_DIRS ${mpi_lib} DIRECTORY) From 240b9f1e1dc4bc3fd2fd3f38d6ded0a8f236e9ce Mon Sep 17 00:00:00 2001 From: Jing Xu Date: Fri, 25 Jun 2021 15:17:29 +0900 Subject: [PATCH 4/4] fix rpath issue --- patches/oneCCL.patch | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/patches/oneCCL.patch b/patches/oneCCL.patch index 84b2755..78db2c2 100644 --- a/patches/oneCCL.patch +++ b/patches/oneCCL.patch @@ -35,9 +35,18 @@ index e4dce00..fa3f90c 100644 if (BUILD_UT) #add_subdirectory(tests/unit) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index d5a25c0..a15695c 100644 +index d5a25c0..b644a4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt +@@ -232,7 +232,7 @@ link_directories(${LIBFABRIC_LIB_DIR}) + add_library(ccl SHARED $) + target_include_directories(ccl PUBLIC ${CCL_INC_DIRS}) + # link with release_mt libmpi.so for oneAPI Base toolkit +-set_target_properties(ccl PROPERTIES LINK_FLAGS "-Wl,-rpath,../../../../mpi/latest/lib/release_mt/") ++set_target_properties(ccl PROPERTIES LINK_FLAGS "-Wl,-rpath,${CMAKE_INSTALL_RPATH}") + target_link_libraries(ccl PUBLIC dl pthread ${EXTERNAL_LIBS} ${COMPUTE_RUNTIME_TARGET_NAME} fabric mpi) + if (NOT LIB_SO_VERSION AND NOT LIB_MAJOR_VERSION) + set_target_properties(ccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CCL_BUILD_DIR}) @@ -240,20 +240,21 @@ else() set_target_properties(ccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CCL_BUILD_DIR} VERSION ${LIB_SO_VERSION} SOVERSION ${LIB_MAJOR_VERSION}) endif()