diff --git a/t/transformers/LICENSE b/t/transformers/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/t/transformers/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/t/transformers/build_info.json b/t/transformers/build_info.json new file mode 100644 index 0000000000..d1dd3841e0 --- /dev/null +++ b/t/transformers/build_info.json @@ -0,0 +1,17 @@ +{ + "maintainer": "rushi-sathe", + "package_name": "transformers", + "github_url": "https://github.com/huggingface/transformers.git", + "version": "v4.52.4", + "default_branch": "master", + "build_script": "transformers_ubi_9.6.sh", + "package_dir": "t/transformers", + "docker_build": false, + "wheel_build": true, + "validate_build_script": true, + "use_non_root_user": false, + + "*" : { + "build_script": "transformers_ubi_9.6.sh" + } +} diff --git a/t/transformers/transformers_ubi_9.6.sh b/t/transformers/transformers_ubi_9.6.sh new file mode 100644 index 0000000000..f68af07833 --- /dev/null +++ b/t/transformers/transformers_ubi_9.6.sh @@ -0,0 +1,807 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# +# Package : transformers +# Version : v4.52.4 +# Source repo : https://github.com/huggingface/transformers.git +# Tested on : UBI 9.6 +# Language : Python +# Ci-Check : True +# Script License: Apache License 2.0 +# Maintainer : Rushikesh Sathe +# +# Disclaimer: This script has been tested in root mode on given +# ========== platform using the mentioned version of the package. +# It may not work as expected with newer versions of the +# package and/or distribution. In such case, please +# contact "Maintainer" of this script. +# +# ---------------------------------------------------------------------------- + + +PACKAGE_NAME=transformers +PACKAGE_VERSION=${1:-v4.52.4} +PACKAGE_URL=https://github.com/huggingface/transformers.git +CURRENT_DIR=$(pwd) +SCRIPT_DIR=$(pwd) +PACKAGE_DIR=transformers + +echo "-----------------Installing dependencies---------------------" +yum install -y git make libtool wget tar openssl-devel cmake zlib-devel libjpeg-devel gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran cmake libevent-devel zlib-devel openssl-devel clang libtool pkg-config brotli-devel bzip2-devel utf8proc xz bzip2-devel libffi-devel patch ninja-build +yum install -y wget git make python-devel xz-devel openssl-devel cmake zlib-devel libjpeg-devel gcc-toolset-13 cmake libevent libtool pkg-config brotli-devel.ppc64le bzip2-devel lz4-devel python3.12 python3.12-devel python3.12-pip +export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH +export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH +export LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib/gcc/ppc64le-redhat-linux/13:$LIBRARY_PATH +export CPATH=/opt/rh/gcc-toolset-13/root/usr/include:$CPATH +GCC_BIN_DIR=$(echo "$PATH" | cut -d':' -f1) +export GCC_HOME=$(dirname "$GCC_BIN_DIR") +export CC="$GCC_BIN_DIR/gcc" +export CXX="$GCC_BIN_DIR/g++" + +CURRENT_DIR=$(pwd) +mkdir -p builder/wheels +pip3.12 install ninja setuptools setuptools-scm Cython wheel + +echo "-------Installing cmake---------" +#install cmake +wget https://cmake.org/files/v3.28/cmake-3.28.0.tar.gz +tar -zxvf cmake-3.28.0.tar.gz +cd cmake-3.28.0 +./bootstrap +make +make install +cd $CURRENT_DIR + + +echo "-----------flex installing------------------" +wget https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz +tar -xvf flex-2.6.4.tar.gz +cd flex-2.6.4 +./configure --prefix=/usr/local +make -j$(nproc) +make install +cd $CURRENT_DIR + +echo "-------bison installing----------------------" +wget https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.gz +tar -xvf bison-3.8.2.tar.gz +cd bison-3.8.2 +./configure --prefix=/usr/local +make -j$(nproc) +make install +cd $CURRENT_DIR + +echo "------------ gflags installing-------------------" +git clone https://github.com/gflags/gflags.git +cd gflags +mkdir build && cd build +cmake .. +make -j$(nproc) +make install +cd $CURRENT_DIR + +echo "----------Installing c-ares----------------" +#Building c-areas +git clone https://github.com/c-ares/c-ares.git +cd c-ares +git checkout cares-1_19_1 + + +target_platform=$(uname)-$(uname -m) +AR=$(which ar) +PKG_NAME=c-ares + +mkdir -p c_ares_prefix +export C_ARES_PREFIX=$(pwd)/c_ares_prefix + +echo "Building ${PKG_NAME}." + +# Isolate the build. +mkdir build && cd build + +if [[ "$PKG_NAME" == *static ]]; then + CARES_STATIC=ON + CARES_SHARED=OFF +else + CARES_STATIC=OFF + CARES_SHARED=ON +fi + +if [[ "${target_platform}" == Linux-* ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR}" +fi + + +# Generate the build files. +echo "Generating the build files..." +cmake ${CMAKE_ARGS} .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$C_ARES_PREFIX" \ + -DCARES_STATIC=${CARES_STATIC} \ + -DCARES_SHARED=${CARES_SHARED} \ + -DCARES_INSTALL=ON \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -GNinja + #${SRC_DIR} + +# Build. +echo "Building..." +ninja || exit 1 + +# Installing +echo "Installing..." +ninja install || exit 1 + +cd $CURRENT_DIR + +echo "----------c-areas installed-----------------------" + +echo "----------------rapidjson installing------------------" +git clone https://github.com/Tencent/rapidjson.git +cd rapidjson +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local +make -j$(nproc) +make install +cd $CURRENT_DIR + +echo "--------------xsimd installing-------------------------" +git clone https://github.com/xtensor-stack/xsimd.git +cd xsimd +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local +make -j$(nproc) +make install +cd $CURRENT_DIR + + +echo "-----------------snappy installing----------------" +git clone https://github.com/google/snappy.git +cd snappy +git submodule update --init --recursive + +mkdir -p local/snappy +export SNAPPY_PREFIX=$(pwd)/local/snappy +mkdir build +cd build + +cmake -DCMAKE_INSTALL_PREFIX=$SNAPPY_PREFIX \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_LIBDIR=lib \ + .. +make -j$(nproc) +make install +cd .. +cd $CURRENT_DIR + + +echo "------------libzstd installing-------------------------" +git clone https://github.com/facebook/zstd.git +cd zstd +make +make install +export ZSTD_HOME=/usr/local +export CMAKE_PREFIX_PATH=$ZSTD_HOME +export LD_LIBRARY_PATH=$ZSTD_HOME/lib64:$LD_LIBRARY_PATH +cd $CURRENT_DIR + +#Installing re2,orc utf8proc,boost_cpp,thrift_cpp,abseil_cpp,libprotobuf, grpc_cpp,openblas as dependencies + + +#re2 install from sosurce +echo "------------ re2 installing-------------------" + +git clone http://github.com/google/re2 +cd re2 +git checkout 2022-04-01 + +git submodule update --init + +mkdir re2-prefix + +export RE2_PREFIX=$(pwd)/re2-prefix + +export CPU_COUNT=`nproc` + +mkdir build-cmake +pushd build-cmake + +cmake ${CMAKE_ARGS} -GNinja \ + -DCMAKE_PREFIX_PATH=$RE2_PREFIX \ + -DCMAKE_INSTALL_PREFIX="${RE2_PREFIX}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DENABLE_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + .. + + ninja -v install + popd +make -j "${CPU_COUNT}" prefix=${RE2_PREFIX} shared-install +cd $CURRENT_DIR + + + +echo "------------ utf8proc installing-------------------" + +git clone https://github.com/JuliaStrings/utf8proc.git +cd utf8proc +git submodule update --init +git checkout v2.6.1 + +mkdir utf8proc_prefix +export UTF8PROC_PREFIX=$(pwd)/utf8proc_prefix + +# Create build directory +mkdir build +cd build + +# Run cmake to configure the build +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE="Release" \ + -DCMAKE_INSTALL_PREFIX="${UTF8PROC_PREFIX}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + -DBUILD_SHARED_LIBS=1 \ + .. +# Build and install +cmake --build . +cmake --build . --target install +cd $CURRENT_DIR + +echo "------------ abseil_cpp cloning-------------------" + +ABSEIL_VERSION=20240116.2 +ABSEIL_URL="https://github.com/abseil/abseil-cpp" + + +git clone $ABSEIL_URL -b $ABSEIL_VERSION + + +echo "------------ libprotobuf installing-------------------" + +export C_COMPILER=$CC +export CXX_COMPILER=$CXX + +#Build libprotobuf +git clone https://github.com/protocolbuffers/protobuf +cd protobuf +git checkout v4.25.8 + +LIBPROTO_DIR=$(pwd) +mkdir -p $LIBPROTO_DIR/local/libprotobuf +LIBPROTO_INSTALL=$LIBPROTO_DIR/local/libprotobuf + +git submodule update --init --recursive +rm -rf ./third_party/googletest | true +rm -rf ./third_party/abseil-cpp | true + +cp -r $CURRENT_DIR/abseil-cpp ./third_party/ + +mkdir build +cd build + +cmake -G "Ninja" \ + ${CMAKE_ARGS} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_C_COMPILER=$C_COMPILER \ + -DCMAKE_CXX_COMPILER=$CXX_COMPILER \ + -DCMAKE_INSTALL_PREFIX=$LIBPROTO_INSTALL \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_BUILD_LIBUPB=OFF \ + -Dprotobuf_BUILD_SHARED_LIBS=ON \ + -Dprotobuf_ABSL_PROVIDER="module" \ + -Dprotobuf_JSONCPP_PROVIDER="package" \ + -Dprotobuf_USE_EXTERNAL_GTEST=OFF \ + .. + +cmake --build . --verbose +cmake --install . +cd $CURRENT_DIR + +echo "------------ orc installing-------------------" + +git clone https://github.com/apache/orc +cd orc +git checkout v2.0.3 + + +wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o/orc/orc.patch +git apply orc.patch + +mkdir orc_prefix +export ORC_PREFIX=$(pwd)/orc_prefix + +mkdir -p build +cd build + +export PROTOBUF_PREFIX=$LIBPROTO_INSTALL +export CMAKE_PREFIX_PATH=$LIBPROTO_INSTALL +export LD_LIBRARY_PATH=$LIBPROTO_INSTALL/lib64 + +export GCC=$CC +export GXX=$CXX + +export HOST=$(uname)-$(uname -m) +CPPFLAGS="${CPPFLAGS} -Wl,-rpath,$VIRTUAL_ENV_PATH/**/lib" +declare -a _CMAKE_EXTRA_CONFIG +if [[ "$CONDA_BUILD_CROSS_COMPILATION" == 1 ]]; then + _CMAKE_EXTRA_CONFIG+=(-DHAS_PRE_1970_EXITCODE=0) + _CMAKE_EXTRA_CONFIG+=(-DHAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT=) + _CMAKE_EXTRA_CONFIG+=(-DHAS_POST_2038_EXITCODE=0) + _CMAKE_EXTRA_CONFIG+=(-DHAS_POST_2038_EXITCODE__TRYRUN_OUTPUT=) +fi +if [[ ${HOST} =~ .*darwin.* ]]; then + _CMAKE_EXTRA_CONFIG+=(-DCMAKE_AR=${AR}) + _CMAKE_EXTRA_CONFIG+=(-DCMAKE_RANLIB=${RANLIB}) + _CMAKE_EXTRA_CONFIG+=(-DCMAKE_LINKER=${LD}) +fi +if [[ ${HOST} =~ .*Linux.* ]]; then + CXXFLAGS="${CXXFLAGS//-std=c++17/-std=c++11}" + # I hate you so much CMake. + LIBPTHREAD=$(find ${PREFIX} -name "libpthread.so") + _CMAKE_EXTRA_CONFIG+=(-DPTHREAD_LIBRARY=${LIBPTHREAD}) +fi + +CPPFLAGS="${CPPFLAGS} -Wl,-rpath,$VIRTUAL_ENV_PATH/**/lib" + +cmake ${CMAKE_ARGS} \ + -DCMAKE_PREFIX_PATH=$ORC_PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_JAVA=False \ + -DLZ4_HOME=/usr \ + -DZLIB_HOME=/usr \ + -DZSTD_HOME=/usr/local \ + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW \ + -DProtobuf_ROOT=$PROTOBUF_PREFIX \ + -DPROTOBUF_HOME=$PROTOBUF_PREFIX \ + -DPROTOBUF_EXECUTABLE=$PROTOBUF_PREFIX/bin/protoc \ + -DSNAPPY_HOME=$SNAPPY_PREFIX \ + -DBUILD_LIBHDFSPP=NO \ + -DBUILD_CPP_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=$ORC_PREFIX \ + -DCMAKE_C_COMPILER=$(type -p ${CC}) \ + -DCMAKE_CXX_COMPILER=$(type -p ${CXX}) \ + -DCMAKE_C_FLAGS="$CFLAGS" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS -Wno-unused-parameter" \ + "${_CMAKE_EXTRA_CONFIG[@]}" \ + -GNinja .. + +ninja +ninja install + +cd $CURRENT_DIR + + +echo "-----------------boost_cpp installing-----------------------" + +git clone https://github.com/boostorg/boost +cd boost +git checkout boost-1.81.0 +git submodule update --init + +mkdir Boost_prefix +export BOOST_PREFIX=$(pwd)/Boost_prefix + +INCLUDE_PATH="${BOOST_PREFIX}/include" +LIBRARY_PATH="${BOOST_PREFIX}/lib" + +export target_platform=$(uname)-$(uname -m) +CXXFLAGS="${CXXFLAGS} -fPIC" +TOOLSET=gcc + + # http://www.boost.org/build/doc/html/bbv2/tasks/crosscompile.html +cat < tools/build/example/site-config.jam +using ${TOOLSET} : : ${CXX} ; +EOF + +LINKFLAGS="${LINKFLAGS} -L${LIBRARY_PATH}" + +CXXFLAGS="$(echo ${CXXFLAGS} | sed 's/ -march=[^ ]*//g' | sed 's/ -mcpu=[^ ]*//g' |sed 's/ -mtune=[^ ]*//g')" \ +CFLAGS="$(echo ${CFLAGS} | sed 's/ -march=[^ ]*//g' | sed 's/ -mcpu=[^ ]*//g' |sed 's/ -mtune=[^ ]*//g')" \ + CXX=${CXX_FOR_BUILD:-${CXX}} CC=${CC_FOR_BUILD:-${CC}} ./bootstrap.sh \ + --prefix="${BOOST_PREFIX}" \ + --without-libraries=python \ + --with-toolset=${TOOLSET} \ + --with-icu="${BOOST_PREFIX}" || (cat bootstrap.log; exit 1) + ADDRESS_MODEL=64 + ARCHITECTURE=power + ABI="sysv" + BINARY_FORMAT="elf" + + export CPU_COUNT=$(nproc) + +./b2 -q \ + variant=release \ + address-model="${ADDRESS_MODEL}" \ + architecture="${ARCHITECTURE}" \ + binary-format="${BINARY_FORMAT}" \ + abi="${ABI}" \ + debug-symbols=off \ + threading=multi \ + runtime-link=shared \ + link=shared \ + toolset=${TOOLSET} \ + include="${INCLUDE_PATH}" \ + cxxflags="${CXXFLAGS} -Wno-deprecated-declarations" \ + linkflags="${LINKFLAGS}" \ + --layout=system \ + -j"${CPU_COUNT}" \ + install + +# Remove Python headers as we don't build Boost.Python. +rm -rf "${BOOST_PREFIX}/include/boost/python.hpp" +rm -rf "${BOOST_PREFIX}/include/boost/python" +cd $SCRIPT_DIR + + + +echo "------------thrift_cpp installing-------------------" + +git clone https://github.com/apache/thrift +cd thrift +git checkout 0.21.0 + +Source_DIR=$(pwd) + +mkdir thrit-prefix +export THRIFT_PREFIX=$Source_DIR/thrit-prefix + +export BOOST_ROOT=${BOOST_PREFIX} +export ZLIB_ROOT=/usr +export LIBEVENT_ROOT=/usr + +export OPENSSL_ROOT=/usr +export OPENSSL_ROOT_DIR=/usr + +./bootstrap.sh +./configure --prefix=$THRIFT_PREFIX \ + --with-python=no \ + --with-py3=no \ + --with-ruby=no \ + --with-java=no \ + --with-kotlin=no \ + --with-erlang=no \ + --with-nodejs=no \ + --with-c_glib=no \ + --with-haxe=no \ + --with-rs=no \ + --with-cpp=yes \ + --with-PACKAGE=yes \ + --with-zlib=$ZLIB_ROOT \ + --with-libevent=$LIBEVENT_ROOT \ + --with-boost=$BOOST_ROOT \ + --with-openssl=$OPENSSL_ROOT \ + --enable-tests=no \ + --enable-tutorial=no + +make -j$(nproc) +make install +cd $CURRENT_DIR + + + + +echo "------------ grpc_cpp installing-------------------" + +git clone https://github.com/grpc/grpc +cd grpc +git checkout v1.68.0 + +git submodule update --init + +mkdir grpc-prefix +export GRPC_PREFIX=$(pwd)/grpc-prefix + +AR=`which ar` +RANLIB=`which ranlib` + +PROTOC_BIN=$LIBPROTO_INSTALL/bin/protoc +PROTOBUF_SRC=$LIBPROTO_INSTALL + +export CMAKE_PREFIX_PATH="$C_ARES_PREFIX;$RE2_PREFIX;$LIBPROTO_INSTALL" + +export LD_LIBRARY_PATH=$LIBPROTO_INSTALL/lib64:${LD_LIBRARY_PATH} + +target_platform=$(uname)-$(uname -m) + +if [[ "${target_platform}" == osx* ]]; then + export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_STANDARD=14" +else + export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_STANDARD=17" +fi + + +mkdir -p build-cpp +pushd build-cpp +cmake ${CMAKE_ARGS} .. \ + -GNinja \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$GRPC_PREFIX \ + -DgRPC_CARES_PROVIDER="package" \ + -DgRPC_GFLAGS_PROVIDER="package" \ + -DgRPC_PROTOBUF_PROVIDER="package" \ + -DProtobuf_ROOT=$PROTOBUF_SRC \ + -DgRPC_SSL_PROVIDER="package" \ + -DgRPC_ZLIB_PROVIDER="package" \ + -DgRPC_ABSL_PROVIDER="package" \ + -DgRPC_RE2_PROVIDER="package" \ + -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \ + -DCMAKE_AR=${AR} \ + -DCMAKE_RANLIB=${RANLIB} \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DProtobuf_PROTOC_EXECUTABLE=$PROTOC_BIN + +ninja install -v +popd + +cd $CURRENT_DIR + + +echo "---------------------openblas installing---------------------" + +git clone https://github.com/OpenMathLib/OpenBLAS +cd OpenBLAS +git checkout v0.3.29 +git submodule update --init +PREFIX=local/openblas +# Set build options +declare -a build_opts +# Fix ctest not automatically discovering tests +LDFLAGS=$(echo "${LDFLAGS}" | sed "s/-Wl,--gc-sections//g") +export CF="${CFLAGS} -Wno-unused-parameter -Wno-old-style-declaration" +unset CFLAGS +export USE_OPENMP=1 +build_opts+=(USE_OPENMP=${USE_OPENMP}) +export PREFIX=${PREFIX} + +# Handle Fortran flags +if [ ! -z "$FFLAGS" ]; then + export FFLAGS="${FFLAGS/-fopenmp/ }" + export FFLAGS="${FFLAGS} -frecursive" + export LAPACK_FFLAGS="${FFLAGS}" +fi +export PLATFORM=$(uname -m) +build_opts+=(BINARY="64") +build_opts+=(DYNAMIC_ARCH=1) +build_opts+=(TARGET="POWER9") +BUILD_BFLOAT16=1 + +# Placeholder for future builds that may include ILP64 variants. +build_opts+=(INTERFACE64=0) +build_opts+=(SYMBOLSUFFIX="") + +# Build LAPACK +build_opts+=(NO_LAPACK=0) + +# Enable threading and set the number of threads +build_opts+=(USE_THREAD=1) +build_opts+=(NUM_THREADS=8) + +# Disable CPU/memory affinity handling to avoid problems with NumPy and R +build_opts+=(NO_AFFINITY=1) + +# Build OpenBLAS +make -j8 ${build_opts[@]} CFLAGS="${CF}" FFLAGS="${FFLAGS}" prefix=${PREFIX} + +# Install OpenBLAS +CFLAGS="${CF}" FFLAGS="${FFLAGS}" make install PREFIX="${PREFIX}" ${build_opts[@]} +OpenBLASInstallPATH=$(pwd)/$PREFIX +OpenBLASConfigFile=$(find . -name OpenBLASConfig.cmake) +OpenBLASPCFile=$(find . -name openblas.pc) +sed -i "/OpenBLAS_INCLUDE_DIRS/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" ${OpenBLASConfigFile} +sed -i "/OpenBLAS_LIBRARIES/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" ${OpenBLASConfigFile} +sed -i "s|libdir=local/openblas/lib|libdir=${OpenBLASInstallPATH}/lib|" ${OpenBLASPCFile} +sed -i "s|includedir=local/openblas/include|includedir=${OpenBLASInstallPATH}/include|" ${OpenBLASPCFile} +export LD_LIBRARY_PATH="$OpenBLASInstallPATH/lib" +export PKG_CONFIG_PATH="$OpenBLASInstallPATH/lib/pkgconfig:${PKG_CONFIG_PATH}" + +cd $CURRENT_DIR +echo "------------openblas installed--------------------" + +echo "Installing NumPy" +python3.12 -m pip install numpy==2.0.2 + +echo "-----------------installing pyarrow----------------------" + +#cloning pyarrow + +git clone https://github.com/apache/arrow +cd arrow +git checkout apache-arrow-19.0.0 +git submodule update --init + +mkdir pyarrow_prefix +export PYARROW_PREFIX=$(pwd)/pyarrow_prefix + + +export ARROW_HOME=$PYARROW_PREFIX +export target_platform=$(uname)-$(uname -m) +export CMAKE_PREFIX_PATH=$C_ARES_PREFIX:$LIBPROTO_INSTALL:$RE2_PREFIX:$GRPC_PREFIX:$ORC_PREFIX:$BOOST_PREFIX:${UTF8PROC_PREFIX}:$THRIFT_PREFIX:$SNAPPY_PREFIX:/usr +export LD_LIBRARY_PATH=$GRPC_PREFIX/lib:$LIBPROTO_INSTALL/lib64 + +mkdir cpp/build +pushd cpp/build + +EXTRA_CMAKE_ARGS="" + +# Include g++'s system headers +if [ "$(uname)" == "Linux" ]; then + SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') + EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" + sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake +fi + +# Enable CUDA support +if [ "${build_type}" = "cuda" ]; then + if [[ -z "${CUDA_HOME+x}" ]] + then + echo "cuda version=${cudatoolkit} CUDA_HOME=$CUDA_HOME" + CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) + if [[ -n "$CUDA_GDB_EXECUTABLE" ]] + then + CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) + else + echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" + return 1 + fi + fi + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" +else + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" +fi +# Disable Gandiva +EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" + +export BOOST_ROOT="${BOOST_PREFIX}" +export Boost_ROOT="${BOOST_PREFIX}" + +export CXXFLAGS="-I$${BOOST_PREFIX}/include -I${THRIFT_PREFIX}/include" + +#SIMD Settings +if [[ "${target_platform}" == "Linux-x86_64" ]]; then + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_SIMD_LEVEL=SSE4_2" +fi +if [[ "${target_platform}" == "Linux-ppc64le" ]]; then + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_ALTIVEC=ON" +fi +if [[ "${target_platform}" != "Linux-s390x" ]]; then + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_USE_LD_GOLD=ON" +fi + +cmake \ + -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \ + -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -DARROW_FLIGHT=ON \ + -DARROW_HDFS=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_MIMALLOC=ON \ + -DARROW_ORC=ON \ + -DARROW_PACKAGE_PREFIX=$PYARROW_PREFIX \ + -DARROW_PARQUET=ON \ + -DARROW_PYTHON=ON \ + -DARROW_S3=OFF \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_WITH_THRIFT=ON \ + -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DPYTHON_EXECUTABLE=python \ + -DPython3_EXECUTABLE=python \ + -DProtobuf_PROTOC_EXECUTABLE=${LIBPROTO_INSTALL}/bin/protoc \ + -DORC_INCLUDE_DIR=${ORC_PREFIX}/include \ + -DgRPC_DIR=${GRPC_PREFIX} \ + -DBoost_DIR=${BOOST_PREFIX} \ + -DBoost_INCLUDE_DIR=${BOOST_PREFIX}/include/ \ + -Dutf8proc_LIB=${UTF8PROC_PREFIX}/lib/libutf8proc.so ${UTF8PROC_PREFIX}/lib/libutf8proc.so.2 ${UTF8PROC_PREFIX}/lib/libutf8proc.so.2.4.1 \ + -Dutf8proc_INCLUDE_DIR=${UTF8PROC_PREFIX}/include \ + -DCMAKE_AR=${AR} \ + -DCMAKE_RANLIB=${RANLIB} \ + -GNinja \ + ${EXTRA_CMAKE_ARGS} \ + .. + +ninja install +popd + +cd $CURRENT_DIR +export PYARROW_BUNDLE_ARROW_CPP=1 +export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} + +export build_type=cpu +cd arrow + +export CMAKE_PREFIX_PATH=$ARROW_HOME + +# Build dependencies +export PARQUET_HOME=$ARROW_HOME +export SETUPTOOLS_SCM_PRETEND_VERSION=19.0.0 +export PYARROW_BUILD_TYPE=release +export PYARROW_BUNDLE_ARROW_CPP_HEADERS=1 +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 +# Disable Gandiva +export PYARROW_WITH_GANDIVA=0 +export PYARROW_WITH_HDFS=1 +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_WITH_S3=0 +export PYARROW_CMAKE_GENERATOR=Ninja +BUILD_EXT_FLAGS="" + +# Enable CUDA support +if [ "${build_type}" = "cuda" ]; then + export PYARROW_WITH_CUDA=1 +else + export PYARROW_WITH_CUDA=0 +fi + +cd python + +pip3.12 install . + +echo "-----------------pyarrow installed----------------------" +python3.12 -m pip install wheel setuptools + +python3.12 -m pip install sentencepiece==0.1.97 parameterized datasets==4.0.0 setuptools wheel pytest timeout_decorator evaluate GitPython ruff psutil packaging pyyaml +python3.12 -m pip install --prefer-binary llvmlite numpy scipy pillow numba==0.62.0.dev0 c-ares thrift-cpp snappy libprotobuf orc utf8proc re2 openblas grpcio ml-dtypes h5py hdf5==1.12.3 torch==2.6.0 --extra-index-url=https://wheels.developerfirst.ibm.com/ppc64le/linux + +python3.12 -m pip install accelerate>=1.12.0 + +export VENV_SITE="$VENV_DIR/lib/python$PYTHON_VERSION/site-packages" +export LD_LIBRARY_PATH="$VENV_SITE/hdf5/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/snappy/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/libprotobuf/lib64:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/thriftcpp/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/orc/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/utf8proc/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/re2/lib:$LD_LIBRARY_PATH" +export LD_LIBRARY_PATH="$VENV_SITE/openblas/lib:$LD_LIBRARY_PATH" +echo "-----------------cloning transformers package---------------------" + +cd $CURRENT_DIR +git clone $PACKAGE_URL +cd $PACKAGE_DIR +git checkout $PACKAGE_VERSION +SRC_DIR=$(pwd) +wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/t/transformers/transformers_4.52.4.patch +git apply transformers_4.52.4.patch +python3.12 setup.py bdist_wheel --dist-dir $CURRENT_DIR --plat-name=linux_$(uname -m) +#Install +if ! (python3.12 -m pip install .) ; then + echo "------------------$PACKAGE_NAME:Install_fails-------------------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_Fails" + exit 1 +fi + +#pytest skipping test that are in parity with x86, also skipping jax/jaxlib related tests + +if ! pytest --ignore=templates --ignore=examples --ignore=docs --ignore=i18n --ignore=tests/jax --ignore=tests/utils --ignore=tests/pipelines --ignore=tests/models --ignore=tests/extended --ignore=tests/trainer --ignore=tests/generation --ignore=tests/tokenization --ignore=tests/tensor_parallel --ignore=tests/test_pipeline_mixin.py --ignore=tests/repo_utils/modular -k "not test_evaluation_with_keys_to_drop" -q --disable-warnings +then + echo "------------------$PACKAGE_NAME:Install_success_but_test_fails---------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_success_but_test_Fails" + exit 2 +else + echo "------------------$PACKAGE_NAME:Install_&_test_both_success-------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Pass | Both_Install_and_Test_Success" + exit 0 +fi +