diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index 361edeb2c..bac7141a9 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -63,6 +63,25 @@ if [[ "${sha:-}" == "" ]]; then sha=$(git rev-parse HEAD) fi +if [[ "${OSX_SDK_DIR:-}" == "" ]]; then + if [[ "${CI:-}" == "" ]]; then + echo "Please set OSX_SDK_DIR to a directory where SDKs can be downloaded to. Aborting" + exit 1 + else + export OSX_SDK_DIR=/opt/conda-sdks + /usr/bin/sudo mkdir -p "${OSX_SDK_DIR}" + /usr/bin/sudo chown "${USER}" "${OSX_SDK_DIR}" + fi +else + if tmpf=$(mktemp -p "$OSX_SDK_DIR" tmp.XXXXXXXX 2>/dev/null); then + rm -f "$tmpf" + echo "OSX_SDK_DIR is writeable without sudo, continuing" + else + echo "User-provided OSX_SDK_DIR is not writeable for current user! Aborting" + exit 1 + fi +fi + echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup diff --git a/README.md b/README.md index 739a20c89..c190c2a10 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ Current release info | Name | Downloads | Version | Platforms | | --- | --- | --- | --- | | [![Conda Recipe](https://img.shields.io/badge/recipe-cpython-green.svg)](https://anaconda.org/conda-forge/cpython) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/cpython.svg)](https://anaconda.org/conda-forge/cpython) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/cpython.svg)](https://anaconda.org/conda-forge/cpython) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/cpython.svg)](https://anaconda.org/conda-forge/cpython) | +| [![Conda Recipe](https://img.shields.io/badge/recipe-libpython-green.svg)](https://anaconda.org/conda-forge/libpython) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libpython.svg)](https://anaconda.org/conda-forge/libpython) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libpython.svg)](https://anaconda.org/conda-forge/libpython) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libpython.svg)](https://anaconda.org/conda-forge/libpython) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libpython--static-green.svg)](https://anaconda.org/conda-forge/libpython-static) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libpython-static.svg)](https://anaconda.org/conda-forge/libpython-static) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libpython-static.svg)](https://anaconda.org/conda-forge/libpython-static) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libpython-static.svg)](https://anaconda.org/conda-forge/libpython-static) | | [![Conda Recipe](https://img.shields.io/badge/recipe-python-green.svg)](https://anaconda.org/conda-forge/python) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/python.svg)](https://anaconda.org/conda-forge/python) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/python.svg)](https://anaconda.org/conda-forge/python) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/python.svg)](https://anaconda.org/conda-forge/python) | | [![Conda Recipe](https://img.shields.io/badge/recipe-python--freethreading-green.svg)](https://anaconda.org/conda-forge/python-freethreading) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/python-freethreading.svg)](https://anaconda.org/conda-forge/python-freethreading) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/python-freethreading.svg)](https://anaconda.org/conda-forge/python-freethreading) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/python-freethreading.svg)](https://anaconda.org/conda-forge/python-freethreading) | @@ -235,48 +236,48 @@ Current release info Installing python ================= -Installing `python` from the `conda-forge/label/python_dev` channel can be achieved by adding `conda-forge/label/python_dev` to your channels with: +Installing `python` from the `conda-forge/label/python_dev_debug` channel can be achieved by adding `conda-forge/label/python_dev_debug` to your channels with: ``` -conda config --add channels conda-forge/label/python_dev +conda config --add channels conda-forge/label/python_dev_debug conda config --set channel_priority strict ``` -Once the `conda-forge/label/python_dev` channel has been enabled, `cpython, libpython-static, python, python-freethreading, python-gil, python-jit` can be installed with `conda`: +Once the `conda-forge/label/python_dev_debug` channel has been enabled, `cpython, libpython, libpython-static, python, python-freethreading, python-gil, python-jit` can be installed with `conda`: ``` -conda install cpython libpython-static python python-freethreading python-gil python-jit +conda install cpython libpython libpython-static python python-freethreading python-gil python-jit ``` or with `mamba`: ``` -mamba install cpython libpython-static python python-freethreading python-gil python-jit +mamba install cpython libpython libpython-static python python-freethreading python-gil python-jit ``` It is possible to list all of the versions of `cpython` available on your platform with `conda`: ``` -conda search cpython --channel conda-forge/label/python_dev +conda search cpython --channel conda-forge/label/python_dev_debug ``` or with `mamba`: ``` -mamba search cpython --channel conda-forge/label/python_dev +mamba search cpython --channel conda-forge/label/python_dev_debug ``` Alternatively, `mamba repoquery` may provide more information: ``` # Search all versions available on your platform: -mamba repoquery search cpython --channel conda-forge/label/python_dev +mamba repoquery search cpython --channel conda-forge/label/python_dev_debug # List packages depending on `cpython`: -mamba repoquery whoneeds cpython --channel conda-forge/label/python_dev +mamba repoquery whoneeds cpython --channel conda-forge/label/python_dev_debug # List dependencies of `cpython`: -mamba repoquery depends cpython --channel conda-forge/label/python_dev +mamba repoquery depends cpython --channel conda-forge/label/python_dev_debug ``` diff --git a/recipe/build_base.sh b/recipe/build_base.sh index a376602ca..bf36540ab 100644 --- a/recipe/build_base.sh +++ b/recipe/build_base.sh @@ -385,20 +385,16 @@ if [[ ${_OPTIMIZED} == yes ]]; then _FLAGS_REPLACE+=("") done fi -# Install the shared library (for people who embed Python only, e.g. GDB). -# Linking module extensions to this on Linux is redundant (but harmless). -# Linking module extensions to this on Darwin is harmful (multiply defined symbols). -shopt -s extglob -cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/ -shopt -u extglob -if [[ ${target_platform} =~ .*linux.* ]]; then - ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} -fi -SYSCONFIG=$(find ${_buildd_static}/$(cat ${_buildd_static}/pybuilddir.txt) -name "_sysconfigdata*.py" -print0) +# Use sysconfigdata and build-details.json from the shared build, as we want packages to prefer +# linking against the shared library. Issue #565. +BUILD_DIR=$(< ${_buildd_shared}/pybuilddir.txt) +SYSCONFIG=$(find ${_buildd_shared}/${BUILD_DIR} -name "_sysconfigdata*.py" -print0) cat ${SYSCONFIG} | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \ "${_FLAGS_REPLACE[@]}" \ > ${PREFIX}/lib/python${VERABI_NO_DBG}/$(basename ${SYSCONFIG}) +BUILD_DETAILS=${_buildd_shared}/${BUILD_DIR}/build-details.json +cp ${BUILD_DETAILS} ${PREFIX}/lib/python${VERABI_NO_DBG}/ MAKEFILE=$(find ${PREFIX}/lib/python${VERABI_NO_DBG}/ -path "*config-*/Makefile" -print0) cp ${MAKEFILE} /tmp/Makefile-$$ cat /tmp/Makefile-$$ | ${SYS_PYTHON} "${RECIPE_DIR}"/replace-word-pairs.py \ @@ -515,7 +511,6 @@ rm ${PREFIX}/lib/libpython${VERABI}.a if [[ ${PY_INTERP_DEBUG} == yes ]]; then rm ${PREFIX}/bin/python${VER} ln -s ${PREFIX}/bin/python${VERABI} ${PREFIX}/bin/python${VER} - ln -s ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} ${PREFIX}/lib/libpython${VERABI_NO_DBG}${SHLIB_EXT} ln -s ${PREFIX}/include/python${VERABI} ${PREFIX}/include/python${VER} fi diff --git a/recipe/build_shared.bat b/recipe/build_shared.bat new file mode 100644 index 000000000..e69de29bb diff --git a/recipe/build_shared.sh b/recipe/build_shared.sh new file mode 100644 index 000000000..35812a5d1 --- /dev/null +++ b/recipe/build_shared.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -ex + +_buildd_shared=build-shared +if [[ ${PY_INTERP_DEBUG} == yes ]]; then + DBG=d +else + DBG= +fi +if [[ ${PY_FREETHREADING} == yes ]]; then + # This Python will not be usable with non-free threading Python modules. + THREAD=t +else + THREAD= +fi + +VER=${PKG_VERSION%.*} +ABIFLAGS=${DBG}${THREAD} +VERABI=${VER}${THREAD}${DBG} +VERABI_NO_DBG=${VER}${THREAD} + +# Install the shared library (for people who embed Python only, e.g. GDB). +# Linking module extensions to this on Linux is redundant (but harmless). +# Linking module extensions to this on Darwin is harmful (multiply defined symbols). +shopt -s extglob +cp -pf ${_buildd_shared}/libpython*${SHLIB_EXT}!(.lto) ${PREFIX}/lib/ +shopt -u extglob +if [[ ${target_platform} =~ .*linux.* ]]; then + ln -sf ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT}.1.0 ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} +fi +if [[ ${PY_INTERP_DEBUG} == yes ]]; then + ln -s ${PREFIX}/lib/libpython${VERABI}${SHLIB_EXT} ${PREFIX}/lib/libpython${VERABI_NO_DBG}${SHLIB_EXT} +fi diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 7c6069da7..6266f8cdb 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -4,7 +4,7 @@ {% set ver2 = '.'.join(version.split('.')[0:2]) %} {% set ver2nd = ''.join(version.split('.')[0:2]) %} {% set ver3nd = ''.join(version.split('.')[0:3]) %} -{% set build_number = 0 %} +{% set build_number = 1 %} # this makes the linter happy {% set channel_targets = channel_targets or 'conda-forge main' %} @@ -221,7 +221,6 @@ outputs: files: - tests/cmake/* - tests/cython/* - - tests/prefix-replacement/* - run_test.py commands: - echo on # [win] @@ -247,8 +246,11 @@ outputs: - _CONDA_PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_x86_64_conda_linux_gnu python -c "import sysconfig; print(sysconfig.get_config_var('CC'))" # [linux64] # check for unreplaced @ symbols in sysconfig files, excluding e.g. '"$@".lto' - for f in ${CONDA_PREFIX}/lib/python*/_sysconfig*.py; do echo "Checking $f:"; if [[ `rg @[^@]*@ $f` ]]; then echo "FAILED ON $f"; cat $f; exit 1; fi; done # [linux64 or osx] - - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.a # [unix] - - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.nolto.a # [unix] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so # [unix] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.a # [unix] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.so # [linux] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.dylib # [osx] + - test ! -f ${PREFIX}/lib/libpython${PKG_VERSION%.*}.nolto.a # [unix] {% if freethreading == "yes" %} - if not exist %PREFIX%\\libs\\python3t.lib exit 1 # [win] - if not exist %PREFIX%\\libs\\python{{ ver2nd }}t.lib exit 1 # [win] @@ -265,14 +267,8 @@ outputs: - if not exist %PREFIX%\\Scripts\\pydoc.exe exit 1 # [win] - if not exist %PREFIX%\\include\\pyconfig.h exit 1 # [win] - pushd tests - - pushd prefix-replacement # [unix] - - bash build-and-test.sh # [unix] - - popd # [unix] - pushd cmake -{% if freethreading == "no" %} - # TODO: use the new FindPython3 which supports freethreading - - cmake -GNinja -DPY_VER={{ version }} --debug-find --trace --debug-output --debug-trycompile . -{% endif %} + - cmake -GNinja -DPY_VER={{ version }} -DEXTRA_COMPONENTS="Development.Module" --debug-find --trace --debug-output --debug-trycompile . - popd - popd - python run_test.py @@ -283,6 +279,61 @@ outputs: # Test for wide character supported via ncursesw - TERM=xterm >/dev/null python -c "import curses; scr = curses.initscr(); curses.unget_wch('x'); assert 'x' == scr.get_wch()" # [unix] + - name: libpython + script: build_shared.sh # [unix] + script: build_shared.bat # [win] + build: + number: {{ build_number }} + activate_in_script: true + ignore_run_exports: + - python_abi + script_env: + - PY_INTERP_DEBUG={{ py_interp_debug }} + - PY_FREETHREADING={{ freethreading }} + string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ debug }}_{{ abi_tag }} + requirements: + build: + - {{ compiler('c') }} + - {{ stdlib('c') }} + - {{ compiler('cxx') }} + host: + - {{ pin_subpackage('python', exact=True) }} + run: + - {{ pin_subpackage('python', exact=True) }} + test: + files: + - tests/cmake/* + - tests/prefix-replacement/* + requires: + - {{ stdlib('c') }} + - {{ compiler('c') }} + # cmake expects a C++ compiler for some reason + - {{ compiler('cxx') }} + - ripgrep + - cmake + - ninja + commands: + - VER=${PKG_VERSION%.*} # [not win] + - VERABI=${VER} # [not win] +{% if freethreading == "yes" %} + - VERABI=${VERABI}t # [not win] +{% endif %} + - VERABI_NO_DBG=${VERABI} # [not win] +{% if py_interp_debug == "yes" %} + - VERABI=${VERABI}d # [not win] +{% endif %} + - test -f ${PREFIX}/lib/libpython${VERABI}.so # [linux] + - test -f ${PREFIX}/lib/libpython3.so # [linux and build_type == "release"] + - test -f ${PREFIX}/lib/libpython${VERABI}.dylib # [osx] + - pushd tests + - pushd prefix-replacement # [unix] + - bash build-and-test.sh # [unix] + - popd # [unix] + - pushd cmake + - cmake -GNinja -DPY_VER={{ version }} -DEXTRA_COMPONENTS="Development.Embed" --debug-find --trace --debug-output --debug-trycompile . + - popd + - popd + - name: libpython-static script: build_static.sh # [unix] script: build_static.bat # [win] @@ -306,10 +357,16 @@ outputs: - {{ pin_subpackage('python', exact=True) }} test: files: + - tests/cmake/* - tests/prefix-replacement/* requires: - {{ stdlib('c') }} - {{ compiler('c') }} + # cmake expects a C++ compiler for some reason + - {{ compiler('cxx') }} + - ripgrep + - cmake + - ninja commands: - VER=${PKG_VERSION%.*} # [not win] - VERABI=${VER} # [not win] @@ -323,9 +380,14 @@ outputs: - test -f ${PREFIX}/lib/libpython${VERABI}.a # [unix] - test -f ${PREFIX}/lib/libpython${VERABI_NO_DBG}.nolto.a # [unix] - test -f ${PREFIX}/lib/python${VERABI_NO_DBG}/config-${VERABI}-darwin/libpython${VERABI}.a # [osx] - - pushd tests/prefix-replacement # [unix] - - bash build-and-test.sh # [unix] - - popd # [unix] + - pushd tests + - pushd prefix-replacement # [unix] + - bash build-and-test.sh # [unix] + - popd # [unix] + - pushd cmake + - cmake -GNinja -DPY_VER={{ version }} -DEXTRA_COMPONENTS="Development.Embed" --debug-find --trace --debug-output --debug-trycompile . + - popd + - popd - name: cpython build: diff --git a/recipe/tests/cmake/CMakeLists.txt b/recipe/tests/cmake/CMakeLists.txt index a13e20ae1..f3e41560c 100644 --- a/recipe/tests/cmake/CMakeLists.txt +++ b/recipe/tests/cmake/CMakeLists.txt @@ -1,16 +1,10 @@ -# https://martinopilia.com/posts/2018/09/15/building-python-extension.html -# FindPythonInterp and FindPythonLibs are deprecated since cmake 3.12 -# These can be replaced by find_package(Python ${PY_VER} REQUIRED) -# But these are still used by other packages, so we keep them. +# https://scikit-build-core.readthedocs.io/en/latest/guide/cmakelists.html#finding-python cmake_minimum_required(VERSION 3.10) enable_language(C) project(mymath) -option(PY_VER, "Python version to use") +option(PY_VER "Python version to use") +option(EXTRA_COMPONENTS "Additional Python components to check for") -find_package(PythonInterp ${PY_VER} REQUIRED) -# PATHS $ENV{CONDA_PREFIX} - -# This goes after, since it uses PythonInterp as hint -find_package(PythonLibs ${PY_VER} REQUIRED) +find_package(Python ${PY_VER} COMPONENTS Interpreter ${EXTRA_COMPONENTS} REQUIRED) diff --git a/recipe/tests/prefix-replacement/build-and-test.sh b/recipe/tests/prefix-replacement/build-and-test.sh index 633b741a3..d710bde75 100644 --- a/recipe/tests/prefix-replacement/build-and-test.sh +++ b/recipe/tests/prefix-replacement/build-and-test.sh @@ -2,46 +2,47 @@ set -ex -if [[ "$PKG_NAME" == "libpython-static" ]]; then - # see bpo44182 for why -L${CONDA_PREFIX}/lib is added - ${CC} a.c $(python3-config --cflags) $(python3-config --embed --ldflags) -L${CONDA_PREFIX}/lib -o ${CONDA_PREFIX}/bin/embedded-python-static - if [[ "$target_platform" == linux-* ]]; then - if ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then - echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." - fi - elif [[ "$target_platform" == osx-* ]]; then - if ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then - echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." - fi - fi - ${CONDA_PREFIX}/bin/embedded-python-static - - # I thought this would prefer the shared library for Python. I was wrong: - # EMBED_LDFLAGS=$(python3-config --ldflags) - # re='^(.*)(-lpython[^ ]*)(.*)$' - # if [[ ${EMBED_LDFLAGS} =~ $re ]]; then - # EMBED_LDFLAGS="${BASH_REMATCH[1]} ${BASH_REMATCH[3]} -Wl,-Bdynamic ${BASH_REMATCH[2]}" - # fi - # ${CC} a.c $(python3-config --cflags) ${EMBED_LDFLAGS} -o ${CONDA_PREFIX}/bin/embedded-python-shared +case ${PKG_NAME} in + libpython) + # see bpo44182 for why -L${CONDA_PREFIX}/lib is added + ${CC} a.c $(python3-config --cflags) \ + $(python3-config --embed --ldflags) \ + -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ + -o ${CONDA_PREFIX}/bin/embedded-python-shared - # Brute-force way of linking to the shared library, sorry! - rm -rf ${CONDA_PREFIX}/lib/libpython*.a -fi + if [[ "$target_platform" == linux-* ]]; then + if ! ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then + echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." + fi + elif [[ "$target_platform" == osx-* ]]; then + if ! ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then + echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." + fi + fi + ${CONDA_PREFIX}/bin/embedded-python-shared + ;; -${CC} a.c $(python3-config --cflags) \ - $(python3-config --embed --ldflags) \ - -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ - -o ${CONDA_PREFIX}/bin/embedded-python-shared + libpython-static) + ${CC} a.c $(python3-config --cflags) \ + $(python3-config --embed --ldflags) \ + -L${CONDA_PREFIX}/lib -Wl,-rpath,${CONDA_PREFIX}/lib \ + -o ${CONDA_PREFIX}/bin/embedded-python-static + if [[ "$target_platform" == linux-* ]]; then + if ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then + echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." + fi + elif [[ "$target_platform" == osx-* ]]; then + if ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-static | rg libpython; then + echo "ERROR :: Embedded python linked to shared python library. It is expected to link to the static library." + fi + fi + ${CONDA_PREFIX}/bin/embedded-python-static + ;; -if [[ "$target_platform" == linux-* ]]; then - if ! ${READELF} -d ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then - echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." - fi -elif [[ "$target_platform" == osx-* ]]; then - if ! ${OTOOL} -l ${CONDA_PREFIX}/bin/embedded-python-shared | rg libpython; then - echo "ERROR :: Embedded python linked to static python library. We tried to force it to use the shared library." - fi -fi -${CONDA_PREFIX}/bin/embedded-python-shared + *) + # invalid package + exit 1 + ;; +esac set +x