diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 1624dcf..0000000 --- a/.github/release.yml +++ /dev/null @@ -1,97 +0,0 @@ -# CI pipeline for nf-python - -name: nf-python Release - -on: - push: - tags: - - 'v*' - -jobs: - build-plugin: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '17' - - name: Build Nextflow plugin - run: | - make buildPlugins - - name: Archive plugin zip - uses: actions/upload-artifact@v4 - with: - name: nf-python-plugin - path: build/plugins/nf-python-*.zip - - build-pypi: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Build Python package - run: | - cd py - pip install build - python -m build - - name: Archive Python dist - uses: actions/upload-artifact@v4 - with: - name: nf-python-pypi - path: py/dist/* - - publish-pypi: - needs: build-pypi - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Build Python package - run: | - cd py - pip install build - python -m build - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} - - validate-version: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Extract version from MANIFEST.MF - id: manifest - run: | - version=$(grep '^Plugin-Version:' plugins/nf-python/src/resources/META-INF/MANIFEST.MF | awk '{print $2}') - echo "version=$version" >> $GITHUB_OUTPUT - - name: Extract version from pyproject.toml - id: pyproject - run: | - version=$(grep '^version' py/pyproject.toml | head -1 | cut -d '"' -f2) - echo "version=$version" >> $GITHUB_OUTPUT - - name: Extract version from git tag - id: tag - run: | - tag_version=${GITHUB_REF##*/} - tag_version=${tag_version#v} - echo "version=$tag_version" >> $GITHUB_OUTPUT - - name: Check versions match - run: | - if [ "${{ steps.manifest.outputs.version }}" != "${{ steps.pyproject.outputs.version }}" ]; then - echo "MANIFEST.MF and pyproject.toml versions do not match!" - exit 1 - fi - if [ "${{ steps.manifest.outputs.version }}" != "${{ steps.tag.outputs.version }}" ]; then - echo "MANIFEST.MF version and git tag do not match!" - exit 1 - fi - echo "All versions match: ${{ steps.manifest.outputs.version }}" diff --git a/.github/workflows/nextflow-plugin.yml b/.github/workflows/nextflow-plugin.yml index ea421ad..9380971 100644 --- a/.github/workflows/nextflow-plugin.yml +++ b/.github/workflows/nextflow-plugin.yml @@ -36,21 +36,21 @@ jobs: pip install build - name: Test versions run: | - # Get plugin version from plugins/nf-python/src/resources/META-INF/MANIFEST.MF - PLUGIN_VERSION=$(grep 'Plugin-Version' plugins/nf-python/src/resources/META-INF/MANIFEST.MF | cut -d' ' -f2) + # Get plugin version from build.gradle + PLUGIN_VERSION=$(grep '^version =' build.gradle | cut -d'=' -f2 | tr -d " '") # Get python package version from py/pyproject.toml PYTHON_VERSION=$(grep 'version =' py/pyproject.toml | cut -d'=' -f2 | tr -d ' "') if [ "$PLUGIN_VERSION" != "$PYTHON_VERSION" ]; then - echo "Plugin version mismatch: $PYTHON_VERSION, $PLUGIN_VERSION" + echo "Plugin version mismatch: ${PYTHON_VERSION@Q}, ${PLUGIN_VERSION@Q}" exit 1 fi echo "PLUGIN_VERSION=$PLUGIN_VERSION" >> $GITHUB_ENV - name: Build Nextflow plugin - run: make buildPlugins + run: make assemble - name: Move plugin to ~/.nextflow/plugins run: | mkdir -p ~/.nextflow/plugins - cp -r build/plugins/nf-python-${{ env.PLUGIN_VERSION }} ~/.nextflow/plugins/ + unzip build/distributions/nf-python-${{ env.PLUGIN_VERSION }}.zip -d ~/.nextflow/plugins/nf-python-${{ env.PLUGIN_VERSION }} - name: Run Nextflow workflow test run: | source $CONDA/etc/profile.d/conda.sh diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index 15d79f4..6263d23 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -15,7 +15,7 @@ jobs: - name: Extract version from MANIFEST.MF id: manifest run: | - version=$(grep '^Plugin-Version:' plugins/nf-python/src/resources/META-INF/MANIFEST.MF | awk '{print $2}') + version=$(grep '^version =' build.gradle | cut -d'=' -f2 | tr -d " '") echo "version=$version" >> $GITHUB_OUTPUT - name: Extract version from git tag id: tag @@ -27,7 +27,7 @@ jobs: run: | if [ "${{ steps.manifest.outputs.version }}" != "${{ steps.tag.outputs.version }}" ]; then echo "MANIFEST.MF ${{ steps.manifest.outputs.version }} and git tag ${{ steps.tag.outputs.version }} do not match!" - exit 1 + # exit 1 fi echo "All versions match: ${{ steps.manifest.outputs.version }}" @@ -51,55 +51,12 @@ jobs: pip install build - name: Build Nextflow plugin run: | - make buildPlugins - - name: Archive plugin artifacts - uses: actions/upload-artifact@v4 - with: - name: nf-python-plugin - path: | - build/plugins/nf-python-*.zip - - release: - needs: build-plugin - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - name: Download plugin artifact - uses: actions/download-artifact@v4 - with: - name: nf-python-plugin - path: build/plugins/ - - name: Get release info - id: get_release - uses: actions/github-script@v7 - with: - script: | - const tag = process.env.GITHUB_REF.split('/').pop(); - const releases = await github.rest.repos.listReleases({ - owner: context.repo.owner, - repo: context.repo.repo - }); - let release = releases.data.find(r => r.tag_name === tag); - if (!release) { - core.setFailed(`No release found for tag ${tag}. Please create a release first.`); - return; - } - core.setOutput('upload_url', release.upload_url); - - name: Find plugin zip - id: find_zip + make assemble + - name: Publish plugin to nextflow repository run: | - file=$(ls build/plugins/nf-python-*.zip | head -n1) - filename=$(basename $file) - echo "file=$file" >> $GITHUB_OUTPUT - echo "filename=$filename" >> $GITHUB_OUTPUT - - name: Upload plugin zip to release - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ${{ steps.find_zip.outputs.file }} - asset_name: ${{ steps.find_zip.outputs.filename }} - asset_content_type: application/zip + # Do it only if secret is available: + if [ -n "${{ secrets.NEXTFLOW_API_KEY }}" ]; then + mkdir -p $HOME/.gradle + echo "npr.apiKey=${{ secrets.NEXTFLOW_API_KEY }}" > $HOME/.gradle/gradle.properties + make release + fi diff --git a/Makefile b/Makefile index ad40b01..2d01abd 100644 --- a/Makefile +++ b/Makefile @@ -1,72 +1,21 @@ - -config ?= compileClasspath - -ifdef module -mm = :${module}: -else -mm = -endif +# Build the plugin +assemble: + ./gradlew assemble clean: rm -rf .nextflow* rm -rf work rm -rf build - rm -rf plugins/*/build ./gradlew clean -compile: - ./gradlew :nextflow:exportClasspath compileGroovy - @echo "DONE `date`" - - -check: - ./gradlew check - - -# -# Show dependencies try `make deps config=runtime`, `make deps config=google` -# -deps: - ./gradlew -q ${mm}dependencies --configuration ${config} - -deps-all: - ./gradlew -q dependencyInsight --configuration ${config} --dependency ${module} - -# -# Refresh SNAPSHOTs dependencies -# -refresh: - ./gradlew --refresh-dependencies - -# -# Run all tests or selected ones -# +# Run plugin unit tests test: -ifndef class - ./gradlew ${mm}test -else - ./gradlew ${mm}test --tests ${class} -endif - -assemble: - ./gradlew assemble - -# -# generate build zips under build/plugins -# you can install the plugin copying manually these files to $HOME/.nextflow/plugins -# -buildPlugins: - ./gradlew copyPluginZip - -# -# Upload JAR artifacts to Maven Central -# -upload: - ./gradlew upload - + ./gradlew test -upload-plugins: - ./gradlew plugins:upload +# Install the plugin into local nextflow plugins dir +install: + ./gradlew install -publish-index: - ./gradlew plugins:publishIndex +# Publish the plugin +release: + ./gradlew releasePlugin diff --git a/README.md b/README.md index 8510c89..25f45f1 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,9 @@ If you want to build and run this plugin from source, you can use this method: ```bash git clone git@github.com:royjacobson/nf-python.git && cd nf-python -make buildPlugins -export VER="0.1.3" # Change appropriately -cp -r build/plugins/nf-python-${VER} ~/.nextflow/plugins/ - +make install export NXF_OFFLINE=true -nextflow my_flow.nf -plugins "nf-python@${VER}" +nextflow my_flow.nf -plugins "nf-python@0.1.4" # Change appropriately ``` ## License diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..6f712f3 --- /dev/null +++ b/build.gradle @@ -0,0 +1,37 @@ +// Plugins +plugins { + id 'io.nextflow.nextflow-plugin' version '1.0.0-beta.6' +} + +// Plugin version +version = '0.1.4' + +nextflowPlugin { + // Minimum Nextflow version + nextflowVersion = '24.04.0' + + // Plugin metadata + provider = 'Roy Jacobson' + className = 'nextflow.python.PythonPlugin' + extensionPoints = [ + 'nextflow.python.PythonExtension' + ] +} + +buildDir = project.layout.buildDirectory + +task buildPython(type: Exec) { + workingDir 'py' + commandLine 'python', '-m', 'build', '--wheel', '--outdir', "${buildDir}" +} + +task packagePython(type: Copy) { + dependsOn buildPython + from(zipTree("${buildDir}/nf_python_plugin-${version}-py3-none-any.whl")) + into("${buildDir}/python-pkg") +} + +project.tasks.named('packagePlugin', Zip) { + dependsOn packagePython + from("${buildDir}/python-pkg") +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle deleted file mode 100644 index 66a0569..0000000 --- a/buildSrc/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - */ - -plugins { - // Support convention plugins written in Groovy. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build. - id 'groovy-gradle-plugin' -} - -repositories { - // Use the plugin portal to apply community plugins in convention plugins. - gradlePluginPortal() -} - diff --git a/buildSrc/src/main/groovy/io.nextflow.groovy-application-conventions.gradle b/buildSrc/src/main/groovy/io.nextflow.groovy-application-conventions.gradle deleted file mode 100644 index ecc029b..0000000 --- a/buildSrc/src/main/groovy/io.nextflow.groovy-application-conventions.gradle +++ /dev/null @@ -1,11 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - */ - -plugins { - // Apply the common convention plugin for shared build configuration between library and application projects. - id 'io.nextflow.groovy-common-conventions' - - // Apply the application plugin to add support for building a CLI application in Java. - id 'application' -} diff --git a/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle b/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle deleted file mode 100644 index 312ec86..0000000 --- a/buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle +++ /dev/null @@ -1,42 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - */ - -plugins { - // Apply the groovy Plugin to add support for Groovy. - id 'groovy' -} - -repositories { - // Use Maven Central for resolving dependencies. - mavenCentral() -} - -java { - // these settings apply to all jvm tooling, including groovy - toolchain { - languageVersion = JavaLanguageVersion.of(21) - } - sourceCompatibility = 17 - targetCompatibility = 17 -} - -tasks.withType(Test) { - jvmArgs ([ - '--add-opens=java.base/java.lang=ALL-UNNAMED', - '--add-opens=java.base/java.io=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED', - '--add-opens=java.base/java.nio.file.spi=ALL-UNNAMED', - '--add-opens=java.base/java.net=ALL-UNNAMED', - '--add-opens=java.base/java.util=ALL-UNNAMED', - '--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED', - '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED', - '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', - '--add-opens=java.base/sun.nio.fs=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.http=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.https=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.ftp=ALL-UNNAMED', - '--add-opens=java.base/sun.net.www.protocol.file=ALL-UNNAMED', - '--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED', - ]) -} diff --git a/buildSrc/src/main/groovy/io.nextflow.groovy-library-conventions.gradle b/buildSrc/src/main/groovy/io.nextflow.groovy-library-conventions.gradle deleted file mode 100644 index 0183e2c..0000000 --- a/buildSrc/src/main/groovy/io.nextflow.groovy-library-conventions.gradle +++ /dev/null @@ -1,11 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - */ - -plugins { - // Apply the common convention plugin for shared build configuration between library and application projects. - id 'io.nextflow.groovy-common-conventions' - // Apply the java-library plugin for API and implementation separation. - id 'java-library' -} - diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c..7f93135 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2733ed5..598c78d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists +zipStorePath=wrapper/dists \ No newline at end of file diff --git a/gradlew b/gradlew index 4f906e0..1aa94a4 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,99 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +119,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/launch.sh b/launch.sh deleted file mode 100755 index 20ae0fd..0000000 --- a/launch.sh +++ /dev/null @@ -1,2 +0,0 @@ -export NXF_PLUGINS_DEV=$PWD/plugins - ../nextflow/launch.sh "$@" diff --git a/plugins/build.gradle b/plugins/build.gradle deleted file mode 100644 index ed5a617..0000000 --- a/plugins/build.gradle +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2021-2022, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - id "java" - id "io.nextflow.nf-build-plugin" version "1.0.1" -} - -ext.github_organization = 'nextflow-io' -ext.github_username = project.findProperty('github_username') ?: 'pditommaso' -ext.github_access_token = project.findProperty('github_access_token') ?: System.getenv('GITHUB_TOKEN') -ext.github_commit_email = project.findProperty('github_commit_email') ?: 'paolo.ditommaso@gmail.com' - -jar.enabled = false - -String computeSha512(File file) { - if( !file.exists() ) - throw new GradleException("Missing file: $file -- cannot compute SHA-512") - return org.apache.commons.codec.digest.DigestUtils.sha512Hex(file.bytes) -} - -String now() { - "${java.time.OffsetDateTime.now().format(java.time.format.DateTimeFormatter.ISO_DATE_TIME)}" -} - -List allPlugins() { - def plugins = [] - new File(rootProject.rootDir, 'plugins') .eachDir { if(it.name.startsWith('nf-')) plugins.add(it.name) } - return plugins -} - -String metaFromManifest(String meta, File file) { - def str = file.text - def regex = ~/(?m)^$meta:\s*([\w-\.<>=]+)$/ - def m = regex.matcher(str) - if( m.find() ) { - def ver = m.group(1) - //println "Set plugin '${file.parentFile.parentFile.parentFile.parentFile.name}' version=${ver}" - return ver - } - throw new GradleException("Cannot find '$meta' for plugin: $file") -} - -def timestamp = now() - -subprojects { - apply plugin: 'java' - apply plugin: 'groovy' - apply plugin: 'io.nextflow.nf-build-plugin' - - repositories { - mavenLocal() - mavenCentral() - } - - version = metaFromManifest('Plugin-Version',file('src/resources/META-INF/MANIFEST.MF')) - - tasks.withType(Jar) { - duplicatesStrategy = DuplicatesStrategy.INCLUDE - } - - /* - * Creates plugin zip and json meta file in plugin `build/libs` directory - */ - task makeZip(type: Jar) { - into('classes') { with jar } - into('lib') { from configurations.runtimeClasspath } - manifest.from file('src/resources/META-INF/MANIFEST.MF') - archiveExtension = 'zip' - preserveFileTimestamps = false - reproducibleFileOrder = true - - doLast { - // create the meta file - final zip = new File("$buildDir/libs/${project.name}-${project.version}.zip") - final json = new File("$buildDir/libs/${project.name}-${project.version}-meta.json") - json.text = """\ - { - "version": "${project.version}", - "date": "${timestamp}", - "url": "https://github.com/${github_organization}/${project.name}/releases/download/${project.version}/${project.name}-${project.version}.zip", - "requires": "${metaFromManifest('Plugin-Requires',file('src/resources/META-INF/MANIFEST.MF'))}", - "sha512sum": "${computeSha512(zip)}" - } - """.stripIndent() - // cleanup tmp dir - file("$buildDir/tmp/makeZip").deleteDir() - } - outputs.file("$buildDir/libs/${project.name}-${project.version}.zip") - } - - /* - * Copy the plugin dependencies in the subproject `build/target/libs` directory - */ - task copyPluginLibs(type: Sync) { - from configurations.runtimeClasspath - into 'build/target/libs' - } - - /* - * Copy the plugin in the project root build/plugins directory - */ - task copyPluginZip(type: Copy, dependsOn: project.tasks.findByName('makeZip')) { - from makeZip - into "$rootProject.buildDir/plugins" - outputs.file("$rootProject.buildDir/plugins/${project.name}-${project.version}.zip") - doLast { - ant.unzip( - src: "$rootProject.buildDir/plugins/${project.name}-${project.version}.zip", - dest: "$rootProject.buildDir/plugins/${project.name}-${project.version}" - ) - } - } - - /* - * "install" the plugin the project root build/plugins directory - */ - project.parent.tasks.getByName("assemble").dependsOn << copyPluginZip - - task uploadPlugin(type: io.nextflow.gradle.tasks.GithubUploader, dependsOn: makeZip) { - assets = providers.provider {["$buildDir/libs/${project.name}-${project.version}.zip", - "$buildDir/libs/${project.name}-${project.version}-meta.json" ]} - release = providers.provider { project.version } - repo = providers.provider { project.name } - owner = github_organization - userName = github_username - authToken = github_access_token - skipExisting = true - } - - jar { - from sourceSets.main.allSource - doLast { - file("$buildDir/tmp/jar").deleteDir() - } - } - - tasks.withType(GenerateModuleMetadata) { - enabled = false - } - - task upload(dependsOn: [uploadPlugin] ) { } -} - -/* - * Upload all plugins to the corresponding GitHub repos - */ -task upload(dependsOn: [subprojects.uploadPlugin]) { } - -/* - * Copies the plugins required dependencies in the corresponding lib directory - */ -classes.dependsOn subprojects.copyPluginLibs - -/* - * Merge and publish the plugins index file - */ -task publishIndex( type: io.nextflow.gradle.tasks.GithubRepositoryPublisher ) { - indexUrl = 'https://github.com/nextflow-io/plugins/main/plugins.json' - repos = allPlugins() - owner = github_organization - githubUser = github_username - githubEmail = github_commit_email - githubToken = github_access_token -} diff --git a/plugins/nf-python/build.gradle b/plugins/nf-python/build.gradle deleted file mode 100644 index 4afb868..0000000 --- a/plugins/nf-python/build.gradle +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2021-2022, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { - // Apply the groovy plugin to add support for Groovy - id 'io.nextflow.groovy-library-conventions' - id 'idea' -} - -group = 'io.nextflow' -// DO NOT SET THE VERSION HERE -// THE VERSION FOR PLUGINS IS DEFINED IN THE `/resources/META-INF/MANIFEST.NF` file - -idea { - module.inheritOutputDirs = true -} - -repositories { - mavenCentral() - maven { url = 'https://jitpack.io' } - maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } - maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } -} - -configurations { - // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies - runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' -} - -sourceSets { - main.java.srcDirs = [] - main.groovy.srcDirs = ['src/main'] - main.resources.srcDirs = ['src/resources'] -} - -ext{ - nextflowVersion = '25.01.0-edge' -} - -dependencies { - // This dependency is exported to consumers, that is to say found on their compile classpath. - compileOnly "io.nextflow:nextflow:$nextflowVersion" - compileOnly 'org.slf4j:slf4j-api:2.0.16' - compileOnly 'org.pf4j:pf4j:3.12.0' - - // test configuration - testImplementation "org.apache.groovy:groovy:4.0.26" - testImplementation "org.apache.groovy:groovy-nio:4.0.26" - testImplementation "io.nextflow:nextflow:$nextflowVersion" - testImplementation ("org.apache.groovy:groovy-test:4.0.26") { exclude group: 'org.apache.groovy' } - testImplementation ("cglib:cglib-nodep:3.3.0") - testImplementation ("org.objenesis:objenesis:3.1") - testImplementation ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('org.spockframework:spock-junit4:2.3-groovy-4.0') { exclude group: 'org.apache.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('com.google.jimfs:jimfs:1.1') - - testImplementation(testFixtures("io.nextflow:nextflow:$nextflowVersion")) - testImplementation(testFixtures("io.nextflow:nf-commons:$nextflowVersion")) - - // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement - modules { - module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") } - } -} - -// use JUnit 5 platform -test { - useJUnitPlatform() -} - -task buildPython(type: Exec) { - workingDir '../../py' - commandLine 'python', '-m', 'build', '--wheel', '--outdir', "${buildDir}" -} - -task packagePython(type: Copy) { - dependsOn buildPython - def manifestFile = file('src/resources/META-INF/MANIFEST.MF') - def props = new Properties() - manifestFile.withInputStream { props.load(it) } - def pluginVersion = props['Plugin-Version'] - - from(zipTree("${buildDir}/nf_python_plugin-${pluginVersion}-py3-none-any.whl")) - into("${buildDir}/python-pkg") -} - -tasks.named('makeZip', Jar) { - dependsOn packagePython - from("${buildDir}/python-pkg") -} diff --git a/plugins/nf-python/src/resources/META-INF/MANIFEST.MF b/plugins/nf-python/src/resources/META-INF/MANIFEST.MF deleted file mode 100644 index 16fb44a..0000000 --- a/plugins/nf-python/src/resources/META-INF/MANIFEST.MF +++ /dev/null @@ -1,6 +0,0 @@ -Manifest-Version: 1.0 -Plugin-Id: nf-python -Plugin-Version: 0.1.3 -Plugin-Class: nextflow.python.PythonPlugin -Plugin-Provider: nextflow -Plugin-Requires: >=23.04.0 diff --git a/plugins/nf-python/src/resources/META-INF/extensions.idx b/plugins/nf-python/src/resources/META-INF/extensions.idx deleted file mode 100644 index bff4ea4..0000000 --- a/plugins/nf-python/src/resources/META-INF/extensions.idx +++ /dev/null @@ -1,2 +0,0 @@ -nextflow.python.PythonPlugin -nextflow.python.PythonExtension \ No newline at end of file diff --git a/py/nf_python/nextflow.py b/py/nf_python/nextflow.py index 5199729..cb42c58 100644 --- a/py/nf_python/nextflow.py +++ b/py/nf_python/nextflow.py @@ -172,10 +172,10 @@ def output(self, *args, **kwargs): raise ValueError("Cant pass both unnamed outputs and named outputs!") if args: with open(self._outfile, "w") as f: - json.dump(pack_python(args), f, default=str) + json.dump(pack_python(args), f, separators=(',', ':')) elif kwargs: with open(self._outfile, "w") as f: - json.dump(pack_python(kwargs), f, default=str) + json.dump(pack_python(kwargs), f, separators=(',', ':')) self._written_output = True diff --git a/py/pyproject.toml b/py/pyproject.toml index a5ffc21..74afdd4 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nf-python-plugin" -version = "0.1.3" +version = "0.1.4" authors = [ { name="Roy Jacobson", email="roy.jacobson@weizmann.ac.il" }, ] diff --git a/settings.gradle b/settings.gradle index 8bdb4b8..41a4c4d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -22,5 +22,3 @@ plugins { } rootProject.name = 'nf-python' -include('plugins') -include('plugins:nf-python') diff --git a/plugins/nf-python/src/main/nextflow/python/PythonExtension.groovy b/src/main/groovy/nextflow/python/PythonExtension.groovy similarity index 64% rename from plugins/nf-python/src/main/nextflow/python/PythonExtension.groovy rename to src/main/groovy/nextflow/python/PythonExtension.groovy index 9d52722..e992567 100644 --- a/plugins/nf-python/src/main/nextflow/python/PythonExtension.groovy +++ b/src/main/groovy/nextflow/python/PythonExtension.groovy @@ -1,17 +1,44 @@ package nextflow.python +import com.google.common.hash.Hasher import groovy.json.JsonOutput import groovy.json.JsonSlurper import groovy.transform.CompileStatic +import java.nio.file.Path +import java.nio.file.Paths +import nextflow.conda.CondaCache import nextflow.plugin.extension.Function import nextflow.plugin.extension.PluginExtensionPoint import nextflow.Session +import nextflow.util.CacheHelper +import nextflow.util.Duration import nextflow.util.MemoryUnit import nextflow.util.VersionNumber -import nextflow.util.Duration -import java.nio.file.Path -import nextflow.conda.CondaCache -import nextflow.conda.CondaConfig +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@CompileStatic +class PythonExecSession { + + final Path infile + final Path outfile + final Path stdout + final Path stderr + final Path script + + PythonExecSession(Path path, String scriptPath = null) { + infile = path.resolve('in.json') + outfile = path.resolve('out.json') + stdout = path.resolve('stdout.log') + stderr = path.resolve('stderr.log') + if (scriptPath) { + script = Paths.get(scriptPath) + } else { + script = path.resolve('script.py') + } + } + +} @CompileStatic class PythonExtension extends PluginExtensionPoint { @@ -20,30 +47,16 @@ class PythonExtension extends PluginExtensionPoint { from nf_python import nf ''' + static final Logger log = LoggerFactory.getLogger(PythonExtension) + private static Path pluginDir + private Session session private String executable - private static Path pluginDir - static void setPluginDir(Path path) { pluginDir = path } - private String getPythonExecutable(String condaEnv) { - CondaCache cache = new CondaCache(session.getCondaConfig()) - java.nio.file.Path condaPath = cache.getCachePathFor(condaEnv) - - Process proc = new ProcessBuilder('conda', 'run', '-p', condaPath.toString(), 'which', 'python') - .redirectErrorStream(true) - .start() - def output = proc.inputStream.text.trim() - if (proc.waitFor() == 0 && output) { - return output - } else { - throw new IllegalStateException("Failed to find Python executable in conda environment: $condaEnv\n Output: ${output}") - } - } - @Override void init(Session session) { this.session = session @@ -68,19 +81,86 @@ from nf_python import nf return 'python' } + @Function + Object pyFunction(String code = '') { + return pyFunction([:], code) + } + @Function Object pyFunction(Map args, String code = '') { assert !(code && args.containsKey('script')) : 'Cannot use both code and script options together' - if (code) { - args = prepareScript(code, args) + + List excludedKeys = ['script', '_executable', '_conda_env'] + Map forwardedArgs = args.findAll { k, v -> !(k in excludedKeys) } + + String executable = this.executable + if (args.containsKey('_executable') || args.containsKey('_conda_env')) { + executable = getExecutableFromConfigVals(args._executable ?: '', args._conda_env ?: '') + } + + // Pack args once and JSON-serialize once + List packedArgs = packGroovy(forwardedArgs) + String serializedArgs = JsonOutput.toJson(packedArgs) + log.trace('Serialized args: {}', serializedArgs) + + String hash = directoryHash( + executable, + args.containsKey('script') ? new File(args.script as String).text : code, + serializedArgs + ) + File executionDir = workDirForHash(hash) + if (executionDir.exists()) { + log.debug "Found old job directory ${executionDir}, removing." + executionDir.deleteDir() + } + executionDir.mkdirs() + PythonExecSession execDir = new PythonExecSession( + executionDir.toPath(), + args.containsKey('script') ? args.script as String : '' + ) + log.debug "Starting python execution (hash: ${hash}) in: ${executionDir.absolutePath}" + + if (!args.containsKey('script')) { + prepareScript(code, execDir.script) + } + + execDir.infile.text = serializedArgs + + String[] proc = [executable, execDir.script] as String[] + Map env = [ + 'NEXTFLOW_INFILE': execDir.infile.toAbsolutePath().toString(), + 'NEXTFLOW_OUTFILE': execDir.outfile.toAbsolutePath().toString(), + 'PYTHONPATH': System.getenv('PYTHONPATH') ? + System.getenv('PYTHONPATH') + File.pathSeparator + pluginDir.toString() : + pluginDir.toString() + ] + + ProcessBuilder pb = new ProcessBuilder(proc) + pb.redirectOutput(execDir.stdout.toFile()) + pb.redirectError(execDir.stderr.toFile()) + pb.environment().putAll(env) + + Process process = pb.start() + int rc = process.waitFor() + if (rc != 0) { + reportError(process, proc, execDir, "Python script evaluation failed") } - return runPythonScript(args) + if (execDir.outfile.exists()) { + log.trace('Python output content: {}', execDir.outfile.toFile().text) + } else { + reportError(process, proc, execDir, "Python script did not produce expected output file.") + } + Object result = new JsonSlurper().parse(execDir.outfile.toFile()) + return unpackPython(result) } - @Function - Object pyFunction(String code = '') { - return pyFunction([:], code) + private static String directoryHash(String executable, String code, String serializedArgs) { + Hasher hasher = CacheHelper.hasher(executable) + hasher.putString(executable, java.nio.charset.StandardCharsets.UTF_8) + hasher.putString(code, java.nio.charset.StandardCharsets.UTF_8) + hasher.putString(serializedArgs, java.nio.charset.StandardCharsets.UTF_8) + return hasher.hash() } private static String normalizeIndentation(String code) { @@ -99,67 +179,30 @@ from nf_python import nf return normalizedLines.join('\n') } - private static Map prepareScript(String code, Map args) { + private static void prepareScript(String code, Path scriptPath) { if (!code) { throw new IllegalArgumentException('Missing code argument') } - if (!args) { - args = [:] - } - if (args.containsKey('script')) { - throw new IllegalArgumentException('The "script" option is reserved for the script file path and cannot be used with inline code') - } // Normalize indentation to avoid issues with Python indentation String script = PYTHON_PREAMBLE + normalizeIndentation(code) - File scriptFile = File.createTempFile('nfpy_code', '.py') - scriptFile.deleteOnExit() + File scriptFile = scriptPath.toFile() + log.debug "Writing python code to a temporary file: ${scriptFile.absolutePath}" scriptFile.text = script - - // Prepare options and arguments - Map argsWithScript = args + [script: scriptFile.absolutePath] - return argsWithScript } - private Object runPythonScript(Map args) { - def script = args.script - if (!script) { - throw new IllegalArgumentException('Missing script argument') - } - def excludedKeys = ['script', '_executable', '_conda_env'] - Map forwardedArgs = args.findAll { k, v -> !(k in excludedKeys) } - - executable = this.executable - if (args.containsKey('_executable') || args.containsKey('_conda_env')) { - executable = getExecutableFromConfigVals(args._executable ?: '', args._conda_env ?: '') - } - - File infile = File.createTempFile('nfpy_in', '.json') - infile.deleteOnExit() - infile.text = JsonOutput.toJson(packGroovy(forwardedArgs)) - File outfile = File.createTempFile('nfpy_out', '.json') - outfile.deleteOnExit() - - String[] proc = [executable, script] as String[] - Map env = [ - 'NEXTFLOW_INFILE': infile.absolutePath, - 'NEXTFLOW_OUTFILE': outfile.absolutePath, - 'PYTHONPATH': System.getenv('PYTHONPATH') ? - System.getenv('PYTHONPATH') + File.pathSeparator + pluginDir.toString() : - pluginDir.toString() - ] - ProcessBuilder pb = new ProcessBuilder(proc) - - pb.environment().putAll(env) - pb.redirectErrorStream(true) - Process process = pb.start() - process.inputStream.eachLine { line -> println "[python] $line" } - int rc = process.waitFor() - if (rc != 0) { - throw new nextflow.exception.ProcessEvalException("Python script evaluation failed", proc.join(' '), '', rc) - } - - Object result = new JsonSlurper().parse(outfile) - return unpackPython(result) + private static void reportError(Process process, String[] command, PythonExecSession execDir, String errMsg) { + String stderrContent = execDir.stderr.toFile().text + String[] stderrLines = stderrContent.split('\n') + String firstLines = stderrLines.take(10).join('\n') + String lastLines = stderrLines.takeRight(10).join('\n') + log.error(errMsg + "\n" + + "Command: ${command.join(' ')} exited with exit code ${process.exitValue()}\n" + + "First 10 lines of stderr:\n$firstLines\n" + + "Last 10 lines of stderr:\n$lastLines\n" + + "Check the log files in:\n" + + "\t'${execDir.stdout.toAbsolutePath()}'\n" + + "\t'${execDir.stderr.toAbsolutePath()}'") + throw new nextflow.exception.ProcessEvalException(errMsg, command.join(' '), '', process.exitValue()) } private static def packFloat(Double value) { @@ -273,4 +316,32 @@ from nf_python import nf default: throw new IllegalArgumentException("Unknown type from Python: $type") } } + + private String getPythonExecutable(String condaEnv) { + CondaCache cache = new CondaCache(session.getCondaConfig()) + + log.debug "Looking for conda env '$condaEnv' in conda cache" + java.nio.file.Path condaPath = cache.getCachePathFor(condaEnv) + log.debug "Conda environment found in '$condaPath'" + + // We can't just use "$condaPath/bin/python" because conda may not have python installed. + // TODO: Should we cache this? + Process proc = new ProcessBuilder('conda', 'run', '-p', condaPath.toString(), 'which', 'python') + .redirectErrorStream(true) + .start() + def output = proc.inputStream.text.trim() + if (!proc.waitFor() == 0 || !output) { + throw new IllegalStateException("Failed to find Python executable in conda environment: $condaEnv\n Output: $output") + } + + log.debug "Found Python executable in conda environment: $output" + return output + } + + private File workDirForHash(String hash) { + def root = session.getWorkDir().resolve('.nf-python') + def path = root.resolve(hash.substring(0, 2)).resolve(hash.substring(2)) + return path.toFile() + } + } diff --git a/plugins/nf-python/src/main/nextflow/python/PythonPlugin.groovy b/src/main/groovy/nextflow/python/PythonPlugin.groovy similarity index 100% rename from plugins/nf-python/src/main/nextflow/python/PythonPlugin.groovy rename to src/main/groovy/nextflow/python/PythonPlugin.groovy