Better python environment experience

.github/workflows/nextflow-plugin.yml

@@ -10,7 +10,6 @@ jobs:
   build:
     runs-on: ubuntu-latest
     env:
-      NF_PYTHON_VERSION: '0.1.2'
       NXF_OFFLINE: 'true'
     steps:
     - uses: actions/checkout@v4
@@ -23,22 +22,49 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniconda-version: latest
     - name: Install Nextflow
       run: |
         wget -qO- https://get.nextflow.io | bash
         sudo mv nextflow /usr/local/bin/
         nextflow -version
-    - name: Build Python package
+    - name: Install Python build dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Test versions
       run: |
-        pip install --upgrade pip
-        pip install ./py
+        # Get plugin version from plugins/nf-python/src/resources/META-INF/MANIFEST.MF
+        PLUGIN_VERSION=$(grep 'Plugin-Version' plugins/nf-python/src/resources/META-INF/MANIFEST.MF | cut -d' ' -f2)
+        # Get python package version from py/pyproject.toml
+        PYTHON_VERSION=$(grep 'version =' py/pyproject.toml | cut -d'=' -f2 | tr -d ' "')
+        if [ "$PLUGIN_VERSION" != "$PYTHON_VERSION" ]; then
+          echo "Plugin version mismatch: $PYTHON_VERSION, $PLUGIN_VERSION"
+          exit 1
+        fi
+        echo "PLUGIN_VERSION=$PLUGIN_VERSION" >> $GITHUB_ENV
     - name: Build Nextflow plugin
       run: make buildPlugins
     - name: Move plugin to ~/.nextflow/plugins
       run: |
         mkdir -p ~/.nextflow/plugins
-        cp -r build/plugins/nf-python-${{ env.NF_PYTHON_VERSION }} ~/.nextflow/plugins/
+        cp -r build/plugins/nf-python-${{ env.PLUGIN_VERSION }} ~/.nextflow/plugins/
     - name: Run Nextflow workflow test
       run: |
-        cd test
-        nextflow run test_flow.nf -plugins "nf-python@${{ env.NF_PYTHON_VERSION }}"
+        source $CONDA/etc/profile.d/conda.sh
+
+        pushd test/test-datatypes
+        nextflow run test_flow.nf -plugins "nf-python@${{ env.PLUGIN_VERSION }}"
+        popd
+
+        pushd test/test-conda
+        nextflow run test_conda_config.nf -plugins "nf-python@${{ env.PLUGIN_VERSION }}" -config conda.config
+        nextflow run test_conda_inline.nf -plugins "nf-python@${{ env.PLUGIN_VERSION }}"
+        popd
+
+        pushd test/test-executable
+        nextflow run test_executable_config.nf -plugins "nf-python@${{ env.PLUGIN_VERSION }}" -config executable.config
+        nextflow run test_executable_inline.nf -plugins "nf-python@${{ env.PLUGIN_VERSION }}"
+        popd

.github/workflows/plugin-release.yml

@@ -41,6 +41,14 @@ jobs:
         with:
           distribution: 'temurin'
           java-version: '17'
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install Python build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
       - name: Build Nextflow plugin
         run: |
           make buildPlugins
@@ -85,6 +93,11 @@ jobs:
           filename=$(basename $file)
           echo "file=$file" >> $GITHUB_OUTPUT
           echo "filename=$filename" >> $GITHUB_OUTPUT
+
+          metafile=$(ls build/libs/nf-python-*-meta.json | head -n1)
+          metafilename=$(basename $metafile)
+          echo "metafile=$metafile" >> $GITHUB_OUTPUT
+          echo "metafilename=$metafilename" >> $GITHUB_OUTPUT
       - name: Upload plugin zip to release
         uses: actions/upload-release-asset@v1
         env:
@@ -94,3 +107,12 @@ jobs:
           asset_path: ${{ steps.find_zip.outputs.file }}
           asset_name: ${{ steps.find_zip.outputs.filename }}
           asset_content_type: application/zip
+      - name: Upload plugin zip to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          asset_path: ${{ steps.find_zip.outputs.metafile }}
+          asset_name: ${{ steps.find_zip.outputs.metafilename }}
+          asset_content_type: application/json

.gitignore

@@ -9,3 +9,5 @@ work
 out
 
 py/*.egg-info/
+
+plugins/nf-python/bin

README.md

@@ -3,13 +3,28 @@
 nf-python is a Nextflow plugin enabling seamless integration between Nextflow and Python scripts. Python scripts can be invoked as part of your Nextflow workflow, and arguments and outputs will be serialized back-and-forth automatically, making them available as native pythonic objects.
 
 ## Installation
-`nf-python` does not handle python environment setup itself. `python` needs to be in the system path and have the pypi package `nf-python-plugin` installed:
-```bash
-pip install nf-python-plugin
+To use the plugin in your Nextflow data pipelines, simply include it by writing `include { pyFunction } from 'plugin/nf-python'` and it will be downloaded and installed. For all setup options, please refer to the [Nextflow plugins documentation](https://www.nextflow.io/docs/latest/plugins.html).
+
+`nf-python` requires a working python installation in the execution environment. By default, the `python` in path will be used.
+It is also possible to specify a path to a specific python executable or a conda environment. Either in the configuration file:
+```
+nf_python {
+  // Option 1
+  executable = '/usr/bin/python'
+  // Option 2
+  conda_env = ''
+}
 ```
-🚨 Please note: installing `nf-python-plugin` through the built-in conda support is not working yet.
+The supported values for the `conda_env` option are the same ones supported by [Nextflow's native conda](https://www.nextflow.io/docs/latest/conda.html) support, and is using the same configuration options. Here are a few options:
+1. A list of required packages (e.g. `conda_env = 'numpy biopython'`)
+2. A conda configuration file (e.g. `conda_env = '/opt/task-env.yml'`)
+3. A path to an existing conda environment (e.g. `conda_env = '/home/user/.conda/envs/my-env'`)
 
-Then, to use the plugin in your Nextflow data pipelines, simply include it by writing `include { pyFunction } from 'plugin/nf-python'` and it will be downloaded and installed. For all setup options, please refer to the [Nextflow plugins documentation](https://www.nextflow.io/docs/latest/plugins.html).
+It is also possible to specify different environments on a per-function basis:
+```
+pyFunction(script: "", x: 1, y: 2, _executable: "/usr/bin/python")
+pyFunction(script: "", x: 1, y: 2, _conda_env: "matplotlib")
+```
 
 ## Example Usage
 To use a python script as part of your Nextflow pipeline, import `pyFunction` from the `nf-python` plugin:
@@ -65,9 +80,8 @@ If you want to build and run this plugin from source, you can use this method:
 
 ```bash
 git clone git@github.com:royjacobson/nf-python.git && cd nf-python
-pip install py/  # Install in the appropriate python environment
 make buildPlugins
-export VER="0.1.2"  # Change appropriately
+export VER="0.1.3"  # Change appropriately
 cp -r build/plugins/nf-python-${VER} ~/.nextflow/plugins/
 
 export NXF_OFFLINE=true

doc/protocol.md

@@ -8,7 +8,6 @@ The protocol is described here:
 
 - `NEXTFLOW_INFILE`: Path to a temporary JSON file containing serialized input arguments
 - `NEXTFLOW_OUTFILE`: Path to a temporary JSON file where serialized output are written by the Python code
-- `NEXTFLOW_PYTHON_COMPAT_VER`: Protocol compatibility version (currently "1")
 
 ## JSON Type System
 

plugins/nf-python/build.gradle

@@ -81,3 +81,23 @@ test {
     useJUnitPlatform()
 }
 
+task buildPython(type: Exec) {
+    workingDir '../../py'
+    commandLine 'python', '-m', 'build', '--wheel', '--outdir', "${buildDir}"
+}
+
+task packagePython(type: Copy) {
+    dependsOn buildPython
+    def manifestFile = file('src/resources/META-INF/MANIFEST.MF')
+    def props = new Properties()
+    manifestFile.withInputStream { props.load(it) }
+    def pluginVersion = props['Plugin-Version']
+
+    from(zipTree("${buildDir}/nf_python_plugin-${pluginVersion}-py3-none-any.whl"))
+    into("${buildDir}/python-pkg")
+}
+
+tasks.named('makeZip', Jar) {
+    dependsOn packagePython
+    from("${buildDir}/python-pkg")
+}

plugins/nf-python/src/main/nextflow/python/PythonExtension.groovy

@@ -9,6 +9,9 @@ import nextflow.Session
 import nextflow.util.MemoryUnit
 import nextflow.util.VersionNumber
 import nextflow.util.Duration
+import java.nio.file.Path
+import nextflow.conda.CondaCache
+import nextflow.conda.CondaConfig
 
 @CompileStatic
 class PythonExtension extends PluginExtensionPoint {
@@ -18,10 +21,51 @@ from nf_python import nf
 
 '''
     private Session session
+    private String executable
+
+    private static Path pluginDir
+
+    static void setPluginDir(Path path) {
+        pluginDir = path
+    }
+
+    private String getPythonExecutable(String condaEnv) {
+        CondaCache cache = new CondaCache(session.getCondaConfig())
+        java.nio.file.Path condaPath = cache.getCachePathFor(condaEnv)
+
+        Process proc = new ProcessBuilder('conda', 'run', '-p', condaPath.toString(), 'which', 'python')
+            .redirectErrorStream(true)
+            .start()
+        def output = proc.inputStream.text.trim()
+        if (proc.waitFor() == 0 && output) {
+            return output
+        } else {
+            throw new IllegalStateException("Failed to find Python executable in conda environment: $condaEnv\n Output: ${output}")
+        }
+    }
 
     @Override
     void init(Session session) {
         this.session = session
+        this.executable = getExecutableFromConfigVals(
+            session.config.navigate('nf_python.executable') ?: '',
+            session.config.navigate('nf_python.conda_env') ?: ''
+        )
+    }
+
+    String getExecutableFromConfigVals(_executable, _condaEnv) {
+        String executable = cString(_executable)
+        String condaEnv = cString(_condaEnv)
+        if (executable && condaEnv) {
+            throw new IllegalArgumentException("The 'executable' and 'conda_env' options cannot be used together")
+        }
+        if (executable) {
+            return executable
+        }
+        if (condaEnv) {
+            return getPythonExecutable(condaEnv)
+        }
+        return 'python'
     }
 
     @Function
@@ -76,32 +120,42 @@ from nf_python import nf
         return argsWithScript
     }
 
-    private static Object runPythonScript(Map args) {
+    private Object runPythonScript(Map args) {
         def script = args.script
         if (!script) {
             throw new IllegalArgumentException('Missing script argument')
         }
-        Map forwardedArgs = args.findAll { k, v -> k != 'script' }
+        def excludedKeys = ['script', '_executable', '_conda_env']
+        Map forwardedArgs = args.findAll { k, v -> !(k in excludedKeys) }
+
+        executable = this.executable
+        if (args.containsKey('_executable') || args.containsKey('_conda_env')) {
+            executable = getExecutableFromConfigVals(args._executable ?: '', args._conda_env ?: '')
+        }
+
         File infile = File.createTempFile('nfpy_in', '.json')
         infile.deleteOnExit()
         infile.text = JsonOutput.toJson(packGroovy(forwardedArgs))
         File outfile = File.createTempFile('nfpy_out', '.json')
         outfile.deleteOnExit()
 
-        String[] proc = ['python', script] as String[]
+        String[] proc = [executable, script] as String[]
         Map env = [
             'NEXTFLOW_INFILE': infile.absolutePath,
             'NEXTFLOW_OUTFILE': outfile.absolutePath,
-            'NEXTFLOW_PYTHON_COMPAT_VER': '1',
+            'PYTHONPATH': System.getenv('PYTHONPATH') ?
+                System.getenv('PYTHONPATH') + File.pathSeparator + pluginDir.toString() :
+                pluginDir.toString()
         ]
         ProcessBuilder pb = new ProcessBuilder(proc)
+
         pb.environment().putAll(env)
         pb.redirectErrorStream(true)
         Process process = pb.start()
         process.inputStream.eachLine { line -> println "[python] $line" }
         int rc = process.waitFor()
         if (rc != 0) {
-            throw new nextflow.exception.ProcessException("Python script failed with exit code $rc: $script")
+            throw new nextflow.exception.ProcessEvalException("Python script evaluation failed", proc.join(' '), '', rc)
         }
 
         Object result = new JsonSlurper().parse(outfile)

plugins/nf-python/src/main/nextflow/python/PythonPlugin.groovy

@@ -10,5 +10,6 @@ class PythonPlugin extends BasePlugin {
 
     PythonPlugin(PluginWrapper wrapper) {
         super(wrapper)
+        PythonExtension.setPluginDir(getWrapper().getPluginPath())
     }
 }

plugins/nf-python/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Id: nf-python
-Plugin-Version: 0.1.2
+Plugin-Version: 0.1.3
 Plugin-Class: nextflow.python.PythonPlugin
 Plugin-Provider: nextflow
-Plugin-Requires: >=22.04.0
+Plugin-Requires: >=23.04.0