From 338dff5b9006fb68055e95f26e0336984b7a4a0e Mon Sep 17 00:00:00 2001 From: amardeepio Date: Wed, 16 Oct 2024 14:57:17 +0530 Subject: [PATCH 1/3] Add NADA program and Jupyter notebook for SNP-genotype matching via Nillion --- services/nillion-interactor/compile.sh | 4 + .../large_dataset-test.ipynb | 561 ++++++++++++------ .../nillion-interactor/programs/snp-test.py | 22 + 3 files changed, 409 insertions(+), 178 deletions(-) create mode 100644 services/nillion-interactor/programs/snp-test.py diff --git a/services/nillion-interactor/compile.sh b/services/nillion-interactor/compile.sh index a8a37ee..6fd1d16 100755 --- a/services/nillion-interactor/compile.sh +++ b/services/nillion-interactor/compile.sh @@ -12,3 +12,7 @@ time pynadac --target-dir binaries --generate-mir-json programs/muscle-perform-a echo "Compiling double.py" time pynadac --target-dir binaries --generate-mir-json programs/double.py + +echo "Compiling snp-test.py" +time pynadac --target-dir binaries --generate-mir-json programs/snp-test.py + diff --git a/services/nillion-interactor/large_dataset-test.ipynb b/services/nillion-interactor/large_dataset-test.ipynb index e10935b..df88535 100644 --- a/services/nillion-interactor/large_dataset-test.ipynb +++ b/services/nillion-interactor/large_dataset-test.ipynb @@ -1,110 +1,103 @@ { "cells": [ { - "metadata": {}, "cell_type": "markdown", + "id": "663e6c9a14c8b44e", + "metadata": {}, "source": [ "This notebook loads some genomic data from a single user from a 23andMe file and stores it on the Nillion platform.\n", "\n", "The objective is to benchmark the storage capacity and performance of the Nillion platform for large datasets." - ], - "id": "663e6c9a14c8b44e" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Let's install the necessary libraries.", - "id": "70096bc058cadc0e" + "id": "70096bc058cadc0e", + "metadata": {}, + "source": [ + "Let's install the necessary libraries." + ] }, { + "cell_type": "code", + "execution_count": 18, + "id": "7280ce2eeb321ec3", "metadata": { "ExecuteTime": { "end_time": "2024-08-22T13:57:05.213020Z", "start_time": "2024-08-22T13:56:42.806836Z" } }, - "cell_type": "code", - "source": [ - "!pip install numpy\n", - "!pip install nada_numpy" - ], - "id": "7280ce2eeb321ec3", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Collecting numpy\r\n", - " Downloading numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.3 MB)\r\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m16.3/16.3 MB\u001B[0m \u001B[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m00:01\u001B[0m00:01\u001B[0m\r\n", - "\u001B[?25hInstalling collected packages: numpy\r\n", - "Successfully installed numpy-2.1.0\r\n", - "Collecting nada_numpy\r\n", - " Downloading nada_numpy-0.4.0-py3-none-any.whl (23 kB)\r\n", - "Collecting numpy<2.0.0,>=1.26.4\r\n", - " Downloading numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\r\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m18.2/18.2 MB\u001B[0m \u001B[31m3.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m00:01\u001B[0m00:01\u001B[0m\r\n", - "\u001B[?25hRequirement already satisfied: nada-dsl<0.6.0,>=0.5.0 in ./venv/lib/python3.10/site-packages (from nada_numpy) (0.5.0)\r\n", - "Requirement already satisfied: py-nillion-client<0.6.0,>=0.5.0 in ./venv/lib/python3.10/site-packages (from nada_numpy) (0.5.0)\r\n", - "Requirement already satisfied: nillion-python-helpers<0.3.0,>=0.2.3 in ./venv/lib/python3.10/site-packages (from nada_numpy) (0.2.3)\r\n", - "Requirement already satisfied: richreports~=0.2 in ./venv/lib/python3.10/site-packages (from nada-dsl<0.6.0,>=0.5.0->nada_numpy) (0.2.0)\r\n", - "Requirement already satisfied: sortedcontainers~=2.4 in ./venv/lib/python3.10/site-packages (from nada-dsl<0.6.0,>=0.5.0->nada_numpy) (2.4.0)\r\n", - "Requirement already satisfied: asttokens~=2.4 in ./venv/lib/python3.10/site-packages (from nada-dsl<0.6.0,>=0.5.0->nada_numpy) (2.4.1)\r\n", - "Requirement already satisfied: parsial~=0.1 in ./venv/lib/python3.10/site-packages (from nada-dsl<0.6.0,>=0.5.0->nada_numpy) (0.1.0)\r\n", - "Requirement already satisfied: cosmpy<0.10.0,>=0.9.2 in ./venv/lib/python3.10/site-packages (from nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (0.9.2)\r\n", - "Requirement already satisfied: python-dotenv==1.0.0 in ./venv/lib/python3.10/site-packages (from nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.0.0)\r\n", - "Requirement already satisfied: pytest-asyncio<0.24.0,>=0.23.7 in ./venv/lib/python3.10/site-packages (from nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (0.23.8)\r\n", - "Requirement already satisfied: protobuf<5.0.dev0,>=4.21.6 in ./venv/lib/python3.10/site-packages (from py-nillion-client<0.6.0,>=0.5.0->nada_numpy) (4.25.4)\r\n", - "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.10/site-packages (from asttokens~=2.4->nada-dsl<0.6.0,>=0.5.0->nada_numpy) (1.16.0)\r\n", - "Requirement already satisfied: jsonschema<5,>=3.2.0 in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (4.23.0)\r\n", - "Requirement already satisfied: bech32 in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.2.0)\r\n", - "Requirement already satisfied: googleapis-common-protos in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.63.2)\r\n", - "Requirement already satisfied: requests in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2.32.3)\r\n", - "Requirement already satisfied: ecdsa in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (0.19.0)\r\n", - "Requirement already satisfied: pycryptodome<4.0.0,>=3.18.0 in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (3.20.0)\r\n", - "Requirement already satisfied: grpcio in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.65.4)\r\n", - "Requirement already satisfied: python-dateutil in ./venv/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2.9.0.post0)\r\n", - "Requirement already satisfied: pytest<9,>=7.0.0 in ./venv/lib/python3.10/site-packages (from pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (8.1.1)\r\n", - "Requirement already satisfied: referencing>=0.28.4 in ./venv/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (0.35.1)\r\n", - "Requirement already satisfied: attrs>=22.2.0 in ./venv/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (23.2.0)\r\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in ./venv/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2023.12.1)\r\n", - "Requirement already satisfied: rpds-py>=0.7.1 in ./venv/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (0.19.1)\r\n", - "Requirement already satisfied: iniconfig in ./venv/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2.0.0)\r\n", - "Requirement already satisfied: tomli>=1 in ./venv/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2.0.1)\r\n", - "Requirement already satisfied: packaging in ./venv/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (24.0)\r\n", - "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in ./venv/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.2.0)\r\n", - "Requirement already satisfied: pluggy<2.0,>=1.4 in ./venv/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (1.4.0)\r\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2.2.2)\r\n", - "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (3.7)\r\n", - "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (2024.7.4)\r\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.3.0,>=0.2.3->nada_numpy) (3.3.2)\r\n", - "Installing collected packages: numpy, nada_numpy\r\n", - " Attempting uninstall: numpy\r\n", - " Found existing installation: numpy 2.1.0\r\n", - " Uninstalling numpy-2.1.0:\r\n", - " Successfully uninstalled numpy-2.1.0\r\n", - "Successfully installed nada_numpy-0.4.0 numpy-1.26.4\r\n" + "Requirement already satisfied: numpy in /home/amardeep/jupyter_env/lib/python3.10/site-packages (1.26.4)\n", + "Requirement already satisfied: nada_numpy in /home/amardeep/jupyter_env/lib/python3.10/site-packages (0.5.3)\n", + "Requirement already satisfied: py-nillion-client<0.7.0,>=0.6.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.0)\n", + "Requirement already satisfied: nada-test<0.7.0,>=0.6.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.0)\n", + "Requirement already satisfied: nada-dsl<0.7.0,>=0.6.3 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.3)\n", + "Requirement already satisfied: numpy<2.0.0,>=1.26.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (1.26.4)\n", + "Requirement already satisfied: nillion-python-helpers<0.4.0,>=0.3.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.3.1)\n", + "Requirement already satisfied: parsial~=0.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.1.0)\n", + "Requirement already satisfied: richreports~=0.2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.2.0)\n", + "Requirement already satisfied: asttokens~=2.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (2.4.1)\n", + "Requirement already satisfied: sortedcontainers~=2.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (2.4.0)\n", + "Requirement already satisfied: pytest-asyncio<0.24.0,>=0.23.7 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.23.8)\n", + "Requirement already satisfied: python-dotenv==1.0.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.0.0)\n", + "Requirement already satisfied: cosmpy<0.10.0,>=0.9.2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.9.2)\n", + "Requirement already satisfied: protobuf<5.0.dev0,>=4.21.6 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from py-nillion-client<0.7.0,>=0.6.0->nada_numpy) (4.25.5)\n", + "Requirement already satisfied: six>=1.12.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from asttokens~=2.4->nada-dsl<0.7.0,>=0.6.3->nada_numpy) (1.16.0)\n", + "Requirement already satisfied: ecdsa in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.19.0)\n", + "Requirement already satisfied: python-dateutil in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.9.0.post0)\n", + "Requirement already satisfied: pycryptodome<4.0.0,>=3.18.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.21.0)\n", + "Requirement already satisfied: bech32 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.2.0)\n", + "Requirement already satisfied: jsonschema<5,>=3.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (4.23.0)\n", + "Requirement already satisfied: grpcio in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.66.2)\n", + "Requirement already satisfied: requests in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.32.3)\n", + "Requirement already satisfied: googleapis-common-protos in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.65.0)\n", + "Requirement already satisfied: pytest<9,>=7.0.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (8.3.3)\n", + "Requirement already satisfied: attrs>=22.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (24.2.0)\n", + "Requirement already satisfied: referencing>=0.28.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.35.1)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2023.12.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.20.0)\n", + "Requirement already satisfied: pluggy<2,>=1.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.5.0)\n", + "Requirement already satisfied: tomli>=1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.0.2)\n", + "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.2.2)\n", + "Requirement already satisfied: packaging in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (24.1)\n", + "Requirement already satisfied: iniconfig in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.0.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2024.8.30)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.3.2)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.2.3)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.10)\n" ] } ], - "execution_count": 1 + "source": [ + "!pip install numpy\n", + "!pip install nada_numpy" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Let's load the genomic data from the sample 23andMe file.", - "id": "f8485b0518d8438d" + "id": "f8485b0518d8438d", + "metadata": {}, + "source": [ + "Let's load the genomic data from the sample 23andMe file." + ] }, { "cell_type": "code", + "execution_count": 19, "id": "initial_id", "metadata": { - "collapsed": true, "ExecuteTime": { "end_time": "2024-08-22T13:57:21.425536Z", "start_time": "2024-08-22T13:57:21.319781Z" } }, + "outputs": [], "source": [ "import os\n", "import numpy as np\n", @@ -156,23 +149,18 @@ " lines_read += 1\n", "\n", " return np.array(results, dtype=int)" - ], - "outputs": [], - "execution_count": 2 + ] }, { + "cell_type": "code", + "execution_count": 26, + "id": "d2ede7bba4a995b0", "metadata": { "ExecuteTime": { "end_time": "2024-08-22T14:24:49.198120Z", "start_time": "2024-08-22T14:24:47.342660Z" } }, - "cell_type": "code", - "source": [ - "gene_data = read_and_process_23andme('testdata/hu278AF5_20210124151934.txt', 1000000)\n", - "gene_data[:5, :]" - ], - "id": "d2ede7bba4a995b0", "outputs": [ { "data": { @@ -180,55 +168,93 @@ "array([[548049170, 1, 69869, 9],\n", " [ 9283150, 1, 565508, 0],\n", " [116587930, 1, 727841, 7],\n", - " [ 3131972, 1, 752721, 2],\n", - " [ 12184325, 1, 754105, 4]])" + " ...,\n", + " [143461987, 1, 19099907, 0],\n", + " [ 6603908, 1, 19100057, 6],\n", + " [ 4920344, 1, 19104381, 2]])" ] }, - "execution_count": 13, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 13 + "source": [ + "gene_data = read_and_process_23andme('testdata/hu278AF5_20210124151934.txt', 5000)\n", + "gene_data[:5000, :]" + ] }, { + "cell_type": "code", + "execution_count": 25, + "id": "c1d96e2ea2220a9b", "metadata": { "ExecuteTime": { "end_time": "2024-08-22T14:34:09.411111Z", "start_time": "2024-08-22T14:34:09.406476Z" } }, - "cell_type": "code", - "source": "gene_data.shape", - "id": "c1d96e2ea2220a9b", "outputs": [ { "data": { "text/plain": [ - "(631955, 4)" + "(5000, 4)" ] }, - "execution_count": 15, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 15 + "source": [ + "gene_data.shape" + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Let's do some Nillon setup.", - "id": "40ccec22482ad3d6" + "id": "40ccec22482ad3d6", + "metadata": {}, + "source": [ + "Let's do some Nillon setup." + ] }, { + "cell_type": "code", + "execution_count": 27, + "id": "84da614cf4520b6c", "metadata": { "ExecuteTime": { "end_time": "2024-08-21T23:38:46.733896Z", "start_time": "2024-08-21T23:38:46.710257Z" } }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading environment variables from /home/amardeep/.config/nillion/nillion-devnet.env\n", + "NILLION_CLUSTER_ID: 9e68173f-9c23-4acc-ba81-4f079b639964\n", + "NILLION_BOOTNODE_MULTIADDRESS: /ip4/127.0.0.1/tcp/37939/p2p/12D3KooWMvw1hEqm7EWSDEyqTb6pNetUVkepahKY6hixuAuMZfJS\n", + "NILLION_BOOTNODE_WEBSOCKET: /ip4/127.0.0.1/tcp/54936/ws/p2p/12D3KooWMvw1hEqm7EWSDEyqTb6pNetUVkepahKY6hixuAuMZfJS\n", + "NILLION_NILCHAIN_CHAIN_ID: nillion-chain-devnet\n", + "NILLION_NILCHAIN_JSON_RPC: http://127.0.0.1:48102\n", + "NILLION_NILCHAIN_REST_API: http://localhost:26650\n", + "NILLION_NILCHAIN_GRPC: localhost:26649\n", + "NILLION_NILCHAIN_PRIVATE_KEY_0: 9a975f567428d054f2bf3092812e6c42f901ce07d9711bc77ee2cd81101f42c5\n", + "NILLION_NILCHAIN_PRIVATE_KEY_1: 1e491133b9408b39572a29f91644873decea554224b20e2b0b923aeb860a1c18\n", + "NILLION_NILCHAIN_PRIVATE_KEY_2: 980488572f235316cdb330191f8bafe4e635efbe88b3a40f5bee9bd21047c059\n", + "NILLION_NILCHAIN_PRIVATE_KEY_3: 612bb5173dc60d9e91404fcc0d1f1847fb4459a7d5160d63d84e91aacbf2ab2f\n", + "NILLION_NILCHAIN_PRIVATE_KEY_4: 04f5a984eeea9dce4e5e907da69c01a61568e3071b1a91cbed89225f9fd913b5\n", + "NILLION_NILCHAIN_PRIVATE_KEY_5: 5f992c58921f4af83b4c6b650c4914626664cd02020577b0ada49cfa00d2c8a4\n", + "NILLION_NILCHAIN_PRIVATE_KEY_6: 8f0297d3bb647eb59b95b29550b2aebbedd9be2c954b000e772efe8c9318a42d\n", + "NILLION_NILCHAIN_PRIVATE_KEY_7: c395243df9bb68dc809668efe4125f0eb017771ed8e3747b8d6860551913fecb\n", + "NILLION_NILCHAIN_PRIVATE_KEY_8: 4bb5eaa799e24ae2b48545c41331921afe7e6a8dd7a850f5fbeb20a8226664ec\n", + "NILLION_NILCHAIN_PRIVATE_KEY_9: ef4b944d4fdb0077057925fe2dde365dfa2c83cf320463b14589feccd1b2b938\n", + "Deprecation warning: payments_endpoint should start with http:// or https://\n" + ] + } + ], "source": [ "import os\n", "import py_nillion_client as nillion\n", @@ -266,50 +292,27 @@ " PrivateKey(bytes.fromhex(os.getenv(\"NILLION_NILCHAIN_PRIVATE_KEY_0\"))),\n", " prefix=\"nillion\",\n", ")" - ], - "id": "84da614cf4520b6c", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading environment variables from /Users/vishakh/.config/nillion/nillion-devnet.env\n", - "NILLION_CLUSTER_ID: 9e68173f-9c23-4acc-ba81-4f079b639964\n", - "NILLION_BOOTNODE_MULTIADDRESS: /ip4/127.0.0.1/tcp/37939/p2p/12D3KooWMvw1hEqm7EWSDEyqTb6pNetUVkepahKY6hixuAuMZfJS\n", - "NILLION_BOOTNODE_WEBSOCKET: /ip4/127.0.0.1/tcp/54936/ws/p2p/12D3KooWMvw1hEqm7EWSDEyqTb6pNetUVkepahKY6hixuAuMZfJS\n", - "NILLION_NILCHAIN_CHAIN_ID: nillion-chain-devnet\n", - "NILLION_NILCHAIN_JSON_RPC: http://127.0.0.1:48102\n", - "NILLION_NILCHAIN_REST_API: http://localhost:26650\n", - "NILLION_NILCHAIN_GRPC: localhost:26649\n", - "NILLION_NILCHAIN_PRIVATE_KEY_0: 9a975f567428d054f2bf3092812e6c42f901ce07d9711bc77ee2cd81101f42c5\n", - "NILLION_NILCHAIN_PRIVATE_KEY_1: 1e491133b9408b39572a29f91644873decea554224b20e2b0b923aeb860a1c18\n", - "NILLION_NILCHAIN_PRIVATE_KEY_2: 980488572f235316cdb330191f8bafe4e635efbe88b3a40f5bee9bd21047c059\n", - "NILLION_NILCHAIN_PRIVATE_KEY_3: 612bb5173dc60d9e91404fcc0d1f1847fb4459a7d5160d63d84e91aacbf2ab2f\n", - "NILLION_NILCHAIN_PRIVATE_KEY_4: 04f5a984eeea9dce4e5e907da69c01a61568e3071b1a91cbed89225f9fd913b5\n", - "NILLION_NILCHAIN_PRIVATE_KEY_5: 5f992c58921f4af83b4c6b650c4914626664cd02020577b0ada49cfa00d2c8a4\n", - "NILLION_NILCHAIN_PRIVATE_KEY_6: 8f0297d3bb647eb59b95b29550b2aebbedd9be2c954b000e772efe8c9318a42d\n", - "NILLION_NILCHAIN_PRIVATE_KEY_7: c395243df9bb68dc809668efe4125f0eb017771ed8e3747b8d6860551913fecb\n", - "NILLION_NILCHAIN_PRIVATE_KEY_8: 4bb5eaa799e24ae2b48545c41331921afe7e6a8dd7a850f5fbeb20a8226664ec\n", - "NILLION_NILCHAIN_PRIVATE_KEY_9: ef4b944d4fdb0077057925fe2dde365dfa2c83cf320463b14589feccd1b2b938\n" - ] - } - ], - "execution_count": 47 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "Adapting a utility function from https://github.com/NillionNetwork/nada-numpy/blob/8dbf08c0ee840cab8b6103ef1d7dc5dcf438d442/examples/common/utils.py#L158. ", - "id": "8d964ddbdf389117" + "id": "8d964ddbdf389117", + "metadata": {}, + "source": [ + "Adapting a utility function from https://github.com/NillionNetwork/nada-numpy/blob/8dbf08c0ee840cab8b6103ef1d7dc5dcf438d442/examples/common/utils.py#L158. " + ] }, { + "cell_type": "code", + "execution_count": 28, + "id": "2740bbe472dbdffb", "metadata": { "ExecuteTime": { "end_time": "2024-08-21T23:38:48.516561Z", "start_time": "2024-08-21T23:38:48.512197Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "from typing import Any\n", "import nada_numpy.client as na_client\n", @@ -345,25 +348,37 @@ " cluster_id, stored_secret, permissions, receipt_store\n", " )\n", " return store_id" - ], - "id": "2740bbe472dbdffb", - "outputs": [], - "execution_count": 48 + ] }, { - "metadata": {}, "cell_type": "markdown", - "source": "And now..the moment of truth!", - "id": "19a8548d0ceb970" + "id": "19a8548d0ceb970", + "metadata": {}, + "source": [ + "And now..the moment of truth!" + ] }, { + "cell_type": "code", + "execution_count": 29, + "id": "ad961c332478c206", "metadata": { "ExecuteTime": { "end_time": "2024-08-21T23:38:51.468043Z", "start_time": "2024-08-21T23:38:50.723226Z" } }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Getting quote for operation...\n", + "Quote cost is 1920002 unil\n", + "Submitting payment receipt 1920002 unil, tx hash A84327D511AD28A469259FDFB9B09A80029D7E839E6DC86A335276B77C510EC7\n" + ] + } + ], "source": [ "permissions = nillion.Permissions.default_for_user(monadic_user_id)\n", "\n", @@ -378,35 +393,12 @@ " 1,\n", " permissions,\n", ")" - ], - "id": "ad961c332478c206", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Getting quote for operation...\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "requesting price quote: price quote query failed: price quote query failed: payload size (11171406) exceeds maximum (6291456)", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mRuntimeError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[49], line 3\u001B[0m\n\u001B[1;32m 1\u001B[0m permissions \u001B[38;5;241m=\u001B[39m nillion\u001B[38;5;241m.\u001B[39mPermissions\u001B[38;5;241m.\u001B[39mdefault_for_user(monadic_user_id)\n\u001B[0;32m----> 3\u001B[0m store_id \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m store_secret_array(\n\u001B[1;32m 4\u001B[0m monadic_client,\n\u001B[1;32m 5\u001B[0m payments_wallet,\n\u001B[1;32m 6\u001B[0m payments_client,\n\u001B[1;32m 7\u001B[0m cluster_id,\n\u001B[1;32m 8\u001B[0m gene_data,\n\u001B[1;32m 9\u001B[0m gene_data,\n\u001B[1;32m 10\u001B[0m nillion\u001B[38;5;241m.\u001B[39mSecretInteger,\n\u001B[1;32m 11\u001B[0m \u001B[38;5;241m1\u001B[39m,\n\u001B[1;32m 12\u001B[0m permissions,\n\u001B[1;32m 13\u001B[0m )\n", - "Cell \u001B[0;32mIn[48], line 22\u001B[0m, in \u001B[0;36mstore_secret_array\u001B[0;34m(client, payments_wallet, payments_client, cluster_id, secret_array, secret_name, nada_type, ttl_days, permissions)\u001B[0m\n\u001B[1;32m 17\u001B[0m stored_secret \u001B[38;5;241m=\u001B[39m nillion\u001B[38;5;241m.\u001B[39mNadaValues(\n\u001B[1;32m 18\u001B[0m na_client\u001B[38;5;241m.\u001B[39marray(secret_array, secret_name, nada_type)\n\u001B[1;32m 19\u001B[0m )\n\u001B[1;32m 21\u001B[0m \u001B[38;5;66;03m# Get cost quote, then pay for operation to store the secret\u001B[39;00m\n\u001B[0;32m---> 22\u001B[0m receipt_store \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m get_quote_and_pay(\n\u001B[1;32m 23\u001B[0m client,\n\u001B[1;32m 24\u001B[0m nillion\u001B[38;5;241m.\u001B[39mOperation\u001B[38;5;241m.\u001B[39mstore_values(stored_secret, ttl_days\u001B[38;5;241m=\u001B[39mttl_days),\n\u001B[1;32m 25\u001B[0m payments_wallet,\n\u001B[1;32m 26\u001B[0m payments_client,\n\u001B[1;32m 27\u001B[0m cluster_id,\n\u001B[1;32m 28\u001B[0m )\n\u001B[1;32m 30\u001B[0m \u001B[38;5;66;03m# Store a secret, passing in the receipt that shows proof of payment\u001B[39;00m\n\u001B[1;32m 31\u001B[0m store_id \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m client\u001B[38;5;241m.\u001B[39mstore_values(\n\u001B[1;32m 32\u001B[0m cluster_id, stored_secret, permissions, receipt_store\n\u001B[1;32m 33\u001B[0m )\n", - "File \u001B[0;32m~/dev/MonadicDNA/services/nillion-interactor/nillion-venv/lib/python3.11/site-packages/nillion_python_helpers/payments.py:29\u001B[0m, in \u001B[0;36mget_quote_and_pay\u001B[0;34m(client, operation, payments_wallet, payments_client, cluster_id)\u001B[0m\n\u001B[1;32m 15\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 16\u001B[0m \u001B[38;5;124;03mInitiates a payment for the specified operation using the Nillion client.\u001B[39;00m\n\u001B[1;32m 17\u001B[0m \n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;124;03m nillion.PaymentReceipt: The receipt of the payment containing the quote and transaction hash.\u001B[39;00m\n\u001B[1;32m 27\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mGetting quote for operation...\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m---> 29\u001B[0m quote \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mawait\u001B[39;00m client\u001B[38;5;241m.\u001B[39mrequest_price_quote(cluster_id, operation)\n\u001B[1;32m 30\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mQuote cost is \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mquote\u001B[38;5;241m.\u001B[39mcost\u001B[38;5;241m.\u001B[39mtotal\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m unil\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 31\u001B[0m address \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(Address(payments_wallet\u001B[38;5;241m.\u001B[39mpublic_key(), \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnillion\u001B[39m\u001B[38;5;124m\"\u001B[39m))\n", - "\u001B[0;31mRuntimeError\u001B[0m: requesting price quote: price quote query failed: price quote query failed: payload size (11171406) exceeds maximum (6291456)" - ] - } - ], - "execution_count": 49 + ] }, { - "metadata": {}, "cell_type": "markdown", + "id": "755c0d8d6ef4ceb6", + "metadata": {}, "source": [ "Here are the results so far:\n", "\n", @@ -416,87 +408,300 @@ "* 1000 records - 3s 104ms\n", "\n", "We now need to break the 10,000 record barrier!" - ], - "id": "755c0d8d6ef4ceb6" + ] }, { + "cell_type": "code", + "execution_count": 31, + "id": "a019086389c4aff", "metadata": { "ExecuteTime": { "end_time": "2024-08-22T14:41:58.625914Z", "start_time": "2024-08-22T14:41:58.616792Z" } }, - "cell_type": "code", - "source": "gene_data[gene_data[:, 0] == 1815739]", - "id": "a019086389c4aff", "outputs": [ { "data": { "text/plain": [ - "array([[ 1815739, 11, 66328095, 6]])" + "array([[548049170, 1, 69869, 9]])" ] }, - "execution_count": 16, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 16 + "source": [ + "gene_data[gene_data[:, 0] == 548049170]" + ] }, { + "cell_type": "code", + "execution_count": 32, + "id": "273585be787988f6", "metadata": { "ExecuteTime": { "end_time": "2024-08-22T14:44:11.011121Z", "start_time": "2024-08-22T14:44:11.003410Z" } }, - "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[548049170 1 69869 9]]\n", + "True\n" + ] + } + ], "source": [ - "target_snp_1 = 1815739\n", + "target_snp_1 = 548049170\n", "target_genotype_1 = 9\n", "\n", "rows_of_interest = gene_data[gene_data[:, 0] == target_snp_1]\n", "is_successful = rows_of_interest[0, 3] == target_genotype_1\n", "print(rows_of_interest)\n", "print(is_successful)" + ] + }, + { + "cell_type": "markdown", + "id": "90fb5425f9d91736", + "metadata": {}, + "source": [ + "Now let's do it using a Nada program." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "75b84800-548e-4335-b1fd-7b320123c32c", + "metadata": {}, + "outputs": [], + "source": [ + "import uuid\n", + "def gen_node_key():\n", + " return NodeKey.from_seed(uuid.uuid4().hex)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "445ed311-b674-48f0-bbae-0d612a687a40", + "metadata": {}, + "outputs": [], + "source": [ + "patient_seed = \"patient_seed\"\n", + "patient_userkey = UserKey.from_seed(patient_seed)\n", + "patient_client = create_nillion_client(patient_userkey, gen_node_key())\n", + "patient_party_id = patient_client.party_id\n", + "patient_user_id = patient_client.user_id\n" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "9c073be6-3e83-4c97-8dea-782bb7e1061c", + "metadata": {}, + "outputs": [], + "source": [ + "program_name = \"snp-test\"\n", + "program_mir_path = f\"binaries/snp-test.nada.bin\"" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "0408f81e-be21-4e1d-82c3-837422b19304", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The patient is storing the program on the network\n" + ] + } ], - "id": "273585be787988f6", + "source": [ + "# Function to store the program\n", + "import os\n", + "\n", + "os.environ['RUST_BACKTRACE'] = '1'\n", + "\n", + "print(\"The patient is storing the program on the network\")\n", + "\n", + "async def store_program(client, program_name, program_mir_path):\n", + " print(f\"Storing the program '{program_name}' on the network\")\n", + " receipt_store_program = await get_quote_and_pay(\n", + " patient_client,\n", + " nillion.Operation.store_program(program_mir_path),\n", + " payments_wallet,\n", + " payments_client,\n", + " cluster_id,\n", + " )\n", + " action_id = await patient_client.store_program(\n", + " cluster_id, program_name, program_mir_path, receipt_store_program\n", + " )\n", + " return action_id" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "49133e0c-34a7-435b-9d9b-84325aa958cd", + "metadata": {}, + "outputs": [], + "source": [ + "# Function to store secret data\n", + "async def store_secret(client, user_id, program_name, input_snp, input_genotype):\n", + " print(\"Storing the secret SNP and genotype data on the network\")\n", + " program_id = f\"{patient_user_id}/{program_name}\"\n", + " new_secret = nillion.NadaValues({\n", + " \"input_snp\": nillion.SecretInteger(input_snp),\n", + " \"input_genotype\": nillion.SecretInteger(input_genotype),\n", + " })\n", + " permissions = nillion.Permissions.default_for_user(patient_user_id)\n", + " permissions.add_compute_permissions({user_id: {program_id}})\n", + " receipt_store = await get_quote_and_pay(\n", + " patient_client,\n", + " nillion.Operation.store_values(new_secret, ttl_days=5),\n", + " payments_wallet,\n", + " payments_client,\n", + " cluster_id,\n", + " )\n", + " store_id = await patient_client.store_values(\n", + " cluster_id, new_secret, permissions, receipt_store\n", + " )\n", + " return store_id" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "8047f85b-b5db-40f5-b90a-6e86c58b5d18", + "metadata": {}, + "outputs": [], + "source": [ + "# Function to compute on secret data\n", + "async def compute_on_secret(client, user_id, program_name, party_id, store_id):\n", + " print(\"Running the SNP matching program on the secret data\")\n", + " program_id = f\"{patient_user_id}/{program_name}\"\n", + " party_name = \"Party1\"\n", + " compute_bindings = nillion.ProgramBindings(program_id)\n", + " compute_bindings.add_input_party(party_name, patient_party_id)\n", + " compute_bindings.add_output_party(party_name, patient_party_id)\n", + " computation_time_secrets = nillion.NadaValues({})\n", + " receipt_compute = await get_quote_and_pay(\n", + " patient_client,\n", + " nillion.Operation.compute(program_id, computation_time_secrets),\n", + " payments_wallet,\n", + " payments_client,\n", + " cluster_id,\n", + " )\n", + " compute_id = await client.compute(\n", + " cluster_id,\n", + " compute_bindings,\n", + " [store_id],\n", + " computation_time_secrets,\n", + " receipt_compute,\n", + " )\n", + " return compute_id" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "ecebfdd1-e8ad-4b7b-be79-097a198434f0", + "metadata": {}, + "outputs": [], + "source": [ + "async def main():\n", + " # Store the program\n", + " action_id = await store_program(patient_client, program_name, program_mir_path)\n", + " print(f\"Program stored. Action ID: {action_id}\")\n", + "\n", + " # Store the secret data (example values)\n", + " input_snp = 13 # rs548049170\n", + " input_genotype = 9 # TT genotype\n", + " store_id = await store_secret(patient_client, patient_user_id, program_name, input_snp, input_genotype)\n", + " print(f\"Secret data stored. Store ID: {store_id}\")\n", + "\n", + " # Compute on the secret\n", + " compute_id = await compute_on_secret(patient_client, patient_user_id, program_name, patient_party_id, store_id)\n", + " print(f\"Computation sent to the network. Compute ID: {compute_id}\")\n", + "\n", + " # Wait for and print the result\n", + " while True:\n", + " compute_event = await patient_client.next_compute_event()\n", + " if isinstance(compute_event, nillion.ComputeFinishedEvent):\n", + " print(f\"✅ Compute complete for compute_id {compute_event.uuid}\")\n", + " print(f\"🖥️ The result is {compute_event.result.value}\")\n", + " break\n" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "edb0c6fc-8f8c-4423-8ca8-43ddf7ba6c88", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[[ 1815739 11 66328095 6]]\n", - "False\n" + "Storing the program 'snp-test' on the network\n", + "Getting quote for operation...\n", + "Quote cost is 2 unil\n", + "Submitting payment receipt 2 unil, tx hash 725F5EEE13B51B5AA8151B81332A33D3640B8D0268259A7C0913AD3A7CF3A1F6\n", + "Program stored. Action ID: 229XQCZoSQk9zDx4ixKXg8wAqCy7HnmUjcTuMe7D1bUVhNG7kYNF4xeZYjRsEtJUhKFvXwfM65ixFWDRhptfHCqw/snp-test\n", + "Storing the secret SNP and genotype data on the network\n", + "Getting quote for operation...\n", + "Quote cost is 962 unil\n", + "Submitting payment receipt 962 unil, tx hash D5A4FEFCA08120A2B67BFD82211D43E779A2517074C65C4E9491CAF528ECBF00\n", + "Secret data stored. Store ID: e2287b2d-6f02-4fd7-a841-1a81caca825c\n", + "Running the SNP matching program on the secret data\n", + "Getting quote for operation...\n", + "Quote cost is 19 unil\n", + "Submitting payment receipt 19 unil, tx hash 5255444B40ED906455BADD3E2D43BBB857506E81FD99D7FD46FB3FB3FFD804C4\n", + "Computation sent to the network. Compute ID: 6501007f-9ad8-4616-8dfb-da8b7ef0ddc3\n", + "✅ Compute complete for compute_id 6501007f-9ad8-4616-8dfb-da8b7ef0ddc3\n", + "🖥️ The result is {'match_result': 1}\n" ] } ], - "execution_count": 21 + "source": [ + "await main()" + ] }, { + "cell_type": "code", + "execution_count": null, + "id": "79e1396c-aec0-4695-bfb5-5d7fde5c8e3e", "metadata": {}, - "cell_type": "markdown", - "source": "Now let's do it using a Nada program.", - "id": "90fb5425f9d91736" + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/services/nillion-interactor/programs/snp-test.py b/services/nillion-interactor/programs/snp-test.py new file mode 100644 index 0000000..4ffcd97 --- /dev/null +++ b/services/nillion-interactor/programs/snp-test.py @@ -0,0 +1,22 @@ +from nada_dsl import * + +def nada_main(): + party1 = Party(name="Party1") + + # Input SNP and genotype as secret integers + input_snp = SecretInteger(Input(name="input_snp", party=party1)) + input_genotype = SecretInteger(Input(name="input_genotype", party=party1)) + + # Define target SNP and genotype + # For this example, let's check for rs548049170 (coded as 11) with TT genotype (coded as 9) + target_snp = Integer(13) #rs548049170 + target_genotype = Integer(9) # TT genotype + + # Check if input matches target + snp_match = (input_snp >= target_snp).if_else((target_snp >= input_snp).if_else(Integer(1), Integer(0)), Integer(0)) + genotype_match = (input_genotype >= target_genotype).if_else((target_genotype >= input_genotype).if_else(Integer(1), Integer(0)), Integer(0)) + + # Combine conditions + is_match = snp_match * genotype_match + + return [Output(is_match, "match_result", party1)] \ No newline at end of file From 0e9d9ec851728982a473a4068194a07cbd0937bc Mon Sep 17 00:00:00 2001 From: amardeepio Date: Mon, 21 Oct 2024 20:42:08 +0530 Subject: [PATCH 2/3] reduced rows from 5000 to 500 to work nada-numpy nada program --- services/nillion-interactor/compile.sh | 4 +- .../large_dataset-test.ipynb | 169 ++++++++++-------- .../programs/snp-test-array.py | 37 ++++ 3 files changed, 132 insertions(+), 78 deletions(-) create mode 100644 services/nillion-interactor/programs/snp-test-array.py diff --git a/services/nillion-interactor/compile.sh b/services/nillion-interactor/compile.sh index 6fd1d16..16c8dde 100755 --- a/services/nillion-interactor/compile.sh +++ b/services/nillion-interactor/compile.sh @@ -13,6 +13,6 @@ time pynadac --target-dir binaries --generate-mir-json programs/muscle-perform-a echo "Compiling double.py" time pynadac --target-dir binaries --generate-mir-json programs/double.py -echo "Compiling snp-test.py" -time pynadac --target-dir binaries --generate-mir-json programs/snp-test.py +echo "Compiling snp-test-array.py" +time pynadac --target-dir binaries --generate-mir-json programs/snp-test-array.py diff --git a/services/nillion-interactor/large_dataset-test.ipynb b/services/nillion-interactor/large_dataset-test.ipynb index df88535..10d0608 100644 --- a/services/nillion-interactor/large_dataset-test.ipynb +++ b/services/nillion-interactor/large_dataset-test.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 111, "id": "7280ce2eeb321ec3", "metadata": { "ExecuteTime": { @@ -35,42 +35,42 @@ "text": [ "Requirement already satisfied: numpy in /home/amardeep/jupyter_env/lib/python3.10/site-packages (1.26.4)\n", "Requirement already satisfied: nada_numpy in /home/amardeep/jupyter_env/lib/python3.10/site-packages (0.5.3)\n", + "Requirement already satisfied: numpy<2.0.0,>=1.26.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (1.26.4)\n", "Requirement already satisfied: py-nillion-client<0.7.0,>=0.6.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.0)\n", + "Requirement already satisfied: nillion-python-helpers<0.4.0,>=0.3.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.3.1)\n", "Requirement already satisfied: nada-test<0.7.0,>=0.6.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.0)\n", "Requirement already satisfied: nada-dsl<0.7.0,>=0.6.3 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.6.3)\n", - "Requirement already satisfied: numpy<2.0.0,>=1.26.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (1.26.4)\n", - "Requirement already satisfied: nillion-python-helpers<0.4.0,>=0.3.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada_numpy) (0.3.1)\n", - "Requirement already satisfied: parsial~=0.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.1.0)\n", - "Requirement already satisfied: richreports~=0.2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.2.0)\n", - "Requirement already satisfied: asttokens~=2.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (2.4.1)\n", "Requirement already satisfied: sortedcontainers~=2.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (2.4.0)\n", - "Requirement already satisfied: pytest-asyncio<0.24.0,>=0.23.7 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.23.8)\n", - "Requirement already satisfied: python-dotenv==1.0.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.0.0)\n", + "Requirement already satisfied: asttokens~=2.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (2.4.1)\n", + "Requirement already satisfied: richreports~=0.2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.2.0)\n", + "Requirement already satisfied: parsial~=0.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nada-dsl<0.7.0,>=0.6.3->nada_numpy) (0.1.0)\n", "Requirement already satisfied: cosmpy<0.10.0,>=0.9.2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.9.2)\n", + "Requirement already satisfied: python-dotenv==1.0.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.0.0)\n", + "Requirement already satisfied: pytest-asyncio<0.24.0,>=0.23.7 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.23.8)\n", "Requirement already satisfied: protobuf<5.0.dev0,>=4.21.6 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from py-nillion-client<0.7.0,>=0.6.0->nada_numpy) (4.25.5)\n", "Requirement already satisfied: six>=1.12.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from asttokens~=2.4->nada-dsl<0.7.0,>=0.6.3->nada_numpy) (1.16.0)\n", - "Requirement already satisfied: ecdsa in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.19.0)\n", - "Requirement already satisfied: python-dateutil in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.9.0.post0)\n", - "Requirement already satisfied: pycryptodome<4.0.0,>=3.18.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.21.0)\n", "Requirement already satisfied: bech32 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.2.0)\n", - "Requirement already satisfied: jsonschema<5,>=3.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (4.23.0)\n", - "Requirement already satisfied: grpcio in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.66.2)\n", "Requirement already satisfied: requests in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.32.3)\n", "Requirement already satisfied: googleapis-common-protos in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.65.0)\n", + "Requirement already satisfied: jsonschema<5,>=3.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (4.23.0)\n", + "Requirement already satisfied: ecdsa in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.19.0)\n", + "Requirement already satisfied: grpcio in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.66.2)\n", + "Requirement already satisfied: pycryptodome<4.0.0,>=3.18.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.21.0)\n", + "Requirement already satisfied: python-dateutil in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.9.0.post0)\n", "Requirement already satisfied: pytest<9,>=7.0.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (8.3.3)\n", - "Requirement already satisfied: attrs>=22.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (24.2.0)\n", "Requirement already satisfied: referencing>=0.28.4 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.35.1)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2023.12.1)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (0.20.0)\n", - "Requirement already satisfied: pluggy<2,>=1.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.5.0)\n", + "Requirement already satisfied: attrs>=22.2.0 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from jsonschema<5,>=3.2.0->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2023.12.1)\n", "Requirement already satisfied: tomli>=1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.0.2)\n", - "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.2.2)\n", + "Requirement already satisfied: pluggy<2,>=1.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.5.0)\n", "Requirement already satisfied: packaging in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (24.1)\n", "Requirement already satisfied: iniconfig in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.0.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2024.8.30)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.3.2)\n", + "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from pytest<9,>=7.0.0->pytest-asyncio<0.24.0,>=0.23.7->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (1.2.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2.2.3)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.10)\n" + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (3.3.2)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/amardeep/jupyter_env/lib/python3.10/site-packages (from requests->cosmpy<0.10.0,>=0.9.2->nillion-python-helpers<0.4.0,>=0.3.0->nada_numpy) (2024.8.30)\n" ] } ], @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 112, "id": "initial_id", "metadata": { "ExecuteTime": { @@ -153,7 +153,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 113, "id": "d2ede7bba4a995b0", "metadata": { "ExecuteTime": { @@ -169,24 +169,24 @@ " [ 9283150, 1, 565508, 0],\n", " [116587930, 1, 727841, 7],\n", " ...,\n", - " [143461987, 1, 19099907, 0],\n", - " [ 6603908, 1, 19100057, 6],\n", - " [ 4920344, 1, 19104381, 2]])" + " [148956934, 1, 2817983, 7],\n", + " [ 11807311, 1, 2818217, 0],\n", + " [ 1869970, 1, 2829551, 2]])" ] }, - "execution_count": 26, + "execution_count": 113, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "gene_data = read_and_process_23andme('testdata/hu278AF5_20210124151934.txt', 5000)\n", - "gene_data[:5000, :]" + "gene_data = read_and_process_23andme('testdata/hu278AF5_20210124151934.txt', 500)\n", + "gene_data[:500, :]" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 114, "id": "c1d96e2ea2220a9b", "metadata": { "ExecuteTime": { @@ -198,10 +198,10 @@ { "data": { "text/plain": [ - "(5000, 4)" + "(500, 4)" ] }, - "execution_count": 25, + "execution_count": 114, "metadata": {}, "output_type": "execute_result" } @@ -220,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 115, "id": "84da614cf4520b6c", "metadata": { "ExecuteTime": { @@ -304,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 116, "id": "2740bbe472dbdffb", "metadata": { "ExecuteTime": { @@ -360,7 +360,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 117, "id": "ad961c332478c206", "metadata": { "ExecuteTime": { @@ -374,8 +374,8 @@ "output_type": "stream", "text": [ "Getting quote for operation...\n", - "Quote cost is 1920002 unil\n", - "Submitting payment receipt 1920002 unil, tx hash A84327D511AD28A469259FDFB9B09A80029D7E839E6DC86A335276B77C510EC7\n" + "Quote cost is 192002 unil\n", + "Submitting payment receipt 192002 unil, tx hash D06675A3E36D378CC21CB3EF73FBD2E2F56DDFDC2AD1CE85ABDEEFFE9A0CDE2C\n" ] } ], @@ -412,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 118, "id": "a019086389c4aff", "metadata": { "ExecuteTime": { @@ -427,7 +427,7 @@ "array([[548049170, 1, 69869, 9]])" ] }, - "execution_count": 31, + "execution_count": 118, "metadata": {}, "output_type": "execute_result" } @@ -438,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 119, "id": "273585be787988f6", "metadata": { "ExecuteTime": { @@ -476,7 +476,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 120, "id": "75b84800-548e-4335-b1fd-7b320123c32c", "metadata": {}, "outputs": [], @@ -488,7 +488,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 121, "id": "445ed311-b674-48f0-bbae-0d612a687a40", "metadata": {}, "outputs": [], @@ -502,18 +502,18 @@ }, { "cell_type": "code", - "execution_count": 35, - "id": "9c073be6-3e83-4c97-8dea-782bb7e1061c", + "execution_count": 122, + "id": "91163779-de9a-42d3-93e2-6e9df3ad12ae", "metadata": {}, "outputs": [], "source": [ - "program_name = \"snp-test\"\n", - "program_mir_path = f\"binaries/snp-test.nada.bin\"" + "program_name = \"snp-test-array\"\n", + "program_mir_path = f\"binaries/snp-test-array.nada.bin\"" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 123, "id": "0408f81e-be21-4e1d-82c3-837422b19304", "metadata": {}, "outputs": [ @@ -550,37 +550,38 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 124, "id": "49133e0c-34a7-435b-9d9b-84325aa958cd", "metadata": {}, "outputs": [], "source": [ - "# Function to store secret data\n", - "async def store_secret(client, user_id, program_name, input_snp, input_genotype):\n", - " print(\"Storing the secret SNP and genotype data on the network\")\n", - " program_id = f\"{patient_user_id}/{program_name}\"\n", + "async def store_secret(client, user_id, program_name, gene_data):\n", + " print(\"Storing the gene data on the network\")\n", + " program_id = f\"{user_id}/{program_name}\"\n", " new_secret = nillion.NadaValues({\n", - " \"input_snp\": nillion.SecretInteger(input_snp),\n", - " \"input_genotype\": nillion.SecretInteger(input_genotype),\n", + " f\"gene_data_{i}_{j}\": nillion.SecretInteger(int(gene_data[i][j]))\n", + " for i in range(gene_data.shape[0])\n", + " for j in range(gene_data.shape[1])\n", " })\n", - " permissions = nillion.Permissions.default_for_user(patient_user_id)\n", + " \n", + " permissions = nillion.Permissions.default_for_user(user_id)\n", " permissions.add_compute_permissions({user_id: {program_id}})\n", " receipt_store = await get_quote_and_pay(\n", - " patient_client,\n", + " client,\n", " nillion.Operation.store_values(new_secret, ttl_days=5),\n", " payments_wallet,\n", " payments_client,\n", " cluster_id,\n", " )\n", - " store_id = await patient_client.store_values(\n", + " store_id = await client.store_values(\n", " cluster_id, new_secret, permissions, receipt_store\n", " )\n", - " return store_id" + " return store_id " ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 125, "id": "8047f85b-b5db-40f5-b90a-6e86c58b5d18", "metadata": {}, "outputs": [], @@ -593,7 +594,9 @@ " compute_bindings = nillion.ProgramBindings(program_id)\n", " compute_bindings.add_input_party(party_name, patient_party_id)\n", " compute_bindings.add_output_party(party_name, patient_party_id)\n", - " computation_time_secrets = nillion.NadaValues({})\n", + " computation_time_secrets = nillion.NadaValues({\n", + " \"sum_genotypes\": nillion.SecretInteger(0)\n", + " })\n", " receipt_compute = await get_quote_and_pay(\n", " patient_client,\n", " nillion.Operation.compute(program_id, computation_time_secrets),\n", @@ -613,7 +616,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 126, "id": "ecebfdd1-e8ad-4b7b-be79-097a198434f0", "metadata": {}, "outputs": [], @@ -624,9 +627,7 @@ " print(f\"Program stored. Action ID: {action_id}\")\n", "\n", " # Store the secret data (example values)\n", - " input_snp = 13 # rs548049170\n", - " input_genotype = 9 # TT genotype\n", - " store_id = await store_secret(patient_client, patient_user_id, program_name, input_snp, input_genotype)\n", + " store_id = await store_secret(patient_client, patient_user_id, program_name, gene_data)\n", " print(f\"Secret data stored. Store ID: {store_id}\")\n", "\n", " # Compute on the secret\n", @@ -644,7 +645,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 127, "id": "edb0c6fc-8f8c-4423-8ca8-43ddf7ba6c88", "metadata": {}, "outputs": [ @@ -652,23 +653,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "Storing the program 'snp-test' on the network\n", + "Storing the program 'snp-test-array' on the network\n", "Getting quote for operation...\n", "Quote cost is 2 unil\n", - "Submitting payment receipt 2 unil, tx hash 725F5EEE13B51B5AA8151B81332A33D3640B8D0268259A7C0913AD3A7CF3A1F6\n", - "Program stored. Action ID: 229XQCZoSQk9zDx4ixKXg8wAqCy7HnmUjcTuMe7D1bUVhNG7kYNF4xeZYjRsEtJUhKFvXwfM65ixFWDRhptfHCqw/snp-test\n", - "Storing the secret SNP and genotype data on the network\n", + "Submitting payment receipt 2 unil, tx hash 8561878854DC8DA233A21926B4C3DB20A8F76ACE5877B0230D5682F13979C16B\n", + "Program stored. Action ID: 229XQCZoSQk9zDx4ixKXg8wAqCy7HnmUjcTuMe7D1bUVhNG7kYNF4xeZYjRsEtJUhKFvXwfM65ixFWDRhptfHCqw/snp-test-array\n", + "Storing the gene data on the network\n", "Getting quote for operation...\n", - "Quote cost is 962 unil\n", - "Submitting payment receipt 962 unil, tx hash D5A4FEFCA08120A2B67BFD82211D43E779A2517074C65C4E9491CAF528ECBF00\n", - "Secret data stored. Store ID: e2287b2d-6f02-4fd7-a841-1a81caca825c\n", + "Quote cost is 960002 unil\n", + "Submitting payment receipt 960002 unil, tx hash C98347B707342516258DED2C2AC8F4915287B1C67EBFDF30CA244677CD71B7C1\n", + "Secret data stored. Store ID: 452ab966-8983-4f12-9670-304c776326b6\n", "Running the SNP matching program on the secret data\n", "Getting quote for operation...\n", - "Quote cost is 19 unil\n", - "Submitting payment receipt 19 unil, tx hash 5255444B40ED906455BADD3E2D43BBB857506E81FD99D7FD46FB3FB3FFD804C4\n", - "Computation sent to the network. Compute ID: 6501007f-9ad8-4616-8dfb-da8b7ef0ddc3\n", - "✅ Compute complete for compute_id 6501007f-9ad8-4616-8dfb-da8b7ef0ddc3\n", - "🖥️ The result is {'match_result': 1}\n" + "Quote cost is 2004 unil\n", + "Submitting payment receipt 2004 unil, tx hash 48A0FF5E511F25207F88F8D066F73232E9E5488B5D661B9A9545B424950F117B\n", + "Computation sent to the network. Compute ID: 50120b22-97c6-46ae-aa7c-468b7705d64e\n", + "✅ Compute complete for compute_id 50120b22-97c6-46ae-aa7c-468b7705d64e\n", + "🖥️ The result is {'Result': True}\n" ] } ], @@ -683,6 +684,22 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b74a739f-2a2c-4981-8796-17687fcb9488", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cadce8b-3d57-4766-8593-8921253e32b5", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/services/nillion-interactor/programs/snp-test-array.py b/services/nillion-interactor/programs/snp-test-array.py new file mode 100644 index 0000000..6392c83 --- /dev/null +++ b/services/nillion-interactor/programs/snp-test-array.py @@ -0,0 +1,37 @@ +from nada_dsl import * +import nada_numpy as na + +# Checks based on SNP rs1815739 whether someone is a sprinter or an endurance athelete +# Genotype TT indicates endurance athlete +# We arbitrarily assume sprinter otherwise +# This program returns 0 if sprinter, 1 if endurance athlete +# This is a counterpart to muscle-perform.py which operates on tabular data rather than atomic secrets. +from nada_dsl import * +import nada_numpy as na + + + +def nada_main(): + target_snp_1 = Integer(548049170) + target_genotype_1 = Integer(9) + + party1 = Party(name="Party1") + + gene_data = na.array((500, 4), party1, "gene_data", SecretInteger) + + # Create the boolean mask + boolean_mask = gene_data[:, 0] == target_snp_1 + + # Initialize sum_genotypes as a SecretInteger + sum_genotypes = SecretInteger(Input(name="sum_genotypes", party=party1)) + + # Iterate through the rows and sum the matching genotypes + for i in range(500): + row_match = boolean_mask[i] + row_genotype = gene_data[i, 3] + sum_genotypes = row_match.if_else(sum_genotypes + row_genotype, sum_genotypes) + + # Check if the sum of genotypes matches the target + is_successful = sum_genotypes == target_genotype_1 + + return [Output(is_successful, "Result", party1)] \ No newline at end of file From 79a6903223d30c9956a5147429f4ff0a8307e88e Mon Sep 17 00:00:00 2001 From: amardeepio Date: Mon, 21 Oct 2024 20:45:44 +0530 Subject: [PATCH 3/3] removed unused program --- .../nillion-interactor/programs/snp-test.py | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 services/nillion-interactor/programs/snp-test.py diff --git a/services/nillion-interactor/programs/snp-test.py b/services/nillion-interactor/programs/snp-test.py deleted file mode 100644 index 4ffcd97..0000000 --- a/services/nillion-interactor/programs/snp-test.py +++ /dev/null @@ -1,22 +0,0 @@ -from nada_dsl import * - -def nada_main(): - party1 = Party(name="Party1") - - # Input SNP and genotype as secret integers - input_snp = SecretInteger(Input(name="input_snp", party=party1)) - input_genotype = SecretInteger(Input(name="input_genotype", party=party1)) - - # Define target SNP and genotype - # For this example, let's check for rs548049170 (coded as 11) with TT genotype (coded as 9) - target_snp = Integer(13) #rs548049170 - target_genotype = Integer(9) # TT genotype - - # Check if input matches target - snp_match = (input_snp >= target_snp).if_else((target_snp >= input_snp).if_else(Integer(1), Integer(0)), Integer(0)) - genotype_match = (input_genotype >= target_genotype).if_else((target_genotype >= input_genotype).if_else(Integer(1), Integer(0)), Integer(0)) - - # Combine conditions - is_match = snp_match * genotype_match - - return [Output(is_match, "match_result", party1)] \ No newline at end of file