From a1e0777630a9a5c2452f6e0c1ec2c12ef78b91c1 Mon Sep 17 00:00:00 2001 From: xavinso Date: Fri, 10 Oct 2025 15:26:24 -0400 Subject: [PATCH 1/4] Add linode single agent deployment --- scripts/akamai-single-agent-deployment.sh | 273 ++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100755 scripts/akamai-single-agent-deployment.sh diff --git a/scripts/akamai-single-agent-deployment.sh b/scripts/akamai-single-agent-deployment.sh new file mode 100755 index 0000000..19e58cf --- /dev/null +++ b/scripts/akamai-single-agent-deployment.sh @@ -0,0 +1,273 @@ +#!/bin/bash + +# CONFIGURABLE Akamai Connected Cloud (Linode) + NANDA Agent Deployment Script +# This script creates a Linode instance and deploys a fully configurable modular NANDA agent +# Usage: bash akamai-single-agent-deployment.sh [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE] [ROOT_PASSWORD] + +set -e + +# Parse arguments +AGENT_ID="$1" +ANTHROPIC_API_KEY="$2" +AGENT_NAME="$3" +DOMAIN="$4" +SPECIALIZATION="$5" +DESCRIPTION="$6" +CAPABILITIES="$7" +REGISTRY_URL="${8:-}" +PORT="${9:-6000}" +REGION="${10:-us-east}" +INSTANCE_TYPE="${11:-g6-nanode-1}" +ROOT_PASSWORD="${12:-}" + +# Validate inputs +if [ -z "$AGENT_ID" ] || [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$AGENT_NAME" ] || [ -z "$DOMAIN" ] || [ -z "$SPECIALIZATION" ] || [ -z "$DESCRIPTION" ] || [ -z "$CAPABILITIES" ]; then + echo "โŒ Usage: $0 [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE] [ROOT_PASSWORD]" + echo "" + echo "Example:" + echo " $0 data-scientist sk-ant-xxxxx \"Data Scientist\" \"data analysis\" \"analytical and precise AI assistant\" \"I specialize in data analysis, statistics, and machine learning.\" \"data analysis,statistics,machine learning,Python,R\" \"https://registry.example.com\" 6000 us-east g6-nanode-1 \"SecurePassword123!\"" + echo "" + echo "Parameters:" + echo " AGENT_ID: Unique identifier for the agent" + echo " ANTHROPIC_API_KEY: Your Anthropic API key" + echo " AGENT_NAME: Display name for the agent" + echo " DOMAIN: Primary domain/field of expertise" + echo " SPECIALIZATION: Brief description of agent's role" + echo " DESCRIPTION: Detailed description of the agent" + echo " CAPABILITIES: Comma-separated list of capabilities" + echo " REGISTRY_URL: Optional registry URL for agent discovery" + echo " PORT: Port for the agent service (default: 6000)" + echo " REGION: Linode region (default: us-east)" + echo " INSTANCE_TYPE: Linode plan type (default: g6-nanode-1)" + echo " ROOT_PASSWORD: Root password for the instance (will be generated if not provided)" + echo "" + echo "Common Linode regions: us-east, us-west, eu-west, ap-south" + echo "Common instance types: g6-nanode-1 (1GB), g6-standard-1 (2GB), g6-standard-2 (4GB)" + exit 1 +fi + + +# Generate secure password if not provided +if [ -z "$ROOT_PASSWORD" ]; then + ROOT_PASSWORD=$(openssl rand -base64 32 | tr -d "=+/" | cut -c1-25) + echo "๐Ÿ”‘ Generated root password: $ROOT_PASSWORD" +fi + + +echo "๐Ÿš€ Configurable Akamai Connected Cloud (Linode) + NANDA Agent Deployment" +echo "========================================================================" +echo "Agent ID: $AGENT_ID" +echo "Agent Name: $AGENT_NAME" +echo "Domain: $DOMAIN" +echo "Specialization: $SPECIALIZATION" +echo "Capabilities: $CAPABILITIES" +echo "Registry URL: ${REGISTRY_URL:-"None"}" +echo "Port: $PORT" +echo "Region: $REGION" +echo "Instance Type: $INSTANCE_TYPE" +echo "" + + +# Configuration +FIREWALL_LABEL="nanda-nest-agents" +SSH_KEY_LABEL="nanda-agent-key" +IMAGE_ID="linode/ubuntu25.04" # Ubuntu 2532.04 LTS + + +# Check Linode CLI credentials +echo "[1/6] Checking Linode CLI credentials..." +if ! linode-cli --version >/dev/null 2>&1; then + echo "โŒ Linode CLI not installed. Install it: https://techdocs.akamai.com/cloud-computing/docs/install-and-configure-the-cli" + exit 1 +fi + +CONFIG="$HOME/.config/linode-cli" +echo "$CONFIG is readable: $( [ -r "$CONFIG" ] && echo yes || echo no )" + +if [ ! -s "$CONFIG" ] && [ -z "$LINODE_CLI_TOKEN" ]; then + echo "โŒ Linode CLI not configured. Run 'linode-cli configure' first." + exit 1 +fi +echo "โœ… Linode CLI credentials valid" + + +# Setup firewall +echo "[2/6] Setting up firewall..." +FIREWALL_ID=$(linode-cli firewalls list --text --no-headers --format="id,label" | grep "$FIREWALL_LABEL" | cut -f1 || echo "") + +if [ -z "$FIREWALL_ID" ]; then + echo "Creating firewall..." + linode-cli firewalls create \ + --label "$FIREWALL_LABEL" \ + --rules.inbound_policy DROP \ + --rules.outbound_policy ACCEPT \ + --rules.inbound '[{"protocol": "TCP", "ports": "22", "addresses": {"ipv4": ["0.0.0.0/0"]}, "action": "ACCEPT"}, {"protocol": "TCP", "ports": "'$PORT'", "addresses": {"ipv4": ["0.0.0.0/0"]}, "action": "ACCEPT"}]' + + FIREWALL_ID=$(linode-cli firewalls list --text --no-headers --format="id,label" | grep "$FIREWALL_LABEL" | cut -f1 || echo "") +fi +echo "โœ… Firewall: $FIREWALL_ID - $FIREWALL_LABEL" + + +# Setup SSH key +echo "[3/6] Setting up SSH key..." +if [ ! -f "${SSH_KEY_LABEL}.pub" ]; then + echo "Generating SSH key pair..." + ssh-keygen -t rsa -b 4096 -f "$SSH_KEY_LABEL" -N "" -C "nanda-agent-$AGENT_ID" +fi + + +# Create user data script +echo "[4/6] Creating user data script..." +cat > "user_data_${AGENT_ID}.sh" << EOF +#!/bin/bash +exec > /var/log/user-data.log 2>&1 + +echo "=== NANDA Agent Setup Started: $AGENT_ID ===" +date + +# Update system and install dependencies +apt-get update -y +apt-get install -y python3 python3-venv python3-pip git curl + +# Create ubuntu user (Linode uses root by default) +useradd -m -s /bin/bash ubuntu +mkdir -p /home/ubuntu/.ssh +cp /root/.ssh/authorized_keys /home/ubuntu/.ssh/authorized_keys 2>/dev/null || true +chown -R ubuntu:ubuntu /home/ubuntu/.ssh +chmod 700 /home/ubuntu/.ssh +chmod 600 /home/ubuntu/.ssh/authorized_keys 2>/dev/null || true + +# Setup project as ubuntu user +cd /home/ubuntu +sudo -u ubuntu git clone https://github.com/projnanda/NEST.git nanda-agent-$AGENT_ID +cd nanda-agent-$AGENT_ID + +# Create virtual environment and install +sudo -u ubuntu python3 -m venv env +sudo -u ubuntu bash -c "source env/bin/activate && pip install --upgrade pip && pip install -e . && pip install anthropic" + +# Configure the modular agent with all environment variables +sudo -u ubuntu sed -i "s/PORT = 6000/PORT = $PORT/" examples/nanda_agent.py + +# Get public IP using Linode metadata service +echo "Getting public IP address..." +for attempt in {1..10}; do + # Linode metadata service + TOKEN=\$(curl -s -X PUT -H "Metadata-Token-Expiry-Seconds: 3600" http://169.254.169.254/v1/token 2>/dev/null) + if [ -n "\$TOKEN" ]; then + NETWORK_INFO=\$(curl -s --connect-timeout 5 --max-time 10 -H "Metadata-Token: \$TOKEN" http://169.254.169.254/v1/network 2>/dev/null) + # Extract IPv4 public IP from response like "ipv4.public: 45.79.145.23/32 ipv6.link_local: ..." + PUBLIC_IP=\$(echo "\$NETWORK_INFO" | grep -o 'ipv4\\.public: [0-9]*\\.[0-9]*\\.[0-9]*\\.[0-9]*' | cut -d' ' -f2 | cut -d'/' -f1) + fi + + # Fallback to external service if metadata service fails + if [ -z "\$PUBLIC_IP" ]; then + PUBLIC_IP=\$(curl -s --connect-timeout 5 --max-time 10 https://ipinfo.io/ip 2>/dev/null) + fi + + if [ -n "\$PUBLIC_IP" ] && [[ \$PUBLIC_IP =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$ ]]; then + echo "Retrieved public IP: \$PUBLIC_IP" + break + fi + echo "Attempt \$attempt failed, retrying..." + sleep 3 +done + +if [ -z "\$PUBLIC_IP" ]; then + echo "ERROR: Could not retrieve public IP after 10 attempts" + exit 1 +fi + +# Start the agent with all configuration +echo "Starting NANDA agent with PUBLIC_URL: http://\$PUBLIC_IP:$PORT" +sudo -u ubuntu bash -c " + cd /home/ubuntu/nanda-agent-$AGENT_ID + source env/bin/activate + export ANTHROPIC_API_KEY='$ANTHROPIC_API_KEY' + export AGENT_ID='$AGENT_ID' + export AGENT_NAME='$AGENT_NAME' + export AGENT_DOMAIN='$DOMAIN' + export AGENT_SPECIALIZATION='$SPECIALIZATION' + export AGENT_DESCRIPTION='$DESCRIPTION' + export AGENT_CAPABILITIES='$CAPABILITIES' + export REGISTRY_URL='$REGISTRY_URL' + export PUBLIC_URL='http://\$PUBLIC_IP:$PORT' + export PORT='$PORT' + nohup python3 examples/nanda_agent.py > agent.log 2>&1 & +" + +echo "=== NANDA Agent Setup Complete: $AGENT_ID ===" +echo "Agent URL: http://\$PUBLIC_IP:$PORT/a2a" +EOF + +echo "[5/6] Running Linode instance..." +# Check if instance already exists +INSTANCE_ID=$(linode-cli linodes list --label "nanda-agent-$AGENT_ID" --text --no-headers --format="id" | head -n1) +if [ ! -n "$INSTANCE_ID" ]; then + # Launch Linode instance + INSTANCE_ID=$(linode-cli linodes create \ + --type "$INSTANCE_TYPE" \ + --region "$REGION" \ + --image "$IMAGE_ID" \ + --label "nanda-agent-$AGENT_ID" \ + --tags "NANDA-NEST" \ + --root_pass "$ROOT_PASSWORD" \ + --authorized_keys "$(cat ${SSH_KEY_LABEL}.pub)" \ + --firewall_id "$FIREWALL_ID"\ + --text --no-headers --format="id") +fi +echo "โœ… Instance id: $INSTANCE_ID" + +# Wait for instance to be running +echo "Waiting for instance to be running..." +while true; do + STATUS=$(linode-cli linodes view "$INSTANCE_ID" --text --no-headers --format="status") + if [ "$STATUS" = "running" ]; then + break + fi + echo "Instance status: $STATUS, waiting..." + sleep 10 +done + +# Get public IP +PUBLIC_IP=$(linode-cli linodes view "$INSTANCE_ID" --text --no-headers --format="ipv4") +echo "Public IP: $PUBLIC_IP" + +echo "[6/6] Deploying agent (this process may take a few minutes)..." +# Copy the user data script and execute it +scp -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "user_data_${AGENT_ID}.sh" "root@$PUBLIC_IP:/tmp/" +ssh -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "root@$PUBLIC_IP" "chmod +x /tmp/user_data_${AGENT_ID}.sh && /tmp/user_data_${AGENT_ID}.sh" + + +echo "" +echo "๐ŸŽ‰ NANDA Agent Deployment Complete!" +echo "==================================" +echo "Instance ID: $INSTANCE_ID" +echo "Public IP: $PUBLIC_IP" +echo "Root Password: $ROOT_PASSWORD" +echo "Agent URL: http://$PUBLIC_IP:$PORT/a2a" +echo "" +echo "๐Ÿค– Agent ID for A2A Communication: ${AGENT_ID}-[6-char-hex]" +echo "" +echo "๐Ÿ“ž Use this agent in A2A messages:" +echo " @${AGENT_ID}-[hex] your message here" +echo " (The actual hex suffix is generated at runtime)" + +echo "" +echo "๐Ÿงช Test your agent (direct communication):" +echo "curl -X POST http://$PUBLIC_IP:$PORT/a2a \\" +echo " -H \"Content-Type: application/json\" \\" +echo " -d '{\"content\":{\"text\":\"Hello! What can you help me with?\",\"type\":\"text\"},\"role\":\"user\",\"conversation_id\":\"test123\"}'" + +echo "" +echo "๐Ÿงช Test A2A communication (example with another agent):" +echo "curl -X POST http://$PUBLIC_IP:$PORT/a2a \\" +echo " -H \"Content-Type: application/json\" \\" +echo " -d '{\"content\":{\"text\":\"@$AGENT_ID-[6-char-hex] What can you help me with?\",\"type\":\"text\"},\"role\":\"user\",\"conversation_id\":\"test123\"}'" +echo "" +echo "๐Ÿ” SSH Access:" +echo "ssh -i ${SSH_KEY_LABEL} ubuntu@$PUBLIC_IP" +echo "ssh -i ${SSH_KEY_LABEL} root@$PUBLIC_IP" +echo "" +echo "๐Ÿ›‘ To terminate:" +echo "linode-cli linodes delete $INSTANCE_ID" \ No newline at end of file From 2d2f92f6f55a490e76b545a4a60b73919ea10d49 Mon Sep 17 00:00:00 2001 From: xavinso Date: Sat, 11 Oct 2025 15:54:29 -0400 Subject: [PATCH 2/4] Add Akamai multi agent deployment --- .gitignore | 2 + scripts/akamai-multi-agent-deployment.sh | 418 ++++++++++++++++++++++ scripts/akamai-single-agent-deployment.sh | 2 + 3 files changed, 422 insertions(+) create mode 100755 scripts/akamai-multi-agent-deployment.sh diff --git a/.gitignore b/.gitignore index 26ac513..ea3498f 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,8 @@ Pipfile.lock *.pem *.crt *.key +*.pub +*-key # Testing .pytest_cache/ diff --git a/scripts/akamai-multi-agent-deployment.sh b/scripts/akamai-multi-agent-deployment.sh new file mode 100755 index 0000000..ae83fae --- /dev/null +++ b/scripts/akamai-multi-agent-deployment.sh @@ -0,0 +1,418 @@ +#!/bin/bash + +# CONFIGURABLE Akamai Connected Cloud (Linode) Multi-Agent Deployment Script +# This script creates a Linode instance and deploys multiple fully configurable modular NANDA agents +# Usage: bash akamai-multi-agent-deployment.sh [REGISTRY_URL] [REGION] [INSTANCE_TYPE] [ROOT_PASSWORD] + +set -e + +# Parse arguments +ANTHROPIC_API_KEY="$1" +AGENT_CONFIG_JSON="$2" +REGISTRY_URL="${3:-http://registry.chat39.com:6900}" +REGION="${4:-us-east}" +INSTANCE_TYPE="${5:-g6-standard-2}" # 4GB for multiple agents +ROOT_PASSWORD="${6:-}" + +# Validation +if [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$AGENT_CONFIG_JSON" ]; then + echo "โŒ Usage: $0 [REGISTRY_URL] [REGION] [INSTANCE_TYPE] [ROOT_PASSWORD]" + echo "" + echo "Example:" + echo " $0 sk-ant-xxxxx ./scripts/agent_configs/group-01-business-and-finance-experts.json \"http://registry.chat39.com:6900\" us-east g6-standard-2 \"SecurePassword123!\"" + echo "" + echo "Parameters:" + echo " ANTHROPIC_API_KEY: Your Anthropic API key" + echo " AGENT_CONFIG_JSON: Path to JSON file or JSON string with agent configurations" + echo " REGISTRY_URL: Registry URL for agent discovery (default: http://registry.chat39.com:6900)" + echo " REGION: Linode region (default: us-east)" + echo " INSTANCE_TYPE: Linode plan type (default: g6-standard-2 for 4GB RAM)" + echo " ROOT_PASSWORD: Root password for the instance (will be generated if not provided)" + echo "" + echo "Common Linode regions: us-east, us-west, eu-west, ap-south" + echo "Common instance types: g6-standard-1 (2GB), g6-standard-2 (4GB), g6-standard-4 (8GB)" + exit 1 +fi + +# Parse and validate agent config +if [ -f "$AGENT_CONFIG_JSON" ]; then + AGENTS_JSON=$(cat "$AGENT_CONFIG_JSON") +else + AGENTS_JSON="$AGENT_CONFIG_JSON" +fi + +AGENT_COUNT=$(echo "$AGENTS_JSON" | python3 -c "import json, sys; print(len(json.load(sys.stdin)))") +echo "Agents to deploy: $AGENT_COUNT" + +# Validate instance type for agent count +if [ "$AGENT_COUNT" -gt 5 ] && [ "$INSTANCE_TYPE" = "g6-standard-1" ]; then + echo "โš ๏ธ WARNING: g6-standard-1 (2GB) may be insufficient for $AGENT_COUNT agents. Consider g6-standard-2 (4GB) or larger." + read -p "Continue anyway? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi +fi + +# Validate port uniqueness +echo "Validating port configuration..." +DUPLICATE_PORTS=$(echo "$AGENTS_JSON" | python3 -c " +import json, sys +from collections import Counter +agents = json.load(sys.stdin) +ports = [agent['port'] for agent in agents] +duplicates = [port for port, count in Counter(ports).items() if count > 1] +if duplicates: + print(' '.join(map(str, duplicates))) + sys.exit(1) +") + +if [ $? -eq 1 ]; then + echo "โŒ Duplicate ports found: $DUPLICATE_PORTS" + exit 1 +fi + +# Generate secure password if not provided +if [ -z "$ROOT_PASSWORD" ]; then + ROOT_PASSWORD=$(openssl rand -base64 32 | tr -d "=+/" | cut -c1-25) + echo "๐Ÿ”‘ Generated root password: $ROOT_PASSWORD" +fi + +# Configuration +FIREWALL_LABEL="nanda-nest-multi-agents" +SSH_KEY_LABEL="nanda-multi-agent-key" +IMAGE_ID="linode/ubuntu25.04" # Ubuntu 25.04 LTS +DEPLOYMENT_ID=$(date +%Y%m%d-%H%M%S) + +echo "๐Ÿš€ Configurable Akamai Connected Cloud (Linode) Multi-Agent Deployment" +echo "======================================================================" +echo "Deployment ID: $DEPLOYMENT_ID" +echo "Agent Count: $AGENT_COUNT" +echo "Registry URL: $REGISTRY_URL" +echo "Region: $REGION" +echo "Instance Type: $INSTANCE_TYPE" +echo "" + +# Check Linode CLI credentials +echo "[1/6] Checking Linode CLI credentials..." +if ! linode-cli --version >/dev/null 2>&1; then + echo "โŒ Linode CLI not installed. Install it: https://techdocs.akamai.com/cloud-computing/docs/install-and-configure-the-cli" + exit 1 +fi + +CONFIG="$HOME/.config/linode-cli" +if [ ! -s "$CONFIG" ] && [ -z "$LINODE_CLI_TOKEN" ]; then + echo "โŒ Linode CLI not configured. Run 'linode-cli configure' first." + exit 1 +fi +echo "โœ… Linode CLI credentials valid" + +# Setup firewall +echo "[2/6] Setting up firewall..." +FIREWALL_ID=$(linode-cli firewalls list --text --no-headers --format="id,label" | grep "$FIREWALL_LABEL" | cut -f1 || echo "") + # Get agent ports for firewall rules +AGENT_PORTS=$(echo "$AGENTS_JSON" | python3 -c " +import json, sys +agents = json.load(sys.stdin) +ports = [str(agent['port']) for agent in agents] +print(','.join(ports)) +") + +# Create inbound rules JSON for SSH and all agent ports +INBOUND_RULES='[{"protocol": "TCP", "ports": "22", "addresses": {"ipv4": ["0.0.0.0/0"]}, "action": "ACCEPT"}' +IFS=',' read -ra PORTS <<< "$AGENT_PORTS" +for PORT in "${PORTS[@]}"; do + INBOUND_RULES+=', {"protocol": "TCP", "ports": "'$PORT'", "addresses": {"ipv4": ["0.0.0.0/0"]}, "action": "ACCEPT"}' +done +INBOUND_RULES+=']' + +if [ -z "$FIREWALL_ID" ]; then + echo "Creating firewall..." + + linode-cli firewalls create \ + --label "$FIREWALL_LABEL" \ + --rules.inbound_policy DROP \ + --rules.outbound_policy ACCEPT \ + --rules.inbound "$INBOUND_RULES" + + FIREWALL_ID=$(linode-cli firewalls list --text --no-headers --format="id,label" | grep "$FIREWALL_LABEL" | cut -f1 || echo "") + echo "โœ… Created firewall with ports: SSH, $AGENT_PORTS" +else + echo "Using existing firewall..." + # Add any missing agent ports + linode-cli firewalls rules-update "$FIREWALL_ID" --inbound "$INBOUND_RULES" >/dev/null 2>&1 +fi + +echo "โœ… Firewall: $FIREWALL_ID - $FIREWALL_LABEL" + +# Setup SSH key +echo "[3/6] Setting up SSH key..." +if [ ! -f "${SSH_KEY_LABEL}.pub" ]; then + echo "Generating SSH key pair..." + ssh-keygen -t rsa -b 4096 -f "$SSH_KEY_LABEL" -N "" -C "nanda-multi-agent-$DEPLOYMENT_ID" +fi + +# Create user data script with supervisor configuration +echo "[4/6] Creating improved user data script..." +cat > "user_data_multi_${DEPLOYMENT_ID}.sh" << EOF +#!/bin/bash +exec > /var/log/user-data.log 2>&1 + +echo "=== NANDA Multi-Agent Setup Started: $DEPLOYMENT_ID ===" +date + +# Update system and install dependencies +apt-get update -y +apt-get install -y python3 python3-venv python3-pip git curl jq supervisor + +# Create ubuntu user (Linode uses root by default) +useradd -m -s /bin/bash ubuntu +mkdir -p /home/ubuntu/.ssh +cp /root/.ssh/authorized_keys /home/ubuntu/.ssh/authorized_keys 2>/dev/null || true +chown -R ubuntu:ubuntu /home/ubuntu/.ssh +chmod 700 /home/ubuntu/.ssh +chmod 600 /home/ubuntu/.ssh/authorized_keys 2>/dev/null || true + +# Setup project as ubuntu user +cd /home/ubuntu +sudo -u ubuntu git clone https://github.com/projnanda/NEST.git nanda-multi-agents-$DEPLOYMENT_ID +cd nanda-multi-agents-$DEPLOYMENT_ID + +# Create virtual environment and install +sudo -u ubuntu python3 -m venv env +sudo -u ubuntu bash -c "source env/bin/activate && pip install --upgrade pip && pip install -e . && pip install anthropic" + +# Get public IP using Linode metadata service +echo "Getting public IP address..." +for attempt in {1..10}; do + # Linode metadata service + TOKEN=\$(curl -s -X PUT -H "Metadata-Token-Expiry-Seconds: 3600" http://169.254.169.254/v1/token 2>/dev/null) + if [ -n "\$TOKEN" ]; then + NETWORK_INFO=\$(curl -s --connect-timeout 5 --max-time 10 -H "Metadata-Token: \$TOKEN" http://169.254.169.254/v1/network 2>/dev/null) + # Extract IPv4 public IP from response like "ipv4.public: 45.79.145.23/32 ipv6.link_local: ..." + PUBLIC_IP=\$(echo "\$NETWORK_INFO" | grep -o 'ipv4\\.public: [0-9]*\\.[0-9]*\\.[0-9]*\\.[0-9]*' | cut -d' ' -f2 | cut -d'/' -f1) + fi + + # Fallback to external service if metadata service fails + if [ -z "\$PUBLIC_IP" ]; then + PUBLIC_IP=\$(curl -s --connect-timeout 5 --max-time 10 https://ipinfo.io/ip 2>/dev/null) + fi + + if [ -n "\$PUBLIC_IP" ] && [[ \$PUBLIC_IP =~ ^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\$ ]]; then + echo "Retrieved public IP: \$PUBLIC_IP" + break + fi + echo "Attempt \$attempt failed, retrying..." + sleep 3 +done + +if [ -z "\$PUBLIC_IP" ]; then + echo "ERROR: Could not retrieve public IP after 10 attempts" + exit 1 +fi + +# Save agent configuration +cat > /tmp/agents_config.json << 'AGENTS_EOF' +$AGENTS_JSON +AGENTS_EOF + +# Create supervisor configuration for each agent +echo "Creating supervisor configurations..." +mkdir -p /etc/supervisor/conf.d + +while IFS= read -r agent_config; do + AGENT_ID=\$(echo "\$agent_config" | jq -r '.agent_id') + AGENT_NAME=\$(echo "\$agent_config" | jq -r '.agent_name') + DOMAIN=\$(echo "\$agent_config" | jq -r '.domain') + SPECIALIZATION=\$(echo "\$agent_config" | jq -r '.specialization') + DESCRIPTION=\$(echo "\$agent_config" | jq -r '.description') + CAPABILITIES=\$(echo "\$agent_config" | jq -r '.capabilities') + PORT=\$(echo "\$agent_config" | jq -r '.port') + + echo "Configuring supervisor for agent: \$AGENT_ID" + + # Create supervisor configuration file + cat > "/etc/supervisor/conf.d/agent_\$AGENT_ID.conf" << SUPERVISOR_EOF +[program:agent_\$AGENT_ID] +command=/home/ubuntu/nanda-multi-agents-$DEPLOYMENT_ID/env/bin/python examples/nanda_agent.py +directory=/home/ubuntu/nanda-multi-agents-$DEPLOYMENT_ID +user=ubuntu +autostart=true +autorestart=true +startretries=3 +stderr_logfile=/var/log/agent_\$AGENT_ID.err.log +stdout_logfile=/var/log/agent_\$AGENT_ID.out.log +environment= + ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY", + AGENT_ID="\$AGENT_ID", + AGENT_NAME="\$AGENT_NAME", + AGENT_DOMAIN="\$DOMAIN", + AGENT_SPECIALIZATION="\$SPECIALIZATION", + AGENT_DESCRIPTION="\$DESCRIPTION", + AGENT_CAPABILITIES="\$CAPABILITIES", + REGISTRY_URL="$REGISTRY_URL", + PUBLIC_URL="http://\$PUBLIC_IP:\$PORT", + PORT="\$PORT" + +SUPERVISOR_EOF + + echo "โœ… Supervisor config created for agent \$AGENT_ID on port \$PORT" + +done < <(cat /tmp/agents_config.json | jq -c '.[]') + +# Start supervisor and wait for all agents +echo "Starting supervisor..." +systemctl enable supervisor +systemctl start supervisor +supervisorctl reread +supervisorctl update + +# Wait for all agents to start +echo "Waiting for all agents to start..." +sleep 30 + +# Verify all agents are running +echo "Verifying agent status..." +supervisorctl status + +echo "=== NANDA Multi-Agent Setup Complete: $DEPLOYMENT_ID ===" +echo "All agents managed by supervisor on: \$PUBLIC_IP" +EOF + + +echo "[5/6] Running Linode instance..." +# Check if instance already exists +INSTANCE_ID=$(linode-cli linodes list --label "nanda-multi-agent-$DEPLOYMENT_ID" --text --no-headers --format="id" | head -n1) +if [ ! -n "$INSTANCE_ID" ]; then + # Launch Linode instance + INSTANCE_ID=$(linode-cli linodes create \ + --type "$INSTANCE_TYPE" \ + --region "$REGION" \ + --image "$IMAGE_ID" \ + --label "nanda-multi-agent-$DEPLOYMENT_ID" \ + --tags "NANDA-NEST-Multi" \ + --root_pass "$ROOT_PASSWORD" \ + --authorized_keys "$(cat ${SSH_KEY_LABEL}.pub)" \ + --firewall_id "$FIREWALL_ID" \ + --text --no-headers --format="id") +fi +echo "โœ… Instance id: $INSTANCE_ID" + +# Wait for instance to be running +echo "Waiting for instance to be running..." +while true; do + STATUS=$(linode-cli linodes view "$INSTANCE_ID" --text --no-headers --format="status") + if [ "$STATUS" = "running" ]; then + break + fi + echo "Instance status: $STATUS, waiting..." + sleep 10 +done + +# Get public IP +PUBLIC_IP=$(linode-cli linodes view "$INSTANCE_ID" --text --no-headers --format="ipv4") +echo "Public IP: $PUBLIC_IP" + +echo "[6/6] Deploying multi-agent system (this process may take several minutes)..." +# Copy the user data script and execute it +scp -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "user_data_multi_${DEPLOYMENT_ID}.sh" "root@$PUBLIC_IP:/tmp/" +ssh -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "root@$PUBLIC_IP" "chmod +x /tmp/user_data_multi_${DEPLOYMENT_ID}.sh && /tmp/user_data_multi_${DEPLOYMENT_ID}.sh" + +# Cleanup +rm "user_data_multi_${DEPLOYMENT_ID}.sh" + +# Health check all agents +echo "" +echo "๐Ÿ” Performing health checks..." +echo "$AGENTS_JSON" | python3 -c " +import json, sys +try: + import requests + agents = json.load(sys.stdin) + for agent in agents: + url = f'http://$PUBLIC_IP:{agent[\"port\"]}/health' + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + print(f'โœ… {agent[\"agent_id\"]}: Healthy') + else: + print(f'โš ๏ธ {agent[\"agent_id\"]}: HTTP {response.status_code}') + except Exception as e: + print(f'โŒ {agent[\"agent_id\"]}: {str(e)}') +except ImportError: + print('Health check skipped (requests not available)') +" 2>/dev/null || echo "Health check skipped (requests not available)" + +echo "" +echo "๐ŸŽ‰ NANDA Multi-Agent Deployment Complete!" +echo "==========================================" +echo "Deployment ID: $DEPLOYMENT_ID" +echo "Instance ID: $INSTANCE_ID" +echo "Public IP: $PUBLIC_IP" +echo "Root Password: $ROOT_PASSWORD" + +# Display agent URLs +echo "" +echo "๐Ÿค– Agent URLs:" +echo "$AGENTS_JSON" | python3 -c " +import json, sys +agents = json.load(sys.stdin) +for agent in agents: + print(f\" {agent['agent_id']}: http://$PUBLIC_IP:{agent['port']}/a2a\") +" + +# Get actual agent IDs with hex suffixes from logs +echo "" +echo "Getting actual agent IDs (with hex suffixes)..." +ACTUAL_AGENT_IDS="" +sleep 10 +for attempt in {1..3}; do + ACTUAL_AGENT_IDS=$(ssh -i "${SSH_KEY_LABEL}" -o StrictHostKeyChecking=no ubuntu@$PUBLIC_IP \ + "grep 'Generated agent_id:' /var/log/agent_*.out.log 2>/dev/null | cut -d':' -f3 | tr -d ' '" 2>/dev/null || echo "") + if [ -n "$ACTUAL_AGENT_IDS" ]; then + break + fi + echo "Attempt $attempt: Waiting for agent logs..." + sleep 5 +done + +if [ -n "$ACTUAL_AGENT_IDS" ]; then + echo "" + echo "๐Ÿค– Actual Agent IDs for A2A Communication:" + echo "$ACTUAL_AGENT_IDS" | while read -r agent_id; do + if [ -n "$agent_id" ]; then + echo " @$agent_id" + fi + done + echo "" + echo "๐Ÿ“ž Use these in A2A messages:" + echo " Example: @[agent-id] your message here" +else + echo "" + echo "โš ๏ธ Could not retrieve actual agent IDs from logs." + echo "๐Ÿ“ž Agent IDs will be: [base-id]-[6-char-hex]" +fi + +echo "" +echo "๐Ÿงช Test an agent (direct communication):" +FIRST_PORT=$(echo "$AGENTS_JSON" | python3 -c "import json, sys; agents = json.load(sys.stdin); print(agents[0]['port']) if agents else print('6000')") +echo "curl -X POST http://$PUBLIC_IP:$FIRST_PORT/a2a \\" +echo " -H \"Content-Type: application/json\" \\" +echo " -d '{\"content\":{\"text\":\"Hello! What can you help me with?\",\"type\":\"text\"},\"role\":\"user\",\"conversation_id\":\"test123\"}'" + +echo "" +echo "๐Ÿ” SSH Access:" +echo "ssh -i ${SSH_KEY_LABEL} ubuntu@$PUBLIC_IP" +echo "ssh -i ${SSH_KEY_LABEL} root@$PUBLIC_IP" + +echo "" +echo "๐Ÿ“Š Monitor agents:" +echo "ssh -i ${SSH_KEY_LABEL} root@$PUBLIC_IP 'supervisorctl status'" + +echo "" +echo "๐Ÿ”„ Restart all agents:" +echo "ssh -i ${SSH_KEY_LABEL} root@$PUBLIC_IP 'supervisorctl restart all'" + +echo "" +echo "๐Ÿ›‘ To terminate:" +echo "linode-cli linodes delete $INSTANCE_ID" \ No newline at end of file diff --git a/scripts/akamai-single-agent-deployment.sh b/scripts/akamai-single-agent-deployment.sh index 19e58cf..de31e78 100755 --- a/scripts/akamai-single-agent-deployment.sh +++ b/scripts/akamai-single-agent-deployment.sh @@ -238,6 +238,8 @@ echo "[6/6] Deploying agent (this process may take a few minutes)..." scp -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "user_data_${AGENT_ID}.sh" "root@$PUBLIC_IP:/tmp/" ssh -i "$SSH_KEY_LABEL" -o StrictHostKeyChecking=no "root@$PUBLIC_IP" "chmod +x /tmp/user_data_${AGENT_ID}.sh && /tmp/user_data_${AGENT_ID}.sh" +# Cleanup +rm "user_data_${AGENT_ID}.sh" echo "" echo "๐ŸŽ‰ NANDA Agent Deployment Complete!" From 805519dd9489cc684a83488f2f01572f4145f239 Mon Sep 17 00:00:00 2001 From: disleveK Date: Wed, 10 Dec 2025 14:48:23 -0500 Subject: [PATCH 3/4] Add DATA_PATH hook and data loading functionality - Add DATA_PATH parameter to aws-single-agent-deployment.sh - Implement data loading in nanda_agent.py (CSV and JSON support) - Add pandas dependency to setup.py - Create sample test data (tests/sample_hr.csv) - Add documentation for deployment flow and data loading --- DATA_LOADING_IMPLEMENTATION.md | 162 ++++++++++++ DATA_PATH_HOOK_CHANGES.md | 86 +++++++ DEPLOYMENT_ANALYSIS.md | 337 +++++++++++++++++++++++++ DEPLOYMENT_FLOW_ANALYSIS.md | 290 +++++++++++++++++++++ examples/nanda_agent.py | 70 +++++ scripts/aws-single-agent-deployment.sh | 26 +- setup.py | 3 +- tests/sample_hr.csv | 12 + 8 files changed, 979 insertions(+), 7 deletions(-) create mode 100644 DATA_LOADING_IMPLEMENTATION.md create mode 100644 DATA_PATH_HOOK_CHANGES.md create mode 100644 DEPLOYMENT_ANALYSIS.md create mode 100644 DEPLOYMENT_FLOW_ANALYSIS.md create mode 100644 tests/sample_hr.csv diff --git a/DATA_LOADING_IMPLEMENTATION.md b/DATA_LOADING_IMPLEMENTATION.md new file mode 100644 index 0000000..370305b --- /dev/null +++ b/DATA_LOADING_IMPLEMENTATION.md @@ -0,0 +1,162 @@ +# Data Loading Implementation - Step 3 + +## Summary +Modified `examples/nanda_agent.py` to read `DATA_PATH` from environment and load attached data (CSV and JSON support). The agent now loads data on startup and makes it available via `AGENT_CONFIG["data"]`. + +## Changes Made + +### 1. Added Pandas Import (Lines 28-34) +```python +# Try to import pandas for CSV support +try: + import pandas as pd + PANDAS_AVAILABLE = True +except ImportError: + PANDAS_AVAILABLE = False + print("โš ๏ธ Warning: pandas library not available. CSV data loading will be disabled. Install with: pip install pandas") +``` + +### 2. Added DATA_PATH Environment Variable (Line 100) +```python +# Data path from environment +DATA_PATH = os.getenv("DATA_PATH", None) +``` + +### 3. Added load_attached_data() Function (Lines 106-145) +- Supports CSV files (requires pandas) +- Supports JSON files (built-in json module) +- Graceful error handling +- Informative logging + +**Function signature:** +```python +def load_attached_data(path): + """ + Load data from a file path. Supports CSV and JSON formats. + + Args: + path: File path to load data from + + Returns: + Loaded data (DataFrame for CSV, dict/list for JSON) or None if failed + """ +``` + +**Features:** +- CSV: Returns pandas DataFrame, logs shape +- JSON: Returns dict/list, logs keys or item count +- Error handling: FileNotFoundError, general exceptions +- Format validation: Only .csv and .json supported + +### 4. Updated main() Function (Lines 245-257) +Added data loading logic: +```python +# Load attached data if DATA_PATH is provided +attached_data = None +if DATA_PATH: + print(f"๐Ÿ“‚ Loading data from: {DATA_PATH}") + attached_data = load_attached_data(DATA_PATH) + if attached_data is not None: + if PANDAS_AVAILABLE and isinstance(attached_data, pd.DataFrame): + print(f"๐Ÿ“Š Loaded data with shape: {attached_data.shape}") + AGENT_CONFIG["data"] = attached_data + else: + print("โš ๏ธ Data loading failed, continuing without attached data") +else: + print("โ„น๏ธ No DATA_PATH provided; starting agent without attached data") +``` + +### 5. Updated setup.py +Added `pandas` to requirements (line 21): +```python +"pandas" # For CSV data loading +``` + +### 6. Created Sample Test Data +Created `tests/sample_hr.csv` with sample HR data for testing. + +## Data Access + +After loading, data is available in: +- `AGENT_CONFIG["data"]` - Contains the loaded DataFrame (CSV) or dict/list (JSON) +- Can be accessed in agent logic functions via the config parameter + +## Testing + +### Test Without Data +```bash +python examples/nanda_agent.py +``` +Expected output: +``` +โ„น๏ธ No DATA_PATH provided; starting agent without attached data +``` + +### Test With CSV Data +```bash +# Windows PowerShell +$env:DATA_PATH="tests/sample_hr.csv"; python examples/nanda_agent.py + +# Linux/Mac +DATA_PATH=tests/sample_hr.csv python examples/nanda_agent.py +``` +Expected output: +``` +๐Ÿ“‚ Loading data from: tests/sample_hr.csv +โœ… Loaded CSV data with shape: (10, 5) +๐Ÿ“Š Loaded data with shape: (10, 5) +``` + +### Test With JSON Data +```bash +# Create a sample JSON file first +echo '{"key": "value", "numbers": [1, 2, 3]}' > tests/sample.json + +# Then run +DATA_PATH=tests/sample.json python examples/nanda_agent.py +``` +Expected output: +``` +๐Ÿ“‚ Loading data from: tests/sample.json +โœ… Loaded JSON data + Keys: ['key', 'numbers'] +``` + +### Test With Invalid Path +```bash +DATA_PATH=/nonexistent/file.csv python examples/nanda_agent.py +``` +Expected output: +``` +๐Ÿ“‚ Loading data from: /nonexistent/file.csv +โŒ Failed to load data: File not found at /nonexistent/file.csv +โš ๏ธ Data loading failed, continuing without attached data +``` + +## Next Steps (Step 4 - Tomorrow) + +After data loads correctly, expose it via MCP tools: +- Create MCP tool: `get_row_count` +- Description: returns number of rows in attached dataset +- Input: none +- Output: integer + +## Files Modified + +1. `NEST/examples/nanda_agent.py` - Added data loading functionality +2. `NEST/setup.py` - Added pandas to requirements +3. `NEST/tests/sample_hr.csv` - Created sample test data + +## Verification Checklist + +- โœ… DATA_PATH read from environment +- โœ… CSV loading with pandas support +- โœ… JSON loading with built-in json +- โœ… Error handling for missing files +- โœ… Error handling for unsupported formats +- โœ… Logging for data loading status +- โœ… Data added to AGENT_CONFIG["data"] +- โœ… Agent starts normally without data +- โœ… Agent starts normally with data +- โœ… Sample test data created + diff --git a/DATA_PATH_HOOK_CHANGES.md b/DATA_PATH_HOOK_CHANGES.md new file mode 100644 index 0000000..6438b0d --- /dev/null +++ b/DATA_PATH_HOOK_CHANGES.md @@ -0,0 +1,86 @@ +# DATA_PATH Hook Implementation - Step 2 + +## Summary +Added DATA_PATH parameter support to the NEST AWS deployment script. This allows Maria to optionally pass a data path that will be available to the agent as the `DATA_PATH` environment variable. + +## Changes Made + +### 1. Added DATA_PATH Parameter (Line 21) +```bash +DATA_PATH="${12:-}" # Optional data path +``` + +### 2. Updated Usage/Help Text (Lines 25, 28, 38-42) +- Added `[DATA_PATH]` to usage statement +- Added DATA_PATH example in example command +- Added DATA_PATH parameter description + +### 3. Added DATA_PATH to Deployment Output (Line 54) +```bash +echo "Data Path: ${DATA_PATH:-"None (no data attached)"}" +``` + +### 4. Added DATA_PATH Logging in User-Data Script (Lines 169-174) +```bash +# Log data path status +if [ -n "$DATA_PATH" ]; then + echo "DATA_PATH is set to: $DATA_PATH" +else + echo "No DATA_PATH provided; starting agent without attached data." +fi +``` + +### 5. Exported DATA_PATH Environment Variable (Line 191) +```bash +export DATA_PATH='$DATA_PATH' +``` + +## Updated Defaults +- `REGISTRY_URL` default changed to: `http://registry.chat39.com:6900` +- `INSTANCE_TYPE` default changed to: `t3.small` + +## Testing + +### Quick Syntax Check +```bash +bash -n scripts/aws-single-agent-deployment.sh +``` + +### Simulate Script Call (No AWS) +```bash +# Test with DATA_PATH +bash scripts/aws-single-agent-deployment.sh \ + "test-agent" \ + "sk-ant-test" \ + "Test Agent" \ + "testing" \ + "test specialist" \ + "Test description" \ + "testing" \ + "" \ + "6000" \ + "us-east-1" \ + "t3.small" \ + "/data/hr.csv" + +# Test without DATA_PATH (should work with empty 12th arg) +bash scripts/aws-single-agent-deployment.sh \ + "test-agent" \ + "sk-ant-test" \ + "Test Agent" \ + "testing" \ + "test specialist" \ + "Test description" \ + "testing" +``` + +### Verify User-Data Script Generation +After running the script (even with invalid AWS credentials), check the generated `user_data_${AGENT_ID}.sh` file: +- Should contain `export DATA_PATH='...'` line +- Should contain the logging section for DATA_PATH + +## Next Steps +- Agent code (`examples/nanda_agent.py`) can now read `DATA_PATH` from environment +- Data loading logic can be added to agent initialization +- MCP tools can be registered to access the data + diff --git a/DEPLOYMENT_ANALYSIS.md b/DEPLOYMENT_ANALYSIS.md new file mode 100644 index 0000000..b787482 --- /dev/null +++ b/DEPLOYMENT_ANALYSIS.md @@ -0,0 +1,337 @@ +# NEST Deployment Process Analysis + +## Overview +This document provides a complete understanding of the NEST deployment process, including where agent code lives, where MCP tools are registered, how data is currently NOT handled, and where to hook in the "attach data" step. + +--- + +## 1. Agent Code Location + +### Primary Agent Implementation +**Location:** `NEST/examples/nanda_agent.py` + +This is the main agent code that gets deployed. Key characteristics: +- Uses Anthropic Claude LLM for intelligent responses +- Configurable via environment variables (AGENT_ID, AGENT_NAME, AGENT_DOMAIN, etc.) +- Creates agent logic function that processes messages +- Initializes NANDA adapter with agent logic + +**Key Code Structure:** +```python +# Main entry point +def main(): + agent_logic = create_llm_agent_logic(AGENT_CONFIG) + nanda = NANDA( + agent_id=AGENT_CONFIG["agent_id"], + agent_logic=agent_logic, + port=PORT, + registry_url=AGENT_CONFIG["registry_url"], + public_url=AGENT_CONFIG["public_url"], + enable_telemetry=False + ) + nanda.start() +``` + +### Core Framework Components +- **Adapter:** `NEST/nanda_core/core/adapter.py` - Main NANDA class that wraps agent logic +- **Agent Bridge:** `NEST/nanda_core/core/agent_bridge.py` - Handles A2A communication +- **Registry Client:** `NEST/nanda_core/core/registry_client.py` - Registry integration + +### Deployment Reference +In the deployment script (`scripts/aws-single-agent-deployment.sh`), the agent is started at: +- **Line 178:** `nohup python3 examples/nanda_agent.py > agent.log 2>&1 &` + +--- + +## 2. MCP Tools Registration + +### Current State: MCP Tools Exist But Are NOT Integrated + +**MCP Client Location:** `NEST/nanda_core/core/mcp_client.py` + +The MCP client exists with the following capabilities: +- `MCPClient` class for connecting to MCP servers +- `MCPRegistry` class for discovering MCP servers from registry +- Methods to execute queries on MCP servers + +**However, MCP tools are NOT currently registered or used in the agent flow:** + +1. **No MCP Integration in Agent Bridge:** + - `agent_bridge.py` does NOT import or use `MCPClient` + - Agent logic functions do NOT have access to MCP tools + - No MCP tool registration in the agent initialization + +2. **No MCP Integration in Agent Logic:** + - `nanda_agent.py` does NOT import or use MCP tools + - The LLM agent logic does NOT have MCP tools available in its tool list + +3. **MCP Registry Exists But Unused:** + - `registry_client.py` has `get_mcp_servers()` and `get_mcp_server_config()` methods + - These methods are NOT called during agent initialization or runtime + +### Where MCP Tools SHOULD Be Registered (Future Integration Points) + +To integrate MCP tools, you would need to: + +1. **In `nanda_agent.py` (around line 198):** + - Import MCPClient + - Discover available MCP servers from registry + - Connect to MCP servers and get available tools + - Pass tools to Anthropic API in the `create_llm_agent_logic` function + +2. **In `agent_bridge.py` (in `handle_message` method):** + - Add MCP tool execution capability + - Handle tool use responses from LLM + +3. **In deployment script (user_data section):** + - Add environment variables for MCP registry URL + - Configure MCP server connections + +--- + +## 3. How Data is Currently NOT Handled + +### No Data Attachment Mechanism + +**Current State:** The deployment process does NOT handle data attachment at all. + +1. **No Data Parameters in Deployment Script:** + - `aws-single-agent-deployment.sh` accepts NO data-related parameters + - No data files, data URLs, or data sources are passed to the agent + +2. **No Data in Agent Configuration:** + - `nanda_agent.py` does NOT load or process any data files + - No data directories or data sources are configured + - Agent only uses environment variables for personality/behavior config + +3. **No Data in User Data Script:** + - The EC2 user data script (`user_data_${AGENT_ID}.sh`) does NOT: + - Download data files + - Mount data volumes + - Configure data access + - Set up data directories + +4. **No Data in Agent Logic:** + - The `agent_logic` function receives only `(message: str, conversation_id: str)` + - No data context or data access is provided + - Agent cannot query or access external data sources + +### What's Missing for Data Handling + +To add data handling, you would need: +- Data source configuration (files, URLs, databases) +- Data loading mechanism in agent initialization +- Data access methods in agent logic +- Data attachment step in deployment process + +--- + +## 4. Deployment Script Location + +### Primary Deployment Script +**Location:** `NEST/scripts/aws-single-agent-deployment.sh` + +**Purpose:** Deploys a single agent to AWS EC2 + +**Key Sections:** +1. **Lines 1-39:** Argument parsing and validation +2. **Lines 54-101:** AWS infrastructure setup (security groups, key pairs) +3. **Lines 116-183:** User data script creation (the actual deployment logic) +4. **Lines 185-241:** EC2 instance launch and output + +### Other Deployment Scripts +- `NEST/scripts/aws-multi-agent-deployment.sh` - Multi-agent deployment +- `NEST/scripts/deploy-agent.sh` - Deploy to existing server +- `NEST/scripts/akamai-single-agent-deployment.sh` - Akamai deployment +- `NEST/scripts/akamai-multi-agent-deployment.sh` - Akamai multi-agent + +### User Data Script Generation +The deployment script generates a user data script at **lines 118-183** that: +- Installs system dependencies +- Clones NEST repository +- Sets up Python virtual environment +- Configures agent with environment variables +- Starts the agent process + +--- + +## 5. Where to Hook In "Attach Data" Step + +### Recommended Integration Points + +#### Option 1: In Deployment Script (User Data Section) - RECOMMENDED +**Location:** `NEST/scripts/aws-single-agent-deployment.sh`, lines 163-179 + +**Current Code:** +```bash +# Start the agent with all configuration +echo "Starting NANDA agent with PUBLIC_URL: http://\$PUBLIC_IP:$PORT" +sudo -u ubuntu bash -c " + cd /home/ubuntu/nanda-agent-$AGENT_ID + source env/bin/activate + export ANTHROPIC_API_KEY='$ANTHROPIC_API_KEY' + export AGENT_ID='$AGENT_ID' + # ... more env vars ... + nohup python3 examples/nanda_agent.py > agent.log 2>&1 & +" +``` + +**Hook Point - Add BEFORE agent startup:** +```bash +# ATTACH DATA STEP - Add here (around line 163) +echo "Attaching data to agent..." +# Download data files +# Mount data volumes +# Configure data access +# Set DATA_PATH or similar environment variable + +# Then start agent with data available +``` + +**Advantages:** +- Data is available when agent starts +- Can pass data location via environment variables +- Data setup happens once during deployment + +#### Option 2: In Agent Initialization +**Location:** `NEST/examples/nanda_agent.py`, in `main()` function (around line 180) + +**Current Code:** +```python +def main(): + # ... config loading ... + agent_logic = create_llm_agent_logic(AGENT_CONFIG) + nanda = NANDA(...) + nanda.start() +``` + +**Hook Point - Add BEFORE creating agent_logic:** +```python +def main(): + # ... config loading ... + + # ATTACH DATA STEP - Add here + data_path = os.getenv("DATA_PATH", None) + if data_path: + # Load data files + # Initialize data access + # Pass data to agent_logic creation + + agent_logic = create_llm_agent_logic(AGENT_CONFIG) + # ... +``` + +**Advantages:** +- Data loading happens in Python (more flexible) +- Can validate data before agent starts +- Easier to handle different data formats + +#### Option 3: In Deployment Script Arguments +**Location:** `NEST/scripts/aws-single-agent-deployment.sh`, lines 10-20 + +**Current Code:** +```bash +AGENT_ID="$1" +ANTHROPIC_API_KEY="$2" +AGENT_NAME="$3" +# ... more args ... +``` + +**Hook Point - Add new parameter:** +```bash +AGENT_ID="$1" +ANTHROPIC_API_KEY="$2" +# ... existing args ... +DATA_SOURCE="${12:-}" # New parameter for data source +``` + +**Advantages:** +- Data source specified at deployment time +- Can be different for each agent +- Flexible data source types (URL, S3, local path) + +### Recommended Implementation Flow + +1. **Add data parameter to deployment script** (Option 3) +2. **Download/attach data in user data script** (Option 1) +3. **Load data in agent initialization** (Option 2) +4. **Make data available to agent logic** + +### Specific Code Locations for Data Attachment + +**File:** `NEST/scripts/aws-single-agent-deployment.sh` + +**Line 20:** Add data parameter: +```bash +DATA_SOURCE="${12:-}" +``` + +**Line 163-179:** Add data attachment step: +```bash +# Attach data (if provided) +if [ -n "$DATA_SOURCE" ]; then + echo "Attaching data from: $DATA_SOURCE" + # Download from URL, S3, or mount volume + # Set DATA_PATH environment variable + export DATA_PATH="/home/ubuntu/agent-data" +fi +``` + +**File:** `NEST/examples/nanda_agent.py` + +**Line 180-198:** Add data loading: +```python +def main(): + # ... existing config ... + + # Load data if available + data_path = os.getenv("DATA_PATH", None) + if data_path: + # Load and process data + agent_data = load_agent_data(data_path) + AGENT_CONFIG["data"] = agent_data + + agent_logic = create_llm_agent_logic(AGENT_CONFIG) + # ... +``` + +--- + +## Summary + +### Agent Code Location +- **Main agent:** `NEST/examples/nanda_agent.py` +- **Core framework:** `NEST/nanda_core/core/adapter.py` + +### MCP Tools Registration +- **MCP client exists:** `NEST/nanda_core/core/mcp_client.py` +- **NOT currently integrated** into agent flow +- **Would need integration** in `nanda_agent.py` and `agent_bridge.py` + +### Data Handling +- **Currently NOT handled** at all +- **No data parameters** in deployment +- **No data loading** in agent code +- **No data access** in agent logic + +### Deployment Script +- **Location:** `NEST/scripts/aws-single-agent-deployment.sh` +- **User data section:** Lines 118-183 +- **Agent startup:** Line 178 + +### Data Attachment Hook Points +1. **Deployment script arguments** (line 20) - Add DATA_SOURCE parameter +2. **User data script** (line 163) - Download/attach data before agent startup +3. **Agent initialization** (line 180) - Load data in `main()` function +4. **Agent logic creation** (line 198) - Pass data to agent logic + +--- + +## Next Steps for Data Integration + +1. Modify `aws-single-agent-deployment.sh` to accept data source parameter +2. Add data download/attachment step in user data script (before line 178) +3. Modify `nanda_agent.py` to load data from DATA_PATH environment variable +4. Update agent logic to have access to loaded data +5. Test deployment with sample data source + diff --git a/DEPLOYMENT_FLOW_ANALYSIS.md b/DEPLOYMENT_FLOW_ANALYSIS.md new file mode 100644 index 0000000..bdff6ac --- /dev/null +++ b/DEPLOYMENT_FLOW_ANALYSIS.md @@ -0,0 +1,290 @@ +# NEST Deployment Flow Analysis +## Identifying Where to Hook in "Data Attach" Step + +--- + +## Executive Summary + +**Primary Deployment Script:** `NEST/scripts/aws-single-agent-deployment.sh` + +**Agent Entry Point:** `NEST/examples/nanda_agent.py` (main function) + +**Key Finding:** NEST uses a **simple deployment model** - it clones the entire repository and runs `examples/nanda_agent.py` directly. There is **NO separate agent directory creation, file copying, or MCP tool loader** - everything runs from the cloned repository. + +--- + +## Deployment Flow Trace + +### 1. Deployment Script: `scripts/aws-single-agent-deployment.sh` + +**Location:** Lines 118-183 (user-data script generation) + +**What it does:** +1. **Creates EC2 instance** with user-data script +2. **User-data script runs on instance startup** (cloud-init) + +### 2. User-Data Script (Generated at Lines 124-183) + +**Key Steps:** + +#### Step 1: System Setup (Lines 131-133) +```bash +apt-get update -y +apt-get install -y python3 python3-venv python3-pip git curl +``` +- **Purpose:** Install system dependencies +- **No agent directory created here** + +#### Step 2: Clone Repository (Lines 136-138) +```bash +cd /home/ubuntu +sudo -u ubuntu git clone https://github.com/projnanda/NEST.git nanda-agent-$AGENT_ID +cd nanda-agent-$AGENT_ID +``` +- **Purpose:** Clone entire NEST repository +- **Agent directory:** `/home/ubuntu/nanda-agent-$AGENT_ID` +- **This IS the "agent directory"** - it's the entire cloned repo +- **No file copying** - everything is already in the repo + +#### Step 3: Virtual Environment (Lines 141-142) +```bash +sudo -u ubuntu python3 -m venv env +sudo -u ubuntu bash -c "source env/bin/activate && pip install --upgrade pip && pip install -e . && pip install anthropic" +``` +- **Purpose:** Create Python virtual environment +- **Installs:** NEST package and dependencies +- **No MCP tool loader** - MCP is just a dependency (`pip install -e .` installs it) + +#### Step 4: Configure Agent (Line 145) +```bash +sudo -u ubuntu sed -i "s/PORT = 6000/PORT = $PORT/" examples/nanda_agent.py +``` +- **Purpose:** Modify port in agent file +- **This is the ONLY file modification** + +#### Step 5: Get Public IP (Lines 147-167) +- **Purpose:** Retrieve EC2 public IP for registration + +#### Step 6: Set Environment Variables (Lines 178-191) +```bash +sudo -u ubuntu bash -c " + cd /home/ubuntu/nanda-agent-$AGENT_ID + source env/bin/activate + export ANTHROPIC_API_KEY='$ANTHROPIC_API_KEY' + export AGENT_ID='$AGENT_ID' + export AGENT_NAME='$AGENT_NAME' + export AGENT_DOMAIN='$DOMAIN' + export AGENT_SPECIALIZATION='$SPECIALIZATION' + export AGENT_DESCRIPTION='$DESCRIPTION' + export AGENT_CAPABILITIES='$CAPABILITIES' + export REGISTRY_URL='$REGISTRY_URL' + export PUBLIC_URL='http://$PUBLIC_IP:$PORT' + export PORT='$PORT' + export DATA_PATH='$DATA_PATH' # <-- Already added in Step 2 + nohup python3 examples/nanda_agent.py > agent.log 2>&1 & +" +``` +- **Purpose:** Set all environment variables and start agent +- **This is where environment variables are set** +- **Agent starts here** - runs `examples/nanda_agent.py` + +--- + +## Agent Loading Flow: `examples/nanda_agent.py` + +### Entry Point: `main()` function (Line 236) + +**Flow:** + +1. **Configuration Loading** (Lines 238-243) + - Prints agent info + - Loads from environment variables (already set in deployment script) + +2. **Data Loading** (Lines 245-257) โœ… **ALREADY IMPLEMENTED** + ```python + # Load attached data if DATA_PATH is provided + attached_data = None + if DATA_PATH: + print(f"๐Ÿ“‚ Loading data from: {DATA_PATH}") + attached_data = load_attached_data(DATA_PATH) + if attached_data is not None: + if PANDAS_AVAILABLE and isinstance(attached_data, pd.DataFrame): + print(f"๐Ÿ“Š Loaded data with shape: {attached_data.shape}") + AGENT_CONFIG["data"] = attached_data + ``` + - **This is where data attach happens** + - **Already implemented in Step 3** + +3. **Agent Logic Creation** (Line 268) + ```python + agent_logic = create_llm_agent_logic(AGENT_CONFIG) + ``` + - Creates LLM-powered agent logic + - **AGENT_CONFIG["data"] is available here** but not currently used + +4. **NANDA Agent Creation** (Lines 271-277) + ```python + nanda = NANDA( + agent_id=AGENT_CONFIG["agent_id"], + agent_logic=agent_logic, + port=PORT, + registry_url=AGENT_CONFIG["registry_url"], + public_url=AGENT_CONFIG["public_url"], + enable_telemetry=False + ) + ``` + - Creates NANDA adapter instance + - **No MCP tools registered here** (MCP not integrated) + +5. **Agent Start** (Line 281) + ```python + nanda.start() + ``` + - Starts the A2A server + +--- + +## Key Findings + +### โœ… What EXISTS: + +1. **Agent Directory Creation:** + - **Location:** `aws-single-agent-deployment.sh` line 137 + - **Method:** `git clone` creates `/home/ubuntu/nanda-agent-$AGENT_ID` + - **No separate agent directory** - entire repo is the "agent directory" + +2. **Agent Files:** + - **Location:** Already in cloned repository + - **No file copying** - files are in `examples/nanda_agent.py` + - **Only modification:** Port number (line 145) + +3. **Environment Variables:** + - **Location:** `aws-single-agent-deployment.sh` lines 181-191 + - **Method:** `export` statements in bash script + - **DATA_PATH already added** (line 191) + +4. **Data Loading:** + - **Location:** `examples/nanda_agent.py` lines 245-257 + - **Status:** โœ… Already implemented + - **Data available in:** `AGENT_CONFIG["data"]` + +### โŒ What DOES NOT EXIST: + +1. **MCP Tool Loader:** + - **Status:** MCP client exists (`nanda_core/core/mcp_client.py`) but **NOT integrated** + - **No MCP tool registration** in agent initialization + - **No MCP tools exposed** to agent logic + - **Would need to be added** in `nanda_agent.py` main() function + +2. **Separate Agent Runtime:** + - **No `agent_runtime/` directory** + - **No `agent_loader/` directory** + - **No `start.py` or `deploy_agent.py`** (except `scripts/deploy-agent.sh` which is different) + +3. **File Copying Mechanism:** + - **No copying of agent files** - everything runs from cloned repo + - **No agent-specific file isolation** + +--- + +## Exact Points for "Data Attach" Hook + +### โœ… ALREADY IMPLEMENTED: + +**Location 1: Deployment Script - Environment Variable** (Line 191) +```bash +export DATA_PATH='$DATA_PATH' +``` +- **Status:** โœ… Done in Step 2 +- **Purpose:** Pass DATA_PATH to agent process + +**Location 2: Agent Code - Data Loading** (Lines 245-257) +```python +if DATA_PATH: + attached_data = load_attached_data(DATA_PATH) + AGENT_CONFIG["data"] = attached_data +``` +- **Status:** โœ… Done in Step 3 +- **Purpose:** Load data and make it available to agent + +### ๐Ÿ”ง NEEDS IMPLEMENTATION (Step 4 - MCP Tools): + +**Location 3: Agent Code - MCP Tool Registration** (After line 257, before line 268) +```python +# TODO: Register MCP tools if data is available +if AGENT_CONFIG.get("data") is not None: + # Register MCP tools that expose the data + # Example: get_row_count, query_data, etc. + pass +``` +- **Status:** โŒ Not implemented +- **Purpose:** Expose data via MCP tools +- **Would integrate with:** `nanda_core/core/mcp_client.py` + +**Location 4: Agent Logic - Use Data in Responses** (In `create_llm_agent_logic`, around line 114) +```python +def llm_agent_logic(message: str, conversation_id: str) -> str: + # TODO: Use AGENT_CONFIG["data"] if available + # Could pass to LLM as context or use MCP tools + pass +``` +- **Status:** โŒ Not implemented +- **Purpose:** Actually use the loaded data in agent responses + +--- + +## Alternative Deployment Scripts + +### `scripts/deploy-agent.sh` +- **Purpose:** Deploy to existing server (not AWS EC2) +- **Similar flow:** Clone repo, create venv, run agent +- **Key difference:** Creates `run_agent.py` instead of using `examples/nanda_agent.py` +- **Data attach point:** Would need to add DATA_PATH handling here too + +### `scripts/aws-multi-agent-deployment.sh` +- **Purpose:** Deploy multiple agents on one instance +- **Uses supervisor** to manage multiple agent processes +- **Each agent:** Runs `examples/nanda_agent.py` with different env vars +- **Data attach point:** Same as single-agent (line 191 in user-data script) + +--- + +## Summary: Where to Modify + +### For Data Attachment (Already Done โœ…): +1. โœ… **Deployment script:** `scripts/aws-single-agent-deployment.sh` line 191 +2. โœ… **Agent code:** `examples/nanda_agent.py` lines 245-257 + +### For MCP Tool Integration (Step 4 - TODO): +1. **Agent initialization:** `examples/nanda_agent.py` after line 257 + - Register MCP tools that expose the data + - Integrate with `nanda_core/core/mcp_client.py` + +2. **Agent logic:** `examples/nanda_agent.py` in `create_llm_agent_logic()` (around line 114) + - Make data available to LLM via MCP tools + - Or pass data as context to LLM + +3. **Agent bridge:** `nanda_core/core/agent_bridge.py` (optional) + - Handle MCP tool calls in message processing + - Execute MCP tools when requested + +--- + +## Conclusion + +**The deployment flow is simpler than expected:** +- No separate agent directory creation (just git clone) +- No file copying (everything in repo) +- No MCP tool loader (MCP not integrated yet) +- Environment variables set in deployment script +- Agent runs directly from `examples/nanda_agent.py` + +**Data attach is already wired:** +- โœ… DATA_PATH passed via environment variable +- โœ… Data loaded in `nanda_agent.py` main() +- โœ… Data available in `AGENT_CONFIG["data"]` + +**Next step (Step 4):** +- Expose data via MCP tools +- Use data in agent responses + diff --git a/examples/nanda_agent.py b/examples/nanda_agent.py index 3dadfa4..0b81864 100644 --- a/examples/nanda_agent.py +++ b/examples/nanda_agent.py @@ -25,6 +25,14 @@ ANTHROPIC_AVAILABLE = False print("โš ๏ธ Warning: anthropic library not available. Install with: pip install anthropic") +# Try to import pandas for CSV support +try: + import pandas as pd + PANDAS_AVAILABLE = True +except ImportError: + PANDAS_AVAILABLE = False + print("โš ๏ธ Warning: pandas library not available. CSV data loading will be disabled. Install with: pip install pandas") + # ============================================================================= # AGENT CONFIGURATION - Customize this section for different agents # ============================================================================= @@ -88,6 +96,54 @@ def get_agent_config(): # Port configuration - use environment variable or default to 6000 PORT = int(os.getenv("PORT", "6000")) +# Data path from environment +DATA_PATH = os.getenv("DATA_PATH", None) + +# ============================================================================= +# DATA LOADING - Load attached data if DATA_PATH is provided +# ============================================================================= + +def load_attached_data(path): + """ + Load data from a file path. Supports CSV and JSON formats. + + Args: + path: File path to load data from + + Returns: + Loaded data (DataFrame for CSV, dict/list for JSON) or None if failed + """ + if not path: + return None + + try: + if path.endswith(".csv"): + if not PANDAS_AVAILABLE: + print(f"โŒ Cannot load CSV: pandas not available. Install with: pip install pandas") + return None + df = pd.read_csv(path) + print(f"โœ… Loaded CSV data with shape: {df.shape}") + return df + elif path.endswith(".json"): + import json + with open(path, 'r') as f: + data = json.load(f) + print(f"โœ… Loaded JSON data") + if isinstance(data, dict): + print(f" Keys: {list(data.keys())}") + elif isinstance(data, list): + print(f" Items: {len(data)}") + return data + else: + print(f"โš ๏ธ Unsupported file format. Only .csv and .json are supported.") + return None + except FileNotFoundError: + print(f"โŒ Failed to load data: File not found at {path}") + return None + except Exception as e: + print(f"โŒ Failed to load data from {path}: {e}") + return None + # ============================================================================= # LLM-POWERED AGENT LOGIC - Uses Anthropic Claude for intelligent responses # ============================================================================= @@ -186,6 +242,20 @@ def main(): if AGENT_CONFIG['registry_url']: print(f"๐ŸŒ Registry: {AGENT_CONFIG['registry_url']}") + # Load attached data if DATA_PATH is provided + attached_data = None + if DATA_PATH: + print(f"๐Ÿ“‚ Loading data from: {DATA_PATH}") + attached_data = load_attached_data(DATA_PATH) + if attached_data is not None: + if PANDAS_AVAILABLE and isinstance(attached_data, pd.DataFrame): + print(f"๐Ÿ“Š Loaded data with shape: {attached_data.shape}") + AGENT_CONFIG["data"] = attached_data + else: + print("โš ๏ธ Data loading failed, continuing without attached data") + else: + print("โ„น๏ธ No DATA_PATH provided; starting agent without attached data") + # Check for Anthropic API key if not AGENT_CONFIG.get("anthropic_api_key"): print("โš ๏ธ Warning: ANTHROPIC_API_KEY not found in environment variables") diff --git a/scripts/aws-single-agent-deployment.sh b/scripts/aws-single-agent-deployment.sh index 50c565c..d44ea57 100755 --- a/scripts/aws-single-agent-deployment.sh +++ b/scripts/aws-single-agent-deployment.sh @@ -2,7 +2,7 @@ # CONFIGURABLE AWS EC2 + NANDA Agent Deployment Script # This script creates an EC2 instance and deploys a fully configurable modular NANDA agent -# Usage: bash aws-ec2-deploy-simple.sh [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE] +# Usage: bash aws-ec2-deploy-simple.sh [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE] [DATA_PATH] set -e @@ -14,17 +14,18 @@ DOMAIN="$4" SPECIALIZATION="$5" DESCRIPTION="$6" CAPABILITIES="$7" -REGISTRY_URL="${8:-}" +REGISTRY_URL="${8:-http://registry.chat39.com:6900}" PORT="${9:-6000}" REGION="${10:-us-east-1}" -INSTANCE_TYPE="${11:-t3.micro}" +INSTANCE_TYPE="${11:-t3.small}" +DATA_PATH="${12:-}" # Optional data path # Validate inputs if [ -z "$AGENT_ID" ] || [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$AGENT_NAME" ] || [ -z "$DOMAIN" ] || [ -z "$SPECIALIZATION" ] || [ -z "$DESCRIPTION" ] || [ -z "$CAPABILITIES" ]; then - echo "โŒ Usage: $0 [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE]" + echo "โŒ Usage: $0 [REGISTRY_URL] [PORT] [REGION] [INSTANCE_TYPE] [DATA_PATH]" echo "" echo "Example:" - echo " $0 data-scientist sk-ant-xxxxx \"Data Scientist\" \"data analysis\" \"analytical and precise AI assistant\" \"I specialize in data analysis, statistics, and machine learning.\" \"data analysis,statistics,machine learning,Python,R\" \"https://registry.example.com\" 6000 us-east-1 t3.micro" + echo " $0 data-scientist sk-ant-xxxxx \"Data Scientist\" \"data analysis\" \"analytical and precise AI assistant\" \"I specialize in data analysis, statistics, and machine learning.\" \"data analysis,statistics,machine learning,Python,R\" \"https://registry.example.com\" 6000 us-east-1 t3.small \"/data/hr.csv\"" echo "" echo "Parameters:" echo " AGENT_ID: Unique identifier for the agent" @@ -34,7 +35,11 @@ if [ -z "$AGENT_ID" ] || [ -z "$ANTHROPIC_API_KEY" ] || [ -z "$AGENT_NAME" ] || echo " SPECIALIZATION: Brief description of agent's role" echo " DESCRIPTION: Detailed description of the agent" echo " CAPABILITIES: Comma-separated list of capabilities" - echo " REGISTRY_URL: Optional registry URL for agent discovery" + echo " REGISTRY_URL: Optional registry URL for agent discovery (default: http://registry.chat39.com:6900)" + echo " PORT: Optional port number (default: 6000)" + echo " REGION: Optional AWS region (default: us-east-1)" + echo " INSTANCE_TYPE: Optional EC2 instance type (default: t3.small)" + echo " DATA_PATH: Optional data path to attach to agent (e.g., /data/hr.csv or /mnt/hr/)" exit 1 fi @@ -49,6 +54,7 @@ echo "Registry URL: ${REGISTRY_URL:-"None"}" echo "Port: $PORT" echo "Region: $REGION" echo "Instance Type: $INSTANCE_TYPE" +echo "Data Path: ${DATA_PATH:-"None (no data attached)"}" echo "" # Configuration @@ -160,6 +166,13 @@ if [ -z "\$PUBLIC_IP" ]; then exit 1 fi +# Log data path status +if [ -n "$DATA_PATH" ]; then + echo "DATA_PATH is set to: $DATA_PATH" +else + echo "No DATA_PATH provided; starting agent without attached data." +fi + # Start the agent with all configuration echo "Starting NANDA agent with PUBLIC_URL: http://\$PUBLIC_IP:$PORT" sudo -u ubuntu bash -c " @@ -175,6 +188,7 @@ sudo -u ubuntu bash -c " export REGISTRY_URL='$REGISTRY_URL' export PUBLIC_URL='http://\$PUBLIC_IP:$PORT' export PORT='$PORT' + export DATA_PATH='$DATA_PATH' nohup python3 examples/nanda_agent.py > agent.log 2>&1 & " diff --git a/setup.py b/setup.py index c2afdd5..31e6a53 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ def read_requirements(): "mcp", "python-dotenv", "flask-cors", - "psutil" # For system monitoring + "psutil", # For system monitoring + "pandas" # For CSV data loading ] return requirements diff --git a/tests/sample_hr.csv b/tests/sample_hr.csv new file mode 100644 index 0000000..2e19f38 --- /dev/null +++ b/tests/sample_hr.csv @@ -0,0 +1,12 @@ +employee_id,name,department,salary,years_experience +1,John Smith,Engineering,95000,5 +2,Jane Doe,Marketing,85000,3 +3,Bob Johnson,Sales,75000,2 +4,Alice Williams,Engineering,110000,8 +5,Charlie Brown,HR,70000,1 +6,Diana Prince,Marketing,90000,4 +7,Edward Norton,Engineering,105000,7 +8,Fiona Apple,Sales,80000,3 +9,George Washington,Engineering,120000,10 +10,Hannah Montana,HR,72000,2 + From a7bdcbb8783871b85b32fb94db5b2b1ee1d60747 Mon Sep 17 00:00:00 2001 From: disleveK Date: Fri, 12 Dec 2025 08:46:26 -0500 Subject: [PATCH 4/4] Add sample HR dataset and data setup docs for data-attached agents --- DATA_SETUP_INSTRUCTIONS.md | 111 +++++++++++++++++++++++++++++++++++++ data/README.md | 69 +++++++++++++++++++++++ data/hr.csv | 6 ++ 3 files changed, 186 insertions(+) create mode 100644 DATA_SETUP_INSTRUCTIONS.md create mode 100644 data/README.md create mode 100644 data/hr.csv diff --git a/DATA_SETUP_INSTRUCTIONS.md b/DATA_SETUP_INSTRUCTIONS.md new file mode 100644 index 0000000..9b406c5 --- /dev/null +++ b/DATA_SETUP_INSTRUCTIONS.md @@ -0,0 +1,111 @@ +# Creating HR Dataset for NEST Ecosystem + +## Quick Setup (Already Done โœ…) + +The HR dataset has been created at: `NEST/data/hr.csv` + +## Manual Setup Instructions + +If you need to recreate or modify the dataset: + +### Steps: + +1. **Navigate to NEST directory:** + ```bash + cd NEST + ``` + +2. **Create data directory (if it doesn't exist):** + ```bash + # Windows PowerShell + if (-not (Test-Path "data")) { New-Item -ItemType Directory -Path "data" } + + # Linux/Mac + mkdir -p data + ``` + +3. **Create the HR dataset:** + ```bash + # Windows PowerShell + # Create file: data/hr.csv + + # Linux/Mac + nano data/hr.csv + ``` + +4. **Paste this EXACT content:** + ```csv + employee_id,department,salary,years_at_company + 1001,Engineering,145000,5 + 1002,Sales,92000,2 + 1003,HR,86000,4 + 1004,Marketing,78000,1 + ``` + +5. **Save the file:** + - **Windows**: Save in your editor + - **Linux/Mac**: CTRL+O, ENTER, CTRL+X + +## Using the Dataset + +### Local Testing + +```bash +# From NEST directory +DATA_PATH=data/hr.csv python examples/nanda_agent.py +``` + +### AWS Deployment + +```bash +bash scripts/aws-single-agent-deployment.sh \ + "hr-agent" \ + "sk-ant-xxxxx" \ + "HR Assistant" \ + "human resources" \ + "HR specialist" \ + "I help with HR questions and employee data" \ + "HR,employee data,payroll,benefits" \ + "http://registry.chat39.com:6900" \ + "6000" \ + "us-east-1" \ + "t3.small" \ + "data/hr.csv" # <-- 12th parameter: DATA_PATH +``` + +## Dataset Structure + +The HR dataset contains: +- **4 employees** across different departments +- **Columns:** + - `employee_id`: Unique identifier (1001-1004) + - `department`: Engineering, Sales, HR, Marketing + - `salary`: Annual salary (78k-145k) + - `years_at_company`: Years of service (1-5) + +## Purpose + +This dataset allows the HR agent to: +- Demonstrate "having data" +- Respond to questions based on actual employee data +- Show how agents can use attached datasets + +## File Location in NEST + +``` +NEST/ +โ”œโ”€โ”€ data/ +โ”‚ โ”œโ”€โ”€ hr.csv โ† HR dataset (ready to use) +โ”‚ โ””โ”€โ”€ README.md โ† Data directory documentation +โ”œโ”€โ”€ examples/ +โ”‚ โ””โ”€โ”€ nanda_agent.py โ† Agent that loads data +โ””โ”€โ”€ scripts/ + โ””โ”€โ”€ aws-single-agent-deployment.sh โ† Deployment with DATA_PATH +``` + +## Next Steps + +1. โœ… Dataset created at `data/hr.csv` +2. โœ… Agent can load it via `DATA_PATH=data/hr.csv` +3. ๐Ÿ”œ Step 4: Expose data via MCP tools (get_row_count, query_data, etc.) + diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..a19eb71 --- /dev/null +++ b/data/README.md @@ -0,0 +1,69 @@ +# NEST Data Directory + +This directory contains sample datasets that can be attached to NEST agents via the `DATA_PATH` environment variable. + +## HR Dataset + +**File:** `hr.csv` + +A simple HR dataset with employee information: +- `employee_id`: Unique employee identifier +- `department`: Department name (Engineering, Sales, HR, Marketing) +- `salary`: Annual salary +- `years_at_company`: Years of service + +## Usage + +### Local Testing + +```bash +# From NEST directory +DATA_PATH=data/hr.csv python examples/nanda_agent.py +``` + +### AWS Deployment + +```bash +bash scripts/aws-single-agent-deployment.sh \ + "hr-agent" \ + "sk-ant-xxxxx" \ + "HR Assistant" \ + "human resources" \ + "HR specialist" \ + "I help with HR questions and employee data" \ + "HR,employee data,payroll,benefits" \ + "http://registry.chat39.com:6900" \ + "6000" \ + "us-east-1" \ + "t3.small" \ + "data/hr.csv" # <-- DATA_PATH parameter +``` + +### On Deployed Instance + +The data file will need to be available on the EC2 instance. Options: + +1. **Include in repository** (current approach): + - Data is in the repo, so it's available after `git clone` + - Use path: `data/hr.csv` (relative to cloned repo) + +2. **Upload separately**: + - Upload to S3 and download in user-data script + - Or mount EBS volume with data + - Or use a data URL + +## Adding More Datasets + +To add more sample datasets: + +1. Create CSV or JSON file in this directory +2. Update this README with dataset description +3. Use `DATA_PATH=data/your_file.csv` when deploying + +## File Format + +- **CSV**: Must have header row, comma-separated +- **JSON**: Must be valid JSON (object or array) + +The agent will automatically detect the format based on file extension. + diff --git a/data/hr.csv b/data/hr.csv new file mode 100644 index 0000000..4ba7a85 --- /dev/null +++ b/data/hr.csv @@ -0,0 +1,6 @@ +employee_id,department,salary,years_at_company +1001,Engineering,145000,5 +1002,Sales,92000,2 +1003,HR,86000,4 +1004,Marketing,78000,1 +