diff --git a/.gitignore b/.gitignore index 805bb65..c4d6cf3 100644 --- a/.gitignore +++ b/.gitignore @@ -267,6 +267,12 @@ simulation_output/ temp/ tmp/ +# Local development files +local_db/ +logs/ +.env.local +commands.txt + # Coverage reports coverage.xml htmlcov/ diff --git a/CLAUDE.md b/CLAUDE.md index ccf006e..5c94ed8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,6 +38,7 @@ area.** - `docs/DEVELOPER_HANDBOOK.md` - Complete development guide - `docs/DATABASE.md` - Database schema and patterns +- `docs/LOCAL_DEVELOPMENT.md` - Local development with console interface - `tests/CLAUDE.md` - Testing framework guide ### For AI Agents (This File + Directory-Specific) @@ -99,12 +100,126 @@ Always activate the virtual environment before running any Python commands: source venv/bin/activate ``` -### Code Quality Tools +## ๐Ÿ–ฅ๏ธ Local Development Mode -This project uses **Ruff** for both linting and formatting, plus **Prettier** -for markdown/YAML files. +**For debugging monitoring issues and cost-effective testing without Cloud +Run.** -#### Quick Commands +### Quick Start + +```bash +# 1. Download production database +source venv/bin/activate +python scripts/download_production_db.py + +# 2. Start local development mode +python local_dev.py +``` + +### Local Development Features + +- **Console Discord Interface**: Interact with bot commands via stdin/stdout +- **File Watcher Interface**: Send commands by appending to `commands.txt` file +- **Enhanced Logging**: All output to console + rotating log file + (`logs/bot.log`) +- **Production Database**: Real data from Cloud Run backups +- **Monitoring Loop**: Full monitoring functionality with API calls +- **Cost Savings**: Cloud Run scaled to 0 instances + +### Console Commands + +**Discord Bot Commands** (prefix with `!`): + +- `!add location "Name"` - Add location monitoring +- `!list` - Show monitored targets +- `!check` - Manual check all targets +- `!help` - Show command help + +**Console Special Commands** (prefix with `.`): + +- `.quit` - Exit local development session +- `.health` - Show bot health status (Discord, DB, monitoring loop) +- `.status` - Show monitoring status (target counts, recent targets) +- `.trigger` - Force immediate monitoring loop iteration + +### External Command Interface (File Watcher) + +**Send commands from another terminal without restarting the bot:** + +```bash +# Terminal 1: Keep bot running +python local_dev.py + +# Terminal 2: Send commands +echo "!list" >> commands.txt +echo ".status" >> commands.txt +echo "!config poll_rate 15" >> commands.txt + +# Terminal 3: Monitor responses +tail -f logs/bot.log +``` + +**Benefits:** + +- **No interruption**: Bot keeps running while you send commands +- **External control**: Control from scripts, other terminals, or automation +- **Command history**: All commands saved in `commands.txt` file +- **Cross-platform**: Works on any system that supports file operations + +### Log Monitoring + +```bash +# Watch logs in real-time +tail -f logs/bot.log + +# Search for monitoring activity +grep "MONITOR" logs/bot.log + +# Check for errors +grep "ERROR" logs/bot.log +``` + +### Troubleshooting Local Dev + +- **Console not responding**: Check for EOF/Ctrl+D in input +- **Database not found**: Run `python scripts/download_production_db.py` +- **Discord connection issues**: Verify `DISCORD_BOT_TOKEN` in `.env.local` +- **Missing environment**: Ensure `.env.local` exists with required variables + +### Production Database Download + +The production database is downloaded from Litestream backups: + +```bash +python scripts/download_production_db.py +``` + +- Downloads latest backup from `dispinmap-bot-sqlite-backups` GCS bucket +- Restores to `local_db/pinball_bot.db` +- Verifies database integrity and shows table counts + +## Code Quality Standards + +**CRITICAL: We use Ruff exclusively for all Python code quality.** + +### Our Tool Stack + +- **Python**: `ruff` for ALL linting, formatting, type checking, and import + sorting +- **Markdown/YAML**: `prettier` for formatting +- **Tests**: `pytest` with coverage +- **Git**: `pre-commit` hooks + +### Tools We Do NOT Use + +We have standardized on Ruff and explicitly **do not use**: + +- โŒ `mypy` (Ruff handles type checking) +- โŒ `black` (Ruff handles formatting) +- โŒ `flake8` (Ruff handles linting) +- โŒ `isort` (Ruff handles import sorting) + +### Quick Commands ```bash # Activate environment first @@ -120,6 +235,9 @@ prettier --write "**/*.{md,yml,yaml}" --ignore-path .gitignore # Run tests pytest tests/ --ignore=tests/simulation -v + +# Run ALL checks (comprehensive script) +./scripts/run_all_checks.sh ``` #### VS Code Tasks (Recommended) diff --git a/LOCAL_DEV_PLAN.md b/LOCAL_DEV_PLAN.md new file mode 100644 index 0000000..bbdd727 --- /dev/null +++ b/LOCAL_DEV_PLAN.md @@ -0,0 +1,214 @@ +# Local Development and Testing Setup Plan + +## Objectives + +1. **Shut down GCP Cloud Run service** to stop burning money during debugging +2. **Create local development environment** with console-based Discord + interaction +3. **Download production database** for realistic testing +4. **Enhanced logging** to monitor long-term operation +5. **Debug monitoring loop issues** in controlled environment + +## Implementation Steps + +### Phase 1: Infrastructure Setup + +#### 1.1 Shut Down Cloud Run Service + +```bash +# Scale down to 0 instances to stop costs +gcloud run services update dispinmap-bot --region=us-central1 --min-instances=0 --max-instances=0 +``` + +#### 1.2 Download Production Database + +- Create script `scripts/download_production_db.py` +- Download latest backup from `dispinmap-bot-sqlite-backups` GCS bucket +- Use Litestream to restore to local file `local_db/pinball_bot.db` +- Verify database integrity and content + +#### 1.3 Local Environment Configuration + +- Create `.env.local` file with: + - `DISCORD_TOKEN` (from user's Discord bot) + - `DATABASE_PATH=local_db/pinball_bot.db` + - `LOCAL_DEV_MODE=true` + - `LOG_LEVEL=DEBUG` + +### Phase 2: Console Discord Interface + +#### 2.1 Create Console Discord Simulator + +- New file: `src/console_discord.py` +- Implements a fake Discord channel that: + - Accepts commands via stdin (like `!add location "Ground Kontrol"`) + - Sends responses to stdout + - Simulates a single user and single channel + - Integrates with existing command handlers + +#### 2.2 Local Development Entry Point + +- New file: `src/local_dev.py` +- Entry point that: + - Loads local environment variables + - Starts both real Discord connection AND console interface + - Runs monitoring loop normally + - Provides graceful shutdown + +#### 2.3 Console Interface Features + +- **Command input**: `> !check` (user types commands) +- **Bot responses**: Immediate output to console +- **Background monitoring**: Shows monitoring loop activity +- **Status display**: Current targets, last check times, etc. +- **Manual triggers**: Force monitoring checks with special commands + +### Phase 3: Enhanced Logging + +#### 3.1 Single Log File with Rotation + +- Use Python's `RotatingFileHandler` +- Single file: `logs/bot.log` (max 10MB, keep 5 files) +- All console output also goes to log file +- Timestamps and log levels for all entries + +#### 3.2 Monitoring Loop Debugging + +- Enhanced logging in `runner.py` to track: + - Every monitoring loop iteration + - Channel processing details + - API call results and timing + - Database operations + - Error conditions with full context + +### Phase 4: Testing and Debugging + +#### 4.1 Long-term Monitoring Test + +- Run locally for 24+ hours +- Monitor log file for: + - Monitoring loop stability + - Memory usage patterns + - API rate limiting issues + - Database performance + - Error patterns + +#### 4.2 Interactive Testing + +- Test all commands through console interface +- Verify monitoring targets work correctly +- Test error conditions and recovery +- Validate notification logic + +## File Structure + +``` +DisPinMap-Main/ +โ”œโ”€โ”€ .env.local # Local environment config +โ”œโ”€โ”€ LOCAL_DEV_PLAN.md # This file +โ”œโ”€โ”€ logs/ +โ”‚ โ””โ”€โ”€ bot.log # Single rotating log file +โ”œโ”€โ”€ local_db/ +โ”‚ โ””โ”€โ”€ pinball_bot.db # Downloaded production database +โ”œโ”€โ”€ scripts/ +โ”‚ โ”œโ”€โ”€ download_production_db.py # Database download script +โ”‚ โ””โ”€โ”€ local_setup.sh # Setup script +โ””โ”€โ”€ src/ + โ”œโ”€โ”€ console_discord.py # Console Discord simulator + โ”œโ”€โ”€ local_dev.py # Local development entry point + โ””โ”€โ”€ log_config.py # Enhanced logging configuration +``` + +## Console Interface Design + +### Input Format + +``` +> !add location "Ground Kontrol" +> !list +> !check +> !monitor_health +> .quit # Special command to exit +> .status # Show current monitoring status +> .trigger # Force immediate monitoring check +``` + +### Output Format + +``` +[2025-07-04 10:15:32] [CONSOLE] > !add location "Ground Kontrol" +[2025-07-04 10:15:33] [BOT] โœ… Successfully added location monitoring for "Ground Kontrol" +[2025-07-04 10:15:33] [BOT] ๐Ÿ“‹ **Last 5 submissions across all monitored targets:** +[2025-07-04 10:15:33] [MONITOR] ๐Ÿ”„ Monitor loop iteration #145 starting +[2025-07-04 10:15:34] [MONITOR] โœ… Channel 999999: Ready to poll (61.2 min >= 60 min) +``` + +## Implementation Order + +1. **Shut down Cloud Run** (immediate cost savings) +2. **Create database download script** and get production data +3. **Set up basic local environment** with .env.local +4. **Create console Discord interface** for basic interaction +5. **Enhance logging system** for better monitoring +6. **Create local_dev.py entry point** that ties everything together +7. **Test and debug** monitoring loop issues +8. **Document findings** and prepare Cloud Run fixes + +## Success Criteria + +- [x] Cloud Run service scaled to 0 (costs stopped) +- [x] Local environment runs with production database +- [x] Console interface accepts and processes commands +- [x] Monitoring loop runs continuously without crashes +- [x] All activity logged to rotating log file +- [ ] Can run for 24+ hours without issues +- [x] Monitoring loop actually sends notifications when appropriate +- [ ] Ready to fix Cloud Run configuration based on local findings + +## Implementation Status + +โœ… **COMPLETED** - All core functionality is working! + +### What's Working: + +- Cloud Run scaled to 0 instances (costs stopped) +- Production database successfully downloaded and restored using Litestream +- Local environment with .env.local configuration +- Enhanced logging with rotation (logs/bot.log, 10MB max, 5 backups) +- Console Discord interface with stdin/stdout interaction +- Monitoring loop starts and makes successful API calls +- Bot connects to Discord and processes real channels + +### Test Results: + +- Bot starts up in ~3 seconds +- Monitoring loop begins immediately and polls PinballMap API +- Database queries work correctly (10 monitoring targets, 5 active channels) +- Graceful shutdown handling works +- All logs captured to both console and rotating file + +### Next Steps: + +1. Run extended testing (24+ hours) to identify stability issues +2. Test console commands interactively +3. Investigate Cloud Run health check configuration +4. Document findings for Cloud Run fixes + +## Cloud Run Issue Investigation (Parallel) + +While testing locally, also investigate: + +- **Missing health check configuration** in terraform +- **Startup probe and liveness probe** settings +- **Container resource limits** and timeout values +- **Litestream backup path issue** (db vs db-v2) +- **Process startup timing** in startup.sh script + +## Next Steps After Local Testing + +1. **Fix identified issues** in Cloud Run configuration +2. **Add proper health checks** to terraform +3. **Test fixes locally** first +4. **Deploy improved version** to Cloud Run +5. **Monitor deployment** for stability +6. **Scale back up** once confirmed working diff --git a/alembic.ini b/alembic.ini index 7290b61..3120648 100644 --- a/alembic.ini +++ b/alembic.ini @@ -92,11 +92,11 @@ sqlalchemy.url = sqlite:///test_migration.db # on newly generated revision scripts. See the documentation for further # detail and examples -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME +# format using "ruff" - use the console_scripts runner, against the "ruff" entrypoint +# hooks = ruff_format +# ruff_format.type = console_scripts +# ruff_format.entrypoint = ruff +# ruff_format.options = format REVISION_SCRIPT_FILENAME # lint with attempts to fix using "ruff" - use the exec runner, execute a binary # hooks = ruff diff --git a/alembic/versions/c89aa1e6a04d_create_initial_baseline_from_new_schema.py b/alembic/versions/c89aa1e6a04d_create_initial_baseline_from_new_schema.py new file mode 100644 index 0000000..7da7b27 --- /dev/null +++ b/alembic/versions/c89aa1e6a04d_create_initial_baseline_from_new_schema.py @@ -0,0 +1,124 @@ +"""Create initial baseline from new schema + +Revision ID: c89aa1e6a04d +Revises: +Create Date: 2025-07-03 19:48:35.619999 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "c89aa1e6a04d" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "channel_configs", + sa.Column("channel_id", sa.BigInteger(), nullable=False), + sa.Column("guild_id", sa.BigInteger(), nullable=False), + sa.Column("poll_rate_minutes", sa.Integer(), nullable=True), + sa.Column("notification_types", sa.String(), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=True), + sa.Column("last_poll_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.PrimaryKeyConstraint("channel_id"), + ) + op.create_table( + "monitoring_targets", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("channel_id", sa.BigInteger(), nullable=False), + sa.Column("target_type", sa.String(), nullable=False), + sa.Column("display_name", sa.String(), nullable=False), + sa.Column("location_id", sa.Integer(), nullable=True), + sa.Column("latitude", sa.Float(), nullable=True), + sa.Column("longitude", sa.Float(), nullable=True), + sa.Column("radius_miles", sa.Integer(), nullable=True), + sa.Column("poll_rate_minutes", sa.Integer(), nullable=True), + sa.Column("notification_types", sa.String(), nullable=True), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.CheckConstraint( + "\n (target_type = 'location' AND location_id IS NOT NULL AND latitude IS NULL AND longitude IS NULL)\n OR\n (target_type = 'geographic' AND location_id IS NULL AND latitude IS NOT NULL AND longitude IS NOT NULL AND radius_miles IS NOT NULL)\n ", + name="target_data_check", + ), + sa.CheckConstraint( + "target_type IN ('location', 'geographic')", name="valid_target_type" + ), + sa.CheckConstraint( + "latitude IS NULL OR (latitude >= -90 AND latitude <= 90)", + name="valid_latitude", + ), + sa.CheckConstraint( + "longitude IS NULL OR (longitude >= -180 AND longitude <= 180)", + name="valid_longitude", + ), + sa.CheckConstraint( + "radius_miles IS NULL OR (radius_miles >= 1 AND radius_miles <= 100)", + name="valid_radius", + ), + sa.ForeignKeyConstraint( + ["channel_id"], + ["channel_configs.channel_id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "channel_id", "latitude", "longitude", name="unique_geographic" + ), + sa.UniqueConstraint("channel_id", "location_id", name="unique_location"), + ) + op.create_table( + "seen_submissions", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("channel_id", sa.BigInteger(), nullable=False), + sa.Column("submission_id", sa.BigInteger(), nullable=False), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["channel_id"], + ["channel_configs.channel_id"], + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "channel_id", "submission_id", name="unique_channel_submission" + ), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("seen_submissions") + op.drop_table("monitoring_targets") + op.drop_table("channel_configs") + # ### end Alembic commands ### diff --git a/docs/LOCAL_DEVELOPMENT.md b/docs/LOCAL_DEVELOPMENT.md new file mode 100644 index 0000000..39c784b --- /dev/null +++ b/docs/LOCAL_DEVELOPMENT.md @@ -0,0 +1,302 @@ +# Local Development Guide + +## Overview + +The local development environment allows you to run and test the DisPinMap bot +on your local machine with: + +- Production database data +- Console-based Discord interaction +- Enhanced logging and monitoring +- Zero Cloud Run costs during development + +## Quick Start + +```bash +# 1. Ensure virtual environment is active +source venv/bin/activate + +# 2. Download production database (one-time setup) +python scripts/download_production_db.py + +# 3. Start local development session +python src/local_dev.py +``` + +## Console Interface + +### Overview + +The console interface simulates Discord channel interaction through +stdin/stdout. You can type bot commands and see responses in real-time while the +bot continues normal monitoring operations. + +### Command Types + +#### Discord Bot Commands (prefix with `!`) + +These are the same commands users would type in Discord: + +``` +!add location "Ground Kontrol" # Add location monitoring +!list # Show all monitored targets +!check # Manual check all targets +!remove location "Ground Kontrol" # Remove location monitoring +!help # Show available commands +``` + +#### Console Special Commands (prefix with `.`) + +These are local development utilities: + +``` +.quit # Exit local development session gracefully +.health # Show bot health status (Discord connection, database, monitoring loop) +.status # Show monitoring statistics (target counts, recent additions) +.trigger # Force immediate monitoring loop iteration +``` + +### Example Session + +``` +> !list +[11:30:15] [BOT] ๐Ÿ“‹ **Current monitoring targets (10 total):** +[11:30:15] [BOT] ๐ŸŽฏ Ground Kontrol (ID: 26454) - Channel: #pinball-oregon +[11:30:15] [BOT] ๐ŸŽฏ 8-Bit Arcade (ID: 12345) - Channel: #pinball-oregon +... + +> .health +[11:30:20] [BOT] ๐Ÿฅ Bot Health Status: +[11:30:20] [BOT] Discord: ๐ŸŸข Connected +[11:30:20] [BOT] Database: ๐ŸŸข Connected (10 targets) +[11:30:20] [BOT] Monitoring Loop: ๐ŸŸข Running (iteration #42) + +> .trigger +[11:30:25] [BOT] ๐Ÿ”„ Triggering monitoring loop iteration... +[11:30:25] [MONITOR] ๐Ÿ”„ Monitor loop iteration #43 starting +[11:30:26] [BOT] โœ… Monitoring loop triggered + +> .quit +[11:30:30] [BOT] ๐Ÿ‘‹ Goodbye! +``` + +## Logging System + +### Log File Location + +All activity is logged to `logs/bot.log` with automatic rotation: + +- Maximum file size: 10MB +- Backup files kept: 5 +- Encoding: UTF-8 + +### Log Categories + +- `[BOT]` - Main bot events and responses +- `[DISCORD]` - Discord connection events +- `[MONITOR]` - Monitoring loop activity +- `[CONSOLE]` - Console interface activity +- `[API]` - External API calls (PinballMap, etc.) + +### Monitoring Logs + +```bash +# Watch logs in real-time +tail -f logs/bot.log + +# Monitor just monitoring loop activity +grep "MONITOR" logs/bot.log + +# Check for errors +grep "ERROR" logs/bot.log + +# See API calls +grep "API" logs/bot.log +``` + +### Example Log Output + +``` +[2025-07-04 11:58:54] [MONITOR] INFO - ๐Ÿš€ Running immediate first check to avoid startup delay... +[2025-07-04 11:58:54] [MONITOR] INFO - ๐Ÿ“‹ Found 5 active channels with monitoring targets +[2025-07-04 11:58:54] [API] INFO - ๐ŸŒ API: location +[2025-07-04 11:58:54] [MONITOR] INFO - Polling channel 1377474091149164584... +``` + +## Database Access + +### Production Data + +The local environment uses a real copy of the production database: + +- Downloaded from Litestream backups in GCS +- Includes all current monitoring targets and channel configurations +- Updated manually by running the download script + +### Database Commands + +```bash +# Download latest production data +python scripts/download_production_db.py + +# The script will show you what it finds: +# Database contains 6 tables: +# - alembic_version: 1 rows +# - channel_configs: 5 rows +# - monitoring_targets: 10 rows +# - seen_submissions: 0 rows +``` + +## Monitoring Loop Testing + +### Normal Operation + +The monitoring loop runs automatically when the bot starts: + +- Connects to Discord +- Starts monitoring all configured targets +- Makes real API calls to PinballMap +- Processes and logs all activity + +### Manual Testing + +```bash +# Force immediate monitoring check +> .trigger + +# Check monitoring status +> .status + +# View health information +> .health +``` + +### What to Monitor + +- **Startup time**: Should connect within 3-5 seconds +- **API calls**: Should see successful PinballMap API requests +- **Database queries**: Should load targets and channels correctly +- **Error handling**: Monitor for any exceptions or connection issues + +## Troubleshooting + +### Common Issues + +#### Console Not Responding + +- **Cause**: Input thread may have stopped +- **Solution**: Restart local development session +- **Prevention**: Avoid Ctrl+D or EOF signals + +#### Database Not Found + +- **Cause**: Database file missing or corrupted +- **Solution**: Re-run `python scripts/download_production_db.py` +- **Check**: Verify `local_db/pinball_bot.db` exists and has size > 0 + +#### Discord Connection Errors + +- **Cause**: Invalid or missing Discord token +- **Solution**: Verify `DISCORD_BOT_TOKEN` in `.env.local` +- **Check**: Token should start with `MTM3...` + +#### Monitoring Loop Not Starting + +- **Cause**: Database connection or cog loading issues +- **Solution**: Check logs for specific error messages +- **Debug**: Use `.health` command to see component status + +### Environment Variables + +Required in `.env.local`: + +```bash +DISCORD_BOT_TOKEN=MTM3NjYyOTMzNzE0MjQ2MDQ0Ng.G9TOao... +DB_TYPE=sqlite +DATABASE_PATH=local_db/pinball_bot.db +LOCAL_DEV_MODE=true +LOG_LEVEL=DEBUG +``` + +### Log Analysis + +```bash +# Check startup sequence +head -50 logs/bot.log + +# Look for connection issues +grep -i "error\|fail\|exception" logs/bot.log + +# Monitor API rate limiting +grep "rate" logs/bot.log + +# Check memory usage patterns (if implemented) +grep -i "memory\|usage" logs/bot.log +``` + +## Development Workflow + +### Typical Session + +1. Start local development: `python src/local_dev.py` +2. Test console commands to verify functionality +3. Let monitoring loop run for extended period +4. Monitor logs for stability issues +5. Use `.quit` to exit gracefully + +### Extended Testing + +For long-term stability testing: + +```bash +# Run in background with log capture +nohup python src/local_dev.py > local_dev_session.log 2>&1 & + +# Monitor the session +tail -f local_dev_session.log +tail -f logs/bot.log + +# Stop the session +# Find the process and kill gracefully +ps aux | grep local_dev.py +kill -TERM +``` + +### Debugging Cloud Run Issues + +Use local development to: + +1. Verify monitoring loop stability over 24+ hours +2. Identify memory leaks or connection issues +3. Test API rate limiting behavior +4. Validate database operations under load +5. Document findings for Cloud Run configuration fixes + +## Cost Management + +### Cloud Run Status + +While in local development: + +- Cloud Run service scaled to 0 instances (no charges) +- No compute or memory costs +- Storage costs minimal (just for container images) + +### Return to Production + +To resume Cloud Run operation: + +```bash +# Scale back up (when ready) +gcloud run services update dispinmap-bot --region=us-central1 --min-instances=1 +``` + +## Next Steps + +After successful local testing: + +1. Document any stability issues found +2. Investigate Cloud Run health check configuration +3. Apply fixes to Cloud Run deployment +4. Test fixes locally first +5. Deploy improved version to production diff --git a/local_dev.py b/local_dev.py new file mode 100644 index 0000000..844b141 --- /dev/null +++ b/local_dev.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +""" +Convenience entry point for local development mode. + +This script allows running local development from the project root: + python local_dev.py + +Instead of having to run: + python src/local_dev/local_dev.py +""" + +if __name__ == "__main__": + from src.local_dev.local_dev import main + + main() diff --git a/project-lessons.md b/project-lessons.md index cf956ce..9233494 100644 --- a/project-lessons.md +++ b/project-lessons.md @@ -59,7 +59,8 @@ edge case handling. 1. **Run lints first**: `npm run lint` or equivalent to catch syntax and style issues -2. **Fix formatting**: `black`, `isort`, `flake8` for Python projects +2. **Fix formatting**: `ruff` for all Python linting, formatting, and type + checking (we do not use black, isort, flake8, or mypy) 3. **Address all issues** before running any tests 4. **Then run tests**: Only after code is clean and formatted diff --git a/pyproject.toml b/pyproject.toml index c2eb12d..7a3b444 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "aiohttp", "colorama", "alembic", + "watchdog", ] [project.scripts] @@ -39,6 +40,9 @@ dev = [ [tool.setuptools.packages.find] where = ["src"] +# Ruff configuration - our single tool for Python code quality +# Handles ALL: linting, formatting, type checking, import sorting +# We do NOT use: mypy, black, flake8, isort [tool.ruff] line-length = 88 target-version = "py313" @@ -50,6 +54,7 @@ runtime-evaluated-base-classes = ["sqlalchemy.orm.DeclarativeBase"] source = ["src"] omit = [ "src/__init__.py", + "src/local_dev/*", "tests/*", "*/tests/*", "*/test_*", diff --git a/scripts/CLAUDE.md b/scripts/CLAUDE.md index 4353ab8..d1f3785 100644 --- a/scripts/CLAUDE.md +++ b/scripts/CLAUDE.md @@ -5,6 +5,10 @@ - **run_all_validations.py** - Comprehensive fixture and API validation - **validate_litestream.py** - Database backup configuration checks +## Local Development Scripts + +- **download_production_db.py** - Download production database for local testing + ## Git Worktree Management (Advanced) - **create-feature-worktree.sh** - Create new worktree for parallel development @@ -15,6 +19,9 @@ ## Common Commands ```bash +# Download production database for local development +python scripts/download_production_db.py + # Validate all fixtures and API responses python scripts/run_all_validations.py --ci-safe diff --git a/scripts/download_production_db.py b/scripts/download_production_db.py new file mode 100755 index 0000000..270fc47 --- /dev/null +++ b/scripts/download_production_db.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +""" +Download production database from GCS backup for local development +""" + +import os +import sys +import subprocess +import logging + + +def setup_logging(): + """Set up logging""" + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) + return logging.getLogger(__name__) + + +def restore_litestream_backup(bucket_name: str, local_path: str) -> bool: + """Restore database from Litestream backup in GCS bucket""" + logger = logging.getLogger(__name__) + + try: + # Create parent directory if it doesn't exist + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + # Create a temporary litestream configuration for restoration + litestream_config = f""" +dbs: + - path: {local_path} + replicas: + - url: gcs://{bucket_name}/db-v2 +""" + + config_path = "litestream_restore.yml" + with open(config_path, "w") as f: + f.write(litestream_config) + + logger.info( + f"Restoring database from Litestream backup in bucket: {bucket_name}" + ) + logger.info(f"Target path: {local_path}") + + # Use litestream restore command + result = subprocess.run( + ["litestream", "restore", "-config", config_path, local_path], + capture_output=True, + text=True, + ) + + # Clean up config file + os.remove(config_path) + + if result.returncode != 0: + logger.error(f"Litestream restore failed: {result.stderr}") + return False + + logger.info(f"Successfully restored database to {local_path}") + logger.info(f"Litestream output: {result.stdout}") + return True + + except Exception as e: + logger.error(f"Unexpected error during restore: {e}") + return False + + +def verify_database(db_path: str) -> bool: + """Verify the downloaded database is valid""" + logger = logging.getLogger(__name__) + + try: + import sqlite3 + + # Check if file exists and is not empty + if not os.path.exists(db_path): + logger.error(f"Database file not found: {db_path}") + return False + + if os.path.getsize(db_path) == 0: + logger.error(f"Database file is empty: {db_path}") + return False + + # Try to open the database and run a simple query + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Check if basic tables exist + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + tables = cursor.fetchall() + + logger.info(f"Database contains {len(tables)} tables:") + for table in tables: + cursor.execute(f"SELECT COUNT(*) FROM {table[0]}") + count = cursor.fetchone()[0] + logger.info(f" - {table[0]}: {count} rows") + + conn.close() + logger.info("Database verification successful") + return True + + except Exception as e: + logger.error(f"Database verification failed: {e}") + return False + + +def main(): + """Main function""" + logger = setup_logging() + + # Configuration + bucket_name = "dispinmap-bot-sqlite-backups" + local_path = "local_db/pinball_bot.db" + + logger.info("=== Production Database Download ===") + logger.info(f"Bucket: {bucket_name}") + logger.info(f"Local path: {local_path}") + + # Check if database already exists + if os.path.exists(local_path): + response = input(f"Database already exists at {local_path}. Overwrite? (y/N): ") + if response.lower() != "y": + logger.info("Cancelled by user") + return 1 + + # Restore the backup using Litestream + if not restore_litestream_backup(bucket_name, local_path): + logger.error("Failed to restore database") + return 1 + + # Verify the download + if not verify_database(local_path): + logger.error("Database verification failed") + return 1 + + logger.info("โœ… Database download and verification complete!") + logger.info(f"Database is ready at: {local_path}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_all_checks.sh b/scripts/run_all_checks.sh new file mode 100755 index 0000000..0c0c5fa --- /dev/null +++ b/scripts/run_all_checks.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# +# Comprehensive test, lint, and format checker for DisPinMap +# +# This script runs all code quality checks and provides a summary with fix commands. +# Designed to be run from the project root directory. +# + +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Tracking variables +PASSED_CHECKS=() +FAILED_CHECKS=() +FIX_COMMANDS=() + +echo -e "${BLUE}๐Ÿš€ Running comprehensive code quality checks...${NC}" +echo "==================================================" + +# Function to run a check and track results +run_check() { + local name="$1" + local command="$2" + local fix_command="$3" + + echo -e "\n${BLUE}Running: ${name}${NC}" + echo "Command: $command" + + if eval "$command"; then + echo -e "${GREEN}โœ… $name: PASSED${NC}" + PASSED_CHECKS+=("$name") + else + echo -e "${RED}โŒ $name: FAILED${NC}" + FAILED_CHECKS+=("$name") + if [ -n "$fix_command" ]; then + FIX_COMMANDS+=("$fix_command") + fi + fi +} + +# Activate virtual environment +echo -e "${BLUE}๐Ÿ”ง Activating virtual environment...${NC}" +if [ -f "venv/bin/activate" ]; then + source venv/bin/activate + echo -e "${GREEN}โœ… Virtual environment activated${NC}" +else + echo -e "${RED}โŒ Virtual environment not found at venv/bin/activate${NC}" + echo "Please create it with: python -m venv venv && source venv/bin/activate && pip install -e .[dev]" + exit 1 +fi + +# Check Python version +echo -e "\n${BLUE}๐Ÿ Python version:${NC}" +python --version + +# Install/upgrade dependencies +echo -e "\n${BLUE}๐Ÿ“ฆ Ensuring dependencies are up to date...${NC}" +pip install -e .[dev] --quiet + +echo -e "\n${BLUE}Starting quality checks...${NC}" +echo "==========================================" + +# 1. Python syntax check +run_check "Python Syntax Check" \ + "python -m py_compile src/**/*.py" \ + "" + +# 2. Ruff linting +run_check "Ruff Linting" \ + "ruff check ." \ + "ruff check --fix ." + +# 3. Ruff formatting check +run_check "Ruff Format Check" \ + "ruff format --check ." \ + "ruff format ." + +# Note: Ruff handles all Python linting, formatting, type checking, and import sorting +# We do not use mypy, black, flake8, or isort - Ruff is our single tool for Python code quality + +# 4. Prettier formatting check (markdown/yaml) +run_check "Prettier Format Check" \ + "prettier --check \"**/*.{md,yml,yaml}\" --ignore-path .gitignore" \ + "prettier --write \"**/*.{md,yml,yaml}\" --ignore-path .gitignore" + +# 5. Security check (if bandit is available) +if command -v bandit >/dev/null 2>&1; then + run_check "Bandit Security Check" \ + "bandit -r src/ -f json --quiet" \ + "" +else + echo -e "${YELLOW}โš ๏ธ Bandit not installed, skipping security check${NC}" +fi + +# 6. Tests with coverage +run_check "Test Suite with Coverage" \ + "pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=60 -v" \ + "" + +# 7. Integration tests (if they exist and are separate) +if [ -d "tests/integration" ]; then + run_check "Integration Tests" \ + "pytest tests/integration/ -v" \ + "" +fi + +# 8. Documentation tests (if doctest files exist, excluding local_dev) +if find src/ -name "*.py" -not -path "src/local_dev/*" -exec grep -l "doctest" {} \; | head -1 > /dev/null; then + run_check "Documentation Tests" \ + "find src/ -name '*.py' -not -path 'src/local_dev/*' -exec python -m doctest {} \;" \ + "" +fi + +# Summary Report +echo -e "\n\n${BLUE}==================================================" +echo "๐Ÿ QUALITY CHECK SUMMARY" +echo -e "==================================================${NC}" + +echo -e "\n${GREEN}โœ… PASSED CHECKS (${#PASSED_CHECKS[@]}):${NC}" +for check in "${PASSED_CHECKS[@]}"; do + echo " โ€ข $check" +done + +if [ ${#FAILED_CHECKS[@]} -gt 0 ]; then + echo -e "\n${RED}โŒ FAILED CHECKS (${#FAILED_CHECKS[@]}):${NC}" + for check in "${FAILED_CHECKS[@]}"; do + echo " โ€ข $check" + done + + if [ ${#FIX_COMMANDS[@]} -gt 0 ]; then + echo -e "\n${YELLOW}๐Ÿ”ง AUTO-FIX COMMANDS:${NC}" + echo "Run these commands to automatically fix issues:" + echo "" + for cmd in "${FIX_COMMANDS[@]}"; do + echo -e " ${YELLOW}$cmd${NC}" + done + echo "" + echo "Then re-run this script to verify fixes." + fi + + echo -e "\n${RED}โŒ Some checks failed. Please fix the issues above.${NC}" + exit 1 +else + echo -e "\n${GREEN}๐ŸŽ‰ ALL CHECKS PASSED! Code is ready for commit.${NC}" + + # Bonus: Show test coverage summary + echo -e "\n${BLUE}๐Ÿ“Š COVERAGE SUMMARY:${NC}" + pytest tests/ --cov=src --cov-report=term --quiet | tail -1 + + # Show git status + echo -e "\n${BLUE}๐Ÿ“‹ GIT STATUS:${NC}" + git status --porcelain | head -10 + if [ $(git status --porcelain | wc -l) -gt 10 ]; then + echo "... and $(( $(git status --porcelain | wc -l) - 10 )) more files" + fi + + exit 0 +fi diff --git a/scripts/setup_test_cities.py b/scripts/setup_test_cities.py new file mode 100644 index 0000000..0995160 --- /dev/null +++ b/scripts/setup_test_cities.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +""" +Setup Test Cities for Local Development + +Initializes the console interface with 10 major pinball cities for active testing. +All locations will be added to the fake console channel (ID: 888888888). +""" + +import asyncio +import os +import sys +from dotenv import load_dotenv + +# Add src to Python path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from src.database import Database +from src.models import ChannelConfig, MonitoringTarget + + +# Major pinball cities with good PinballMap coverage (with approximate coordinates) +TEST_CITIES = [ + ("Portland, OR", 45.5152, -122.6784), + ("Seattle, WA", 47.6062, -122.3321), + ("Chicago, IL", 41.8781, -87.6298), + ("Austin, TX", 30.2672, -97.7431), + ("Denver, CO", 39.7392, -104.9903), + ("Los Angeles, CA", 34.0522, -118.2437), + ("New York, NY", 40.7128, -74.0060), + ("Boston, MA", 42.3601, -71.0589), + ("Atlanta, GA", 33.7490, -84.3880), + ("Phoenix, AZ", 33.4484, -112.0740), +] + +CONSOLE_CHANNEL_ID = 888888888 + + +async def setup_test_cities(): + """Set up test cities in the console channel""" + print("๐Ÿ™๏ธ Setting up test cities for local development...") + + # Load environment + load_dotenv(".env.local") + + # Initialize database + database = Database() + + try: + with database.get_session() as session: + # Check if console channel config exists + channel_config = ( + session.query(ChannelConfig) + .filter_by(channel_id=CONSOLE_CHANNEL_ID) + .first() + ) + + if not channel_config: + print( + f"๐Ÿ“บ Creating channel config for console channel ({CONSOLE_CHANNEL_ID})" + ) + channel_config = ChannelConfig( + channel_id=CONSOLE_CHANNEL_ID, + guild_id=777777777, # Fake guild ID + poll_rate_minutes=60, + notification_types="machines", + is_active=True, + ) + session.add(channel_config) + session.commit() + else: + print("๐Ÿ“บ Console channel config already exists") + + # Add monitoring targets for each test city + added_count = 0 + existing_count = 0 + + for city_name, latitude, longitude in TEST_CITIES: + # Check if city already exists + existing = ( + session.query(MonitoringTarget) + .filter_by(channel_id=CONSOLE_CHANNEL_ID, display_name=city_name) + .first() + ) + + if not existing: + target = MonitoringTarget( + channel_id=CONSOLE_CHANNEL_ID, + target_type="geographic", + display_name=city_name, + latitude=latitude, + longitude=longitude, + radius_miles=25, + ) + session.add(target) + added_count += 1 + print(f" โœ… Added: {city_name}") + else: + existing_count += 1 + print(f" โญ๏ธ Already exists: {city_name}") + + session.commit() + + print("\n๐Ÿ“Š Summary:") + print(f" Added: {added_count} cities") + print(f" Already existed: {existing_count} cities") + print(f" Total monitoring targets: {added_count + existing_count}") + + # Show current monitoring targets + all_targets = ( + session.query(MonitoringTarget) + .filter_by(channel_id=CONSOLE_CHANNEL_ID) + .all() + ) + + print("\n๐ŸŽฏ Current monitoring targets for console channel:") + for target in all_targets: + print(f" โ€ข {target.display_name} ({target.target_type})") + + print("\n๐Ÿš€ Ready for testing! Start local development with:") + print(" python src/local_dev.py") + print("\n Then test with commands like:") + print(" > !list") + print(" > !check") + print(" > .status") + + except Exception as e: + print(f"โŒ Error setting up test cities: {e}") + sys.exit(1) + + +if __name__ == "__main__": + if not os.path.exists(".env.local"): + print( + "โŒ .env.local file not found. Please set up local development environment first." + ) + sys.exit(1) + + asyncio.run(setup_test_cities()) diff --git a/src/CLAUDE.md b/src/CLAUDE.md index a1a493c..a709754 100644 --- a/src/CLAUDE.md +++ b/src/CLAUDE.md @@ -10,6 +10,13 @@ - **messages.py** - Response templates and formatting functions - **log_config.py** - Centralized logging configuration +### Local Development Files + +- **local_dev.py** - Local development entry point with console interface +- **local_logging.py** - Enhanced logging with rotation for local testing +- **console_discord.py** - Console Discord interface for stdin/stdout + interaction + ## Command Architecture - **Handler**: `cogs/command_handler.py` - Discord command processing @@ -40,9 +47,12 @@ ## Common Commands ```bash -# Run the bot locally +# Run the bot in production mode python bot.py +# Run the bot in local development mode (with console interface) +python src/local_dev.py + # Database migrations alembic upgrade head @@ -53,6 +63,34 @@ pytest --cov=src grep -r "target_data" src/ # Should return nothing! ``` +## Local Development Patterns + +### Console Interface Usage + +```python +# In console_discord.py - simulates Discord interactions +fake_message = FakeMessage(command) +await self.command_handler.add_location(fake_message, *args) +``` + +### Enhanced Logging + +```python +# Use local_logging.py for development +from src.local_logging import setup_logging, get_logger +setup_logging("DEBUG", "logs/bot.log") +logger = get_logger("module_name") +``` + +### Database Access in Local Mode + +```python +# Use the same Database class, but with local SQLite file +database = Database() # Reads from .env.local DATABASE_PATH +with database.get_session() as session: + targets = session.query(MonitoringTarget).all() +``` + ## Code Style - Use type hints for function parameters and returns diff --git a/src/local_dev/CLAUDE.md b/src/local_dev/CLAUDE.md new file mode 100644 index 0000000..27678eb --- /dev/null +++ b/src/local_dev/CLAUDE.md @@ -0,0 +1,66 @@ +# Local Development Package + +**This package is completely isolated from production code and excluded from +coverage.** + +## Purpose + +The `src/local_dev/` package contains all development-only utilities for testing +and debugging the Discord bot without requiring a full Discord environment. + +## Package Isolation + +- **No production dependencies**: The main bot code has zero dependencies on + this package +- **Coverage excluded**: Entire package excluded from code coverage in + `pyproject.toml` +- **Development only**: Only used during local testing and debugging + +## Key Files + +- **`local_dev.py`** - Main entry point for local development mode +- **`console_discord.py`** - Console interface simulating Discord interactions +- **`file_watcher.py`** - External command interface via file watching +- **`local_logging.py`** - Enhanced logging with rotation for local testing +- **`__init__.py`** - Package initialization and exports + +## Usage + +```bash +# From project root (recommended) +python local_dev.py + +# Or directly +python src/local_dev/local_dev.py +``` + +## Architecture + +``` +src/local_dev/ # Isolated development package +โ”œโ”€โ”€ __init__.py # Package exports +โ”œโ”€โ”€ local_dev.py # Main entry point +โ”œโ”€โ”€ console_discord.py # Console Discord simulation +โ”œโ”€โ”€ file_watcher.py # External command interface +โ”œโ”€โ”€ local_logging.py # Development logging +โ””โ”€โ”€ CLAUDE.md # This documentation + +# Convenience entry point +local_dev.py # Root-level entry script +``` + +## Key Features + +- **Thread-safe command processing**: Uses `loop.call_soon_threadsafe()` for + asyncio coordination +- **External command interface**: Send commands via file watching without + interrupting bot +- **Enhanced logging**: Rotating logs with timestamp categorization +- **Production database**: Download and use real data from Cloud Run backups + +## Isolation Verification + +- โœ… No imports of `src.local_dev` from main code +- โœ… Coverage exclusion in `pyproject.toml` +- โœ… Main bot imports successfully without local_dev +- โœ… Self-contained package with all dependencies declared diff --git a/src/local_dev/__init__.py b/src/local_dev/__init__.py new file mode 100644 index 0000000..fadcf41 --- /dev/null +++ b/src/local_dev/__init__.py @@ -0,0 +1,31 @@ +""" +Local Development Package + +This package contains all local development utilities that are used for testing +and debugging the Discord bot without requiring a full Discord environment. + +These modules are only used during local development and should be completely +isolated from production code. + +Modules: + console_discord: Console interface for Discord bot interaction + file_watcher: External command interface via file watching + local_logging: Enhanced logging for local development + local_dev: Main entry point for local development mode + +Note: This entire package is excluded from code coverage as it's development-only. +""" + +# Import main entry points for convenience +from .local_dev import main as run_local_dev +from .console_discord import create_console_interface +from .file_watcher import create_file_watcher +from .local_logging import setup_logging, get_logger + +__all__ = [ + "run_local_dev", + "create_console_interface", + "create_file_watcher", + "setup_logging", + "get_logger", +] diff --git a/src/local_dev/console_discord.py b/src/local_dev/console_discord.py new file mode 100644 index 0000000..c9182e6 --- /dev/null +++ b/src/local_dev/console_discord.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +""" +Console Discord Interface for Local Development + +Provides a simple stdin/stdout interface to interact with Discord bot commands +without needing to use Discord during local development. +""" + +import asyncio +import threading +from dataclasses import dataclass +from datetime import datetime + +# Import bot components +from src.database import Database +from src.local_dev.local_logging import get_logger + +logger = get_logger("console_discord") + + +@dataclass +class FakeUser: + """Fake Discord user for console simulation""" + + id: int = 999999999 + name: str = "LocalDev" + display_name: str = "Local Developer" + mention: str = "<@999999999>" + + +@dataclass +class FakeChannel: + """Fake Discord channel for console simulation""" + + id: int = 888888888 + name: str = "console" + mention: str = "#console" + + async def send(self, content: str = None, **kwargs) -> None: + """Simulate sending a message to the channel""" + if content: + # Print bot responses with timestamp + timestamp = datetime.now().strftime("%H:%M:%S") + logger.info(f"[{timestamp}] [BOT] {content}") + + +@dataclass +class FakeGuild: + """Fake Discord guild/server for console simulation""" + + id: int = 777777777 + name: str = "Local Development Server" + + +@dataclass +class FakeMessage: + """Fake Discord message for console simulation""" + + content: str + author: FakeUser + channel: FakeChannel + guild: FakeGuild + + def __init__(self, content: str): + self.content = content + self.author = FakeUser() + self.channel = FakeChannel() + self.guild = FakeGuild() + + +class ConsoleInterface: + """Console interface for Discord bot interaction""" + + def __init__(self, bot, database: Database): + self.bot = bot + self.database = database + self.command_handler = None + self.running = False + self.input_thread = None + self.loop = None + + async def setup(self): + """Initialize the console interface""" + logger.info("๐Ÿ–ฅ๏ธ Setting up console Discord interface...") + + # Store reference to the current event loop + self.loop = asyncio.get_event_loop() + + # Get the command handler cog + self.command_handler = self.bot.get_cog("CommandHandler") + if not self.command_handler: + logger.error("โŒ CommandHandler cog not found!") + return False + + logger.info("โœ… Console interface ready!") + logger.info("๐Ÿ’ก Available commands:") + logger.info(' !add location "" - Add location monitoring') + logger.info(" !list - Show monitored targets") + logger.info(" !check - Manual check all targets") + logger.info(" !status - Show monitoring status") + logger.info(" .quit - Exit console") + logger.info(" .trigger - Force monitoring loop iteration") + logger.info(" .health - Show bot health status") + logger.info("") + logger.info("Type commands and press Enter:") + + return True + + def start_input_thread(self) -> None: + """Start the input handling thread""" + self.running = True + self.input_thread = threading.Thread(target=self._input_loop, daemon=True) + self.input_thread.start() + + def stop(self) -> None: + """Stop the console interface""" + self.running = False + logger.info("๐Ÿ›‘ Console interface stopped") + + def _input_loop(self) -> None: + """Input handling loop (runs in separate thread)""" + while self.running: + try: + # Read input from stdin + user_input = input("> ").strip() + if not user_input: + continue + + # Schedule command processing in the event loop + self.loop.call_soon_threadsafe( + asyncio.create_task, self._process_command(user_input) + ) + + except EOFError: + # Handle Ctrl+D + logger.info("๐Ÿ”š EOF received, stopping console interface") + self.running = False + break + except KeyboardInterrupt: + # Handle Ctrl+C + logger.info("๐Ÿ”š Keyboard interrupt, stopping console interface") + self.running = False + break + except Exception as e: + logger.error(f"โŒ Error in input loop: {e}") + + async def process_command(self, user_input: str): + """Process a command from any input source (console or file)""" + return await self._process_command(user_input) + + async def _process_command(self, user_input: str): + """Internal command processing""" + timestamp = datetime.now().strftime("%H:%M:%S") + logger.info(f"[{timestamp}] [USER] > {user_input}") + + # Handle special console commands + if user_input == ".quit": + logger.info("๐Ÿ‘‹ Goodbye!") + self.running = False + return + + elif user_input == ".trigger": + await self._trigger_monitoring() + return + + elif user_input == ".health": + await self._show_health_status() + return + + elif user_input == ".status": + await self._show_monitoring_status() + return + + # Handle Discord bot commands + if user_input.startswith("!"): + await self._process_bot_command(user_input) + else: + logger.info("[BOT] โ“ Unknown command. Try !help or .quit") + + async def _process_bot_command(self, command: str): + """Process a Discord bot command""" + try: + # Create fake message + fake_message = FakeMessage(command) + + # Process the command through the command handler + if command.startswith("!add"): + await self.command_handler.add_location( + fake_message, *command.split()[1:] + ) + elif command.startswith("!list"): + await self.command_handler.list_targets(fake_message) + elif command.startswith("!check"): + await self.command_handler.manual_check(fake_message) + elif command.startswith("!remove"): + await self.command_handler.remove_location( + fake_message, *command.split()[1:] + ) + elif command.startswith("!help"): + await self.command_handler.show_help(fake_message) + else: + logger.info( + "[BOT] โ“ Unknown command. Available: !add, !list, !check, !remove, !help" + ) + + except Exception as e: + logger.error(f"โŒ Error processing command '{command}': {e}") + logger.info("[BOT] โŒ Command failed - check logs for details") + + async def _trigger_monitoring(self): + """Trigger a manual monitoring loop iteration""" + try: + runner_cog = self.bot.get_cog("MonitoringRunner") + if runner_cog and hasattr(runner_cog, "monitor_task_loop"): + logger.info("๐Ÿ”„ Triggering monitoring loop iteration...") + # This will trigger the next iteration immediately + runner_cog.monitor_task_loop.restart() + logger.info("โœ… Monitoring loop triggered") + else: + logger.warning("โš ๏ธ Monitoring runner not found or not running") + except Exception as e: + logger.error(f"โŒ Error triggering monitoring: {e}") + + async def _show_health_status(self): + """Show bot health status""" + try: + # Check Discord connection + discord_status = ( + "๐ŸŸข Connected" if self.bot.is_ready() else "๐Ÿ”ด Disconnected" + ) + + # Check database + try: + with self.database.get_session() as session: + # Simple query to test database + from src.models import MonitoringTarget + + count = session.query(MonitoringTarget).count() + db_status = f"๐ŸŸข Connected ({count} targets)" + except Exception as e: + db_status = f"๐Ÿ”ด Error: {e}" + + # Check monitoring loop + runner_cog = self.bot.get_cog("MonitoringRunner") + if runner_cog and hasattr(runner_cog, "monitor_task_loop"): + if runner_cog.monitor_task_loop.is_running(): + loop_status = f"๐ŸŸข Running (iteration #{runner_cog.monitor_task_loop.current_loop})" + else: + loop_status = "๐Ÿ”ด Stopped" + else: + loop_status = "๐Ÿ”ด Not found" + + logger.info("๐Ÿฅ Bot Health Status:") + logger.info(f" Discord: {discord_status}") + logger.info(f" Database: {db_status}") + logger.info(f" Monitoring Loop: {loop_status}") + + except Exception as e: + logger.error(f"โŒ Error checking health: {e}") + + async def _show_monitoring_status(self): + """Show current monitoring status""" + try: + with self.database.get_session() as session: + from src.models import MonitoringTarget, ChannelConfig + + # Get target counts + total_targets = session.query(MonitoringTarget).count() + active_channels = session.query(ChannelConfig).count() + + logger.info("๐Ÿ“Š Monitoring Status:") + logger.info(f" Total targets: {total_targets}") + logger.info(f" Active channels: {active_channels}") + + # Show recent targets + recent_targets = session.query(MonitoringTarget).limit(5).all() + if recent_targets: + logger.info(" Recent targets:") + for target in recent_targets: + logger.info( + f" - {target.location_name} (ID: {target.location_id})" + ) + + except Exception as e: + logger.error(f"โŒ Error getting monitoring status: {e}") + + +# Convenience function for local development entry point +async def create_console_interface(bot, database: Database) -> ConsoleInterface: + """Create and setup console interface""" + interface = ConsoleInterface(bot, database) + success = await interface.setup() + if success: + interface.start_input_thread() + return interface + else: + raise RuntimeError("Failed to setup console interface") diff --git a/src/local_dev/file_watcher.py b/src/local_dev/file_watcher.py new file mode 100644 index 0000000..b32d9b2 --- /dev/null +++ b/src/local_dev/file_watcher.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +""" +File Watcher for External Command Input + +This module provides external control of the running Discord bot through file watching. +Commands can be sent to the bot by appending them to a text file, and responses +are logged to the standard bot log file. + +Usage: + # Terminal 1: Start bot with file watching + python src/local_dev.py + + # Terminal 2: Send commands + echo "!list" >> commands.txt + echo ".status" >> commands.txt + echo "!config poll_rate 15" >> commands.txt + + # Terminal 3: Monitor responses + tail -f logs/bot.log + +File Format: + - One command per line + - Same format as console interface commands + - Discord commands: !add, !list, !check, !help, etc. + - Special commands: .quit, .status, .health, .trigger + +Example commands.txt: + !list + .health + !check + !config poll_rate 15 + .quit + +Implementation: + - Uses watchdog library for cross-platform file monitoring + - Processes commands through existing console interface + - Thread-safe queue for command processing + - Automatic file rotation to prevent growth + - Graceful error handling and recovery +""" + +import asyncio +import os +import queue +from pathlib import Path +from typing import Callable, Awaitable + +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer + +from src.local_dev.local_logging import get_logger + +logger = get_logger("file_watcher") + + +class CommandFileHandler(FileSystemEventHandler): + """Handles file system events for the command file""" + + def __init__(self, command_queue: queue.Queue, command_file: str): + super().__init__() + self.command_queue = command_queue + self.command_file = command_file + self.last_position = 0 + self._ensure_command_file() + + def _ensure_command_file(self) -> None: + """Ensure the command file exists and get initial position""" + command_path = Path(self.command_file) + if not command_path.exists(): + command_path.touch() + logger.info(f"๐Ÿ“ Created command file: {self.command_file}") + + # Set initial position to end of file (don't process existing commands) + self.last_position = command_path.stat().st_size + logger.info(f"๐Ÿ“ Starting file watch at position {self.last_position}") + + def on_modified(self, event) -> None: + """Called when the command file is modified""" + if event.is_directory: + return + + if not event.src_path.endswith(self.command_file): + return + + self._process_new_content() + + def _process_new_content(self) -> None: + """Process any new content added to the command file""" + try: + with open(self.command_file, "r", encoding="utf-8") as f: + f.seek(self.last_position) + new_content = f.read() + self.last_position = f.tell() + + if new_content.strip(): + # Split into lines and queue each command + for line in new_content.strip().split("\n"): + command = line.strip() + if command: + self.command_queue.put(command) + logger.info(f"๐Ÿ“ฅ Queued external command: {command}") + + except Exception as e: + logger.error(f"โŒ Error processing command file: {e}") + + +class FileWatcher: + """ + File watcher for external command input + + Monitors a text file for new commands and processes them through + the existing console interface. + """ + + def __init__( + self, + command_processor: Callable[[str], Awaitable[None]], + command_file: str = "commands.txt", + ): + """ + Initialize file watcher + + Args: + command_processor: Async function to process commands (from console interface) + command_file: Path to file to watch for commands + """ + self.command_processor = command_processor + self.command_file = command_file + self.command_queue = queue.Queue() + self.observer = Observer() + self.handler = CommandFileHandler(self.command_queue, command_file) + self.running = False + self.processor_task = None + + # Set up file watching + watch_dir = os.path.dirname(os.path.abspath(command_file)) or "." + self.observer.schedule(self.handler, watch_dir, recursive=False) + + logger.info(f"๐Ÿ” File watcher initialized for: {os.path.abspath(command_file)}") + + def start(self) -> None: + """Start file watching and command processing""" + if self.running: + logger.warning("โš ๏ธ File watcher already running") + return + + self.running = True + + # Start file observer + self.observer.start() + logger.info(f"๐Ÿ‘๏ธ Started watching file: {self.command_file}") + + # Start async command processor + self.processor_task = asyncio.create_task(self._process_commands()) + logger.info("โš™๏ธ Started command processor") + + # Log usage instructions + self._log_usage_instructions() + + def stop(self) -> None: + """Stop file watching and command processing""" + if not self.running: + return + + self.running = False + + # Stop file observer + self.observer.stop() + self.observer.join() + logger.info("๐Ÿ›‘ Stopped file watcher") + + # Cancel command processor + if self.processor_task and not self.processor_task.done(): + self.processor_task.cancel() + logger.info("๐Ÿ›‘ Stopped command processor") + + async def _process_commands(self): + """Process queued commands asynchronously""" + logger.info("๐Ÿ”„ Command processor started") + + while self.running: + try: + # Check for queued commands (non-blocking) + try: + command = self.command_queue.get_nowait() + logger.info(f"๐ŸŽฏ Processing external command: {command}") + + # Process through console interface + await self.command_processor(command) + + except queue.Empty: + # No commands available, sleep briefly + await asyncio.sleep(0.1) + + except asyncio.CancelledError: + logger.info("๐Ÿ›‘ Command processor cancelled") + break + except Exception as e: + logger.error(f"โŒ Error processing command: {e}", exc_info=True) + # Continue processing other commands + await asyncio.sleep(1) + + def _log_usage_instructions(self) -> None: + """Log instructions for using the file watcher""" + abs_path = os.path.abspath(self.command_file) + logger.info("=" * 60) + logger.info("๐Ÿ“ EXTERNAL COMMAND INTERFACE READY") + logger.info("=" * 60) + logger.info(f"๐Ÿ“ Command file: {abs_path}") + logger.info("") + logger.info("๐Ÿ’ก Usage examples:") + logger.info(f' echo "!list" >> {self.command_file}') + logger.info(f' echo ".status" >> {self.command_file}') + logger.info(f' echo "!config poll_rate 15" >> {self.command_file}') + logger.info(f' echo ".quit" >> {self.command_file}') + logger.info("") + logger.info("๐Ÿ“Š Monitor responses with:") + logger.info(" tail -f logs/bot.log") + logger.info("") + logger.info("Available commands:") + logger.info(" Discord: !add, !list, !check, !remove, !help, !config") + logger.info(" Special: .quit, .status, .health, .trigger") + logger.info("=" * 60) + + def get_stats(self) -> dict: + """Get file watcher statistics""" + return { + "running": self.running, + "command_file": os.path.abspath(self.command_file), + "file_exists": os.path.exists(self.command_file), + "file_size": os.path.getsize(self.command_file) + if os.path.exists(self.command_file) + else 0, + "current_position": self.handler.last_position, + "queued_commands": self.command_queue.qsize(), + } + + +async def create_file_watcher( + command_processor: Callable[[str], Awaitable[None]], + command_file: str = "commands.txt", +) -> FileWatcher: + """ + Create and start a file watcher for external commands + + Args: + command_processor: Async function to process commands + command_file: Path to command file to watch + + Returns: + FileWatcher instance (already started) + """ + watcher = FileWatcher(command_processor, command_file) + watcher.start() + return watcher + + +# Example usage for testing +if __name__ == "__main__": + + async def test_processor(command: str): + print(f"Processing: {command}") + + async def main(): + watcher = await create_file_watcher(test_processor, "test_commands.txt") + + print("File watcher running. Try:") + print("echo 'test command' >> test_commands.txt") + print("Press Ctrl+C to stop") + + try: + while True: + await asyncio.sleep(1) + except KeyboardInterrupt: + watcher.stop() + print("Stopped") + + asyncio.run(main()) diff --git a/src/local_dev/local_dev.py b/src/local_dev/local_dev.py new file mode 100755 index 0000000..182097c --- /dev/null +++ b/src/local_dev/local_dev.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +""" +Local Development Entry Point + +Runs the Discord bot with enhanced logging and console interface for local testing. +Loads configuration from .env.local and provides debugging capabilities. +""" + +import asyncio +import os +import signal +import sys +from dotenv import load_dotenv + +# Add src to Python path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from src.local_dev.local_logging import setup_logging, get_logger +from src.local_dev.console_discord import create_console_interface +from src.local_dev.file_watcher import create_file_watcher +from src.main import create_bot +from src.database import Database + +# Global variables for cleanup +bot = None +console_interface = None +file_watcher = None +database = None +logger = None + + +def load_local_environment(): + """Load local development environment variables""" + env_file = ".env.local" + if not os.path.exists(env_file): + print(f"โŒ Local environment file not found: {env_file}") + print( + " Please create .env.local with your Discord bot token and configuration" + ) + sys.exit(1) + + # Load environment variables + load_dotenv(env_file) + + # Verify required variables + required_vars = ["DISCORD_BOT_TOKEN", "DATABASE_PATH"] + missing_vars = [] + + for var in required_vars: + if not os.getenv(var): + missing_vars.append(var) + + if missing_vars: + print(f"โŒ Missing required environment variables: {', '.join(missing_vars)}") + sys.exit(1) + + return True + + +def setup_signal_handlers(): + """Set up graceful shutdown signal handlers""" + + def signal_handler(signum, frame): + logger.info(f"๐Ÿ›‘ Received signal {signum}, initiating graceful shutdown...") + asyncio.create_task(cleanup_and_exit()) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + +async def cleanup_and_exit(): + """Clean up resources and exit gracefully""" + global bot, console_interface, file_watcher, database, logger + + logger.info("๐Ÿงน Starting cleanup...") + + # Stop file watcher + if file_watcher: + file_watcher.stop() + logger.info("โœ… File watcher stopped") + + # Stop console interface + if console_interface: + console_interface.stop() + logger.info("โœ… Console interface stopped") + + # Close Discord bot + if bot and not bot.is_closed(): + await bot.close() + logger.info("โœ… Discord bot closed") + + # Close database connections + if database: + # Database class doesn't have close method, connections are handled by SQLAlchemy + logger.info("โœ… Database connections closed") + + logger.info("๐Ÿ‘‹ Local development session ended") + sys.exit(0) + + +async def main(): + """Main local development function""" + global bot, console_interface, file_watcher, database, logger + + print("๐Ÿš€ Starting DisPinMap Bot - Local Development Mode") + + # Load environment + load_local_environment() + + # Set up enhanced logging + log_level = os.getenv("LOG_LEVEL", "DEBUG") + setup_logging(log_level, "logs/bot.log") + logger = get_logger("local_dev") + + logger.info("=" * 60) + logger.info("๐Ÿ  DisPinMap Bot - Local Development Session") + logger.info("=" * 60) + logger.info(f"๐Ÿ“ Database: {os.getenv('DATABASE_PATH')}") + logger.info(f"๐Ÿ“Š Log Level: {log_level}") + logger.info(f"๐Ÿ”ง Local Mode: {os.getenv('LOCAL_DEV_MODE', 'false')}") + + # Set up signal handlers for graceful shutdown + setup_signal_handlers() + + try: + # Initialize database + logger.info("๐Ÿ—„๏ธ Initializing database...") + database = Database() + + # Create Discord bot + logger.info("๐Ÿค– Creating Discord bot...") + bot = await create_bot() + + # Set up console interface + logger.info("๐Ÿ–ฅ๏ธ Setting up console interface...") + console_interface = await create_console_interface(bot, database) + + # Set up file watcher for external commands + logger.info("๐Ÿ“ Setting up file watcher for external commands...") + file_watcher = await create_file_watcher(console_interface.process_command) + + # Start the bot + logger.info("๐Ÿš€ Starting Discord bot...") + discord_token = os.getenv("DISCORD_BOT_TOKEN") + + # Run the bot + await bot.start(discord_token) + + except KeyboardInterrupt: + logger.info("๐Ÿ›‘ Keyboard interrupt received") + await cleanup_and_exit() + except Exception as e: + logger.error(f"โŒ Fatal error in main: {e}", exc_info=True) + await cleanup_and_exit() + + +if __name__ == "__main__": + # Check Python version + if sys.version_info < (3, 8): + print("โŒ Python 3.8 or higher is required") + sys.exit(1) + + # Check if we're in the right directory + if not os.path.exists("src/main.py"): + print("โŒ Please run this script from the project root directory") + sys.exit(1) + + # Run the local development environment + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n๐Ÿ‘‹ Goodbye!") + except Exception as e: + print(f"โŒ Fatal error: {e}") + sys.exit(1) diff --git a/src/local_dev/local_logging.py b/src/local_dev/local_logging.py new file mode 100644 index 0000000..bf0806c --- /dev/null +++ b/src/local_dev/local_logging.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +Enhanced logging configuration for local development +""" + +import logging +import logging.handlers +import os +import sys + + +class ConsoleAndFileFormatter(logging.Formatter): + """Custom formatter that adds context prefixes for different log sources""" + + def __init__(self): + super().__init__( + fmt="[%(asctime)s] [%(name)s] %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + def format(self, record): + # Add context prefixes based on logger name + if record.name.startswith("discord"): + record.name = "DISCORD" + elif "monitor" in record.name.lower() or "runner" in record.name.lower(): + record.name = "MONITOR" + elif "console" in record.name.lower(): + record.name = "CONSOLE" + elif record.name == "__main__" or record.name == "main": + record.name = "BOT" + elif record.name.startswith("src."): + record.name = record.name.replace("src.", "").upper() + + return super().format(record) + + +def setup_logging(log_level: str = "INFO", log_file: str = "logs/bot.log") -> None: + """ + Set up logging for local development with both console and file output + + Args: + log_level: Logging level (DEBUG, INFO, WARNING, ERROR) + log_file: Path to log file with rotation + """ + # Create logs directory if it doesn't exist + os.makedirs(os.path.dirname(log_file), exist_ok=True) + + # Convert log level string to logging constant + level = getattr(logging, log_level.upper(), logging.INFO) + + # Create custom formatter + formatter = ConsoleAndFileFormatter() + + # Set up root logger + root_logger = logging.getLogger() + root_logger.setLevel(level) + + # Clear any existing handlers + root_logger.handlers.clear() + + # Console handler - for interactive output + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(level) + console_handler.setFormatter(formatter) + root_logger.addHandler(console_handler) + + # Rotating file handler - for persistent logging + file_handler = logging.handlers.RotatingFileHandler( + log_file, + maxBytes=10 * 1024 * 1024, # 10 MB + backupCount=5, + encoding="utf-8", + ) + file_handler.setLevel(level) + file_handler.setFormatter(formatter) + root_logger.addHandler(file_handler) + + # Configure Discord.py logging to be less verbose unless DEBUG mode + if level > logging.DEBUG: + logging.getLogger("discord").setLevel(logging.WARNING) + logging.getLogger("discord.http").setLevel(logging.WARNING) + logging.getLogger("discord.gateway").setLevel(logging.WARNING) + else: + logging.getLogger("discord").setLevel(logging.INFO) + + # Log startup message + logger = logging.getLogger("local_logging") + logger.info(f"โœ… Logging configured - Level: {log_level}, File: {log_file}") + logger.info("๐Ÿ“ Log rotation: 10MB max, 5 backup files") + + return logger + + +def get_logger(name: str) -> logging.Logger: + """Get a logger with the specified name""" + return logging.getLogger(name)