diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..92bd096 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,111 @@ +name: Build and Deploy + +on: + push: + branches: ['**'] + pull_request: + branches: [master] + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup R + uses: r-lib/actions/setup-r@v2 + with: + r-version: '4.3' + + - name: Install R packages + run: | + R -e "install.packages(c('ggplot2', 'mgcv'), repos='https://cran.rstudio.com/')" + + - name: Verify R installation + run: | + echo "R_HOME=$R_HOME" + R --version + R -e "library(ggplot2); library(mgcv); cat('R packages loaded successfully\n')" + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Install Python dependencies + run: | + uv pip install -r requirements.txt --system + + - name: Verify Python packages + run: | + python -c "import pandas, plotly, seaborn, plotnine, altair, rpy2; print('Python packages imported successfully')" + + - name: Setup Firefox for Selenium + uses: browser-actions/setup-firefox@v1 + + - name: Install geckodriver + run: | + wget -q https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz + tar -xzf geckodriver-v0.34.0-linux64.tar.gz + sudo mv geckodriver /usr/local/bin/ + geckodriver --version + + - name: Setup Xvfb (virtual display) + run: | + sudo apt-get update + sudo apt-get install -y xvfb + + - name: Run tests + run: | + xvfb-run -a python -m pytest tests/ -v + env: + R_HOME: /opt/R/4.3.3/lib/R + + - name: Build site + run: | + xvfb-run -a make travis + env: + R_HOME: /opt/R/4.3.3/lib/R + + - name: Deploy to Netlify (Production) + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + uses: nwtgck/actions-netlify@v3 + with: + publish-dir: ./web + production-deploy: true + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions - ${{ github.event.head_commit.message }}" + enable-pull-request-comment: false + enable-commit-comment: false + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + + - name: Deploy to Netlify (Preview) + if: github.ref != 'refs/heads/master' + uses: nwtgck/actions-netlify@v3 + with: + publish-dir: ./web + production-deploy: false + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Preview deploy from GitHub Actions" + enable-pull-request-comment: true + enable-commit-comment: false + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: site-build + path: web/ + retention-days: 7 diff --git a/.travis/authenticate_plotly.py b/.travis/authenticate_plotly.py index 374b5c6..9425ca3 100644 --- a/.travis/authenticate_plotly.py +++ b/.travis/authenticate_plotly.py @@ -1,3 +1,15 @@ -import plotly -import os -plotly.tools.set_credentials_file(username=os.environ.get('PLOTLY_USER'), api_key=os.environ.get('PLOTLY_API_KEY')) +# DEPRECATED: This file is no longer used. +# +# With plotly v5+, credentials are not required for local rendering. +# Plotly authentication is only needed if uploading to Chart Studio. +# This project renders plots locally to PNG using Kaleido, so no +# authentication is necessary. +# +# Historical code (plotly v4): +# import plotly +# import os +# plotly.tools.set_credentials_file(username=os.environ.get('PLOTLY_USER'), api_key=os.environ.get('PLOTLY_API_KEY')) + +import sys +print("Warning: plotly authentication is no longer needed with plotly v5+", file=sys.stderr) +print("This script is deprecated and can be removed.", file=sys.stderr) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..213f24a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,186 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +pythonplot.com is a static website that provides a visual comparison of different Python plotting libraries (pandas, matplotlib, seaborn, plotnine, plotly, altair) and R's ggplot2 for exploratory data analysis. It serves as a "Rosetta Stone" showing how to create the same plots across different libraries. + +## Architecture + +The site is generated from a Jupyter notebook (Examples.ipynb) that contains tagged code cells. The build process: + +1. Executes the notebook using `jupyter nbconvert` +2. Extracts tagged cells using `render.py` +3. Generates static HTML using Jinja2 templates +4. Outputs PNG images and an HTML file to the `web/` directory + +### Key Components + +- **Examples.ipynb**: Source notebook containing plot examples for each library +- **render.py**: Core build script that: + - Extracts cells tagged with metadata (ex, name:*, package:*) + - Extracts base64 PNG images from cell outputs and saves them to web/img/plots/ + - Parses cell source code and optional markdown comments + - Renders the final HTML using Jinja2 +- **templates/t_index.html**: Jinja2 template for the website +- **web/**: Output directory for generated site +- **INTRO.md**: Markdown content for the site introduction + +### Cell Tagging System + +Notebook cells must be tagged with metadata for the render script to process them: + +``` +ex # Marks cell as an example +name:scatter-plot # Maps to names dict in render.py +package:seaborn # Maps to packages dict in render.py +``` + +Valid plot names and packages are defined in dictionaries at the top of render.py (lines 19-43). + +## Development Commands + +### Setup Environment + +```bash +make dev_environment # Installs Python and R dependencies using uv +make setup # Alternative command (same as dev_environment) +``` + +**Prerequisites:** +- Python 3.11+ installed on system +- R 4.0+ installed separately (via Homebrew, apt, or CRAN) +- uv package manager (auto-installed by make targets if missing) + +**Manual Setup:** +```bash +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install Python dependencies +uv pip install -r requirements.txt + +# Install R packages (ggplot2, mgcv) +./setup_r.sh +``` + +### Build Site + +```bash +make qrender # Quick render from Examples.ipynb without executing +make render # Full build: execute notebook, then render (creates timestamped .ipynb) +make # Full build + S3 upload +``` + +The render process: +1. `make render` runs the notebook with `jupyter nbconvert --execute` +2. Creates a git-commit-stamped copy (Examples..ipynb) +3. Runs `python render.py` to extract cells and generate web/index.html +4. PNG images are extracted and saved to web/img/plots/ with MD5-based filenames + +### Testing + +```bash +make test # Run pytest tests +python -m pytest tests/ +``` + +The test suite (tests/test_plots.py) validates that Examples.ipynb contains all expected plot/package combinations defined in the defined_plots dictionary. + +### Local Development + +After rendering, serve locally: + +```bash +cd web && python -m http.server +``` + +### Image Rendering + +All plots are rendered to static PNG images: +- **Plotly**: Uses Kaleido for local rendering (no authentication required with v5+) +- **Altair**: Uses native rendering or selenium/geckodriver +- **R/ggplot2**: Uses rpy2 to interface with system R installation + +**Note**: Plotly authentication is no longer needed with plotly v5+. The old authentication code in `.travis/authenticate_plotly.py` is deprecated. + +## Adding New Plots + +1. Add the plot name to the `names` dictionary in render.py +2. Add a new cell in Examples.ipynb with code that produces a PNG output +3. Tag the cell with: `ex`, `name:`, `package:` +4. Keep code lines under ~46 characters to avoid horizontal scrolling in the UI +5. Optionally add a markdown comment in triple quotes on the first line +6. Update tests/test_plots.py to include the new plot in defined_plots +7. Run `make qrender` to test (or `make render` for full rebuild) + +## Technical Constraints + +- Plot code must generate PNG output in the notebook cell +- For plotly, images must be generated via their server (requires credentials) +- R code cells must start with `%%R` magic command +- Altair cells must start with `%%altair` magic command +- Code lines should wrap to ~46 characters for proper display +- All image paths are MD5 hashes of the base64-encoded PNG data + +## Dependencies + +### Python Environment +- **Python**: 3.11+ (specified in runtime.txt and pyproject.toml) +- **Package Manager**: uv (modern, fast alternative to pip) +- **Jupyter**: For notebook execution + +### Plotting Libraries (all latest versions) +- pandas 2.0+ +- matplotlib 3.7+ +- seaborn 0.13+ +- plotnine 0.13+ +- plotly 5.24+ (with Kaleido for image export) +- altair 5.0+ +- statsmodels 0.14+ + +### R Environment +- **R**: 4.0+ (system installation required) +- **R Packages**: ggplot2, mgcv +- **Python-R Bridge**: rpy2 3.5+ + +### Other Key Dependencies +- Jinja2 with jinja2-highlight for templating +- selenium 4.15+ with geckodriver for browser automation +- markdown for text processing + +**Configuration Files:** +- `pyproject.toml`: Modern Python project metadata and dependencies +- `requirements.txt`: Pin-free dependency list +- `setup_r.sh`: R package installation script + +## CI/CD + +The project uses **GitHub Actions** for continuous integration and deployment (migrated from Travis CI). + +### Workflow (.github/workflows/deploy.yml) + +**On Every Push:** +1. Setup R 4.3 using r-lib/setup-r action +2. Install R packages (ggplot2, mgcv) +3. Setup Python 3.11 +4. Install uv and Python dependencies +5. Setup Firefox and geckodriver for selenium +6. Run pytest tests with xvfb (virtual display) +7. Execute notebook and render site +8. Deploy to Netlify: + - **master branch**: Production deployment + - **Other branches**: Preview deployments + +**Required Secrets:** +- `NETLIFY_AUTH_TOKEN`: Netlify authentication token +- `NETLIFY_SITE_ID`: Netlify site identifier + +**Environment Variables:** +- `R_HOME`: Set to R installation path (e.g., `/opt/R/4.3.3/lib/R` on GitHub runners) + +### Legacy CI Files +- `.travis.yml`: Old Travis CI config (deprecated) +- `.travis/`: Old CI scripts (mostly deprecated) +- `.travis/authenticate_plotly.py`: No longer needed with plotly v5+ diff --git a/MIGRATION_NOTES.md b/MIGRATION_NOTES.md new file mode 100644 index 0000000..afd27ad --- /dev/null +++ b/MIGRATION_NOTES.md @@ -0,0 +1,282 @@ +# Migration Notes: Modernization to 2025 + +This document summarizes the modernization work completed on pythonplot.com. + +## Overview + +The project has been modernized from 2020-era dependencies and infrastructure to 2025 standards: +- Python 3.6 → 3.11 +- Conda → uv package manager +- Travis CI → GitHub Actions +- Outdated plotting libraries → Latest versions +- Deprecated packages removed + +## Changes Made + +### 1. Python Environment Modernization + +**New Files:** +- `pyproject.toml`: Modern Python project configuration with all dependencies +- `setup_r.sh`: Script to install R packages (ggplot2, mgcv) + +**Updated Files:** +- `runtime.txt`: 3.6 → 3.11 +- `requirements.txt`: All packages updated to latest versions with minimum version constraints + +**Removed Dependencies:** +- `altair-saver` → Deprecated, using native altair rendering +- `chart-studio` → Not needed with plotly v5 local rendering +- `plotly-orca` → Replaced by Kaleido +- Duplicate `selenium` entry removed + +**Updated Dependencies:** +| Package | Old Version | New Version | Notes | +|---------|-------------|-------------|-------| +| plotly | 4.10.0 | 5.24+ | No authentication needed for local rendering | +| plotnine | <0.7.0 | 0.13+ | Better ggplot2 compatibility | +| statsmodels | 0.11.1 | 0.14+ | Security fixes | +| seaborn | (latest) | 0.13+ | Updated API | +| altair | (latest) | 5.0+ | Major version bump | +| rpy2 | (unversioned) | 3.5+ | Python 3.11+ compatible | +| selenium | (duplicated) | 4.15+ | API changes in v4 | +| pandas | (latest) | 2.0+ | Major improvements | +| matplotlib | (latest) | 3.7+ | Modern features | + +### 2. R Integration Updates + +**Approach:** Separate system R installation (no longer using conda) + +**Changes:** +- Created `setup_r.sh` to install R packages +- R must be installed separately (via Homebrew, apt, or CRAN) +- rpy2 updated to work with system R and Python 3.11+ +- Documentation updated to explain R setup + +**Environment Variable:** `R_HOME` must be set correctly for rpy2 to work + +### 3. Build System Updates + +**Makefile Changes:** +- Removed `plotly_auth` target (no longer needed) +- Added `setup` target for uv-based dependency installation +- Updated `dev_environment` to use uv instead of conda +- Updated `.PHONY` declarations + +**What Didn't Change:** +- Core build targets (`render`, `qrender`, `test`, `run_nb`) +- render.py script (compatible with new libraries) +- Notebook structure and tagging system + +### 4. CI/CD Migration + +**From:** Travis CI +**To:** GitHub Actions + +**New File:** `.github/workflows/deploy.yml` + +**Workflow Features:** +- Uses r-lib/setup-r for R installation +- Uses uv for Python dependencies +- Runs on every push and pull request +- Automatic Netlify deployment: + - Production: master branch + - Preview: all other branches + +**Required GitHub Secrets:** +- `NETLIFY_AUTH_TOKEN` +- `NETLIFY_SITE_ID` + +**Legacy Files (Now Deprecated):** +- `.travis.yml` +- `.travis/authenticate_plotly.py` (marked deprecated with explanation) +- `.travis/install_conda.sh` (wasn't being used) +- `.travis/invalidate_cloudfront.py` (not needed with Netlify) + +### 5. Documentation Updates + +**README.md:** +- Added modern development setup instructions +- Updated to reflect uv usage +- Added build and CI/CD sections +- Clarified plotly no longer needs authentication + +**CLAUDE.md:** +- Updated all dependency versions +- Documented new setup process +- Added CI/CD section +- Updated technical constraints + +**New Documentation:** +- `TESTING_CHECKLIST.md`: Comprehensive testing guide for first build +- `MIGRATION_NOTES.md`: This file + +## Next Steps + +### Before First Build + +1. **Install System Dependencies:** + ```bash + # macOS + brew install r + + # Ubuntu/Debian + sudo apt-get install r-base r-base-dev + ``` + +2. **Set Up Development Environment:** + ```bash + make dev_environment + ``` + +3. **Verify Setup:** + ```bash + # Test Python packages + python -c "import pandas, plotly, seaborn, plotnine, altair, rpy2; print('✓ Python packages OK')" + + # Test R integration + python -c "import rpy2.robjects as ro; ro.r('library(ggplot2)'); print('✓ R integration OK')" + ``` + +### Testing the Build + +4. **Run Tests:** + ```bash + make test + ``` + +5. **Try Quick Render (no notebook execution):** + ```bash + make qrender + ``` + +6. **Full Build (executes notebook):** + ```bash + make render + ``` + - This will take several minutes + - Watch for any library compatibility issues + - Check `TESTING_CHECKLIST.md` for specific things to verify + +### Setting Up CI/CD + +7. **Configure GitHub Secrets:** + - Go to your GitHub repo → Settings → Secrets and variables → Actions + - Add `NETLIFY_AUTH_TOKEN` (from Netlify account settings) + - Add `NETLIFY_SITE_ID` (from Netlify site settings) + +8. **Test GitHub Actions:** + - Push to a test branch + - Check Actions tab to see workflow run + - Verify Netlify preview deployment + +9. **Deploy to Production:** + - Merge to master branch + - GitHub Actions will automatically deploy to production + +## Potential Issues to Watch For + +### Library Compatibility + +1. **plotly v5 Changes:** + - API changes from v4 to v5 + - Different image rendering (Kaleido vs Orca) + - No authentication needed + +2. **altair v5 Changes:** + - May need updates to save PNG images + - Selenium/geckodriver integration + +3. **plotnine 0.13:** + - Better ggplot2 compatibility but some API changes + +4. **statsmodels 0.14:** + - Check regression plots and confidence intervals + +### Environment Issues + +1. **R_HOME:** + - Must be set correctly for rpy2 + - GitHub Actions sets this automatically + - Locally: `export R_HOME=$(R RHOME)` + +2. **Headless Rendering:** + - Selenium needs xvfb on Linux + - GitHub Actions handles this automatically + - Locally on Linux: `xvfb-run make render` + +3. **geckodriver:** + - Must be in PATH for selenium + - GitHub Actions installs this automatically + - Locally: Install via package manager or download + +## Rollback Plan + +If you encounter critical issues: + +1. **Keep Travis CI temporarily:** + - Don't delete `.travis.yml` until GitHub Actions is proven + - Both can run simultaneously + +2. **Pin problematic packages:** + - If specific library versions cause issues, pin to older versions in requirements.txt + - Example: `plotly==4.14.3` if v5 has problems + +3. **Gradual migration:** + - Update libraries one at a time + - Test each change thoroughly + +## Files Changed Summary + +**New Files:** +- `pyproject.toml` +- `setup_r.sh` +- `.github/workflows/deploy.yml` +- `TESTING_CHECKLIST.md` +- `MIGRATION_NOTES.md` + +**Modified Files:** +- `runtime.txt` +- `requirements.txt` +- `Makefile` +- `README.md` +- `CLAUDE.md` +- `.travis/authenticate_plotly.py` (deprecated with explanation) + +**Deprecated (Not Deleted):** +- `environment.yml` (conda environment - no longer used) +- `.travis.yml` (Travis CI config) +- `.travis/` directory (old CI scripts) + +**No Changes:** +- `render.py` (compatible with new libraries) +- `Examples.ipynb` (may need updates if libraries break examples) +- `templates/t_index.html` +- `tests/test_plots.py` +- `INTRO.md` + +## Success Criteria + +The migration is successful when: + +- [ ] `make dev_environment` completes without errors +- [ ] `make test` passes all tests +- [ ] `make render` executes notebook and generates all plots +- [ ] GitHub Actions workflow runs successfully +- [ ] Site deploys correctly to Netlify +- [ ] All plotting libraries render correctly +- [ ] R ggplot2 examples work via rpy2 + +## Support + +If you encounter issues: + +1. Check `TESTING_CHECKLIST.md` for common problems +2. Review GitHub Actions logs for CI issues +3. Check package compatibility in individual notebook cells +4. File issues in the GitHub repository + +--- + +**Migration completed:** 2025-11-04 +**Python version:** 3.11+ +**Primary changes:** uv adoption, library updates, CI/CD modernization diff --git a/Makefile b/Makefile index 80afa27..c10853b 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,14 @@ clean: travis: render -plotly_auth: - python ./.travis/authenticate_plotly.py +setup: + @echo "Setting up development environment..." + @echo "1. Installing Python dependencies with uv..." + @command -v uv >/dev/null 2>&1 || { echo "Installing uv..."; curl -LsSf https://astral.sh/uv/install.sh | sh; } + uv pip install -r requirements.txt + @echo "2. Installing R packages..." + @./setup_r.sh + @echo "✓ Setup complete!" test: python -m pytest tests/ @@ -30,11 +36,13 @@ s3_upload: run_nb: jupyter nbconvert --to notebook --execute "Examples.ipynb" --output "Examples.$(GIT_COMMIT).ipynb" -dev_environment: - conda env update -q - source activate pythonplot && pip install -q -r requirements.txt +dev_environment: setup + @echo "Development environment ready!" + @echo "" + @echo "Note: This project now uses uv instead of conda." + @echo "R must be installed separately on your system." cloudfront_invalidate: python .travis/invalidate_cloudfront.py -.PHONY: all qrender render s3_upload run_nb travis clean cloudfront_invalidate test dev_environment plotly_auth +.PHONY: all qrender render s3_upload run_nb travis clean cloudfront_invalidate test dev_environment setup diff --git a/README.md b/README.md index 02be915..4b9a40f 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,28 @@ Available at [pythonplot.com](http://pythonplot.com/) The site is generated from plots in the `Examples.ipynb` Python 3, Jupyter notebook. -You can create a [Conda](https://conda.io/docs/install/quick.html) dev environment to run the notebook with `make dev_environment`. +#### Development Setup + +**Requirements:** +- Python 3.11+ +- R 4.0+ (install via [Homebrew](https://brew.sh/) on macOS: `brew install r`, or from [CRAN](https://cran.r-project.org/)) +- [uv](https://github.com/astral-sh/uv) for Python package management + +**Quick Start:** +```bash +# Set up development environment (installs Python & R dependencies) +make dev_environment + +# Or manually: +# 1. Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 2. Install Python dependencies +uv pip install -r requirements.txt + +# 3. Install R packages +./setup_r.sh +``` Each plot in the notebook is tagged with metadata using Jupyter cell tags. You can view the cell tags in your notebook with `View > Cell Toolbar > Tags`. @@ -76,8 +97,31 @@ If you are using Jupyter lab, these are available through the `Cell Tools` left The site layout current allows for ~46 characters in the code window. Please keep this in mind and wrap your code to avoid too much side scrolling. -At the moment, the code must return a png image into the output cell. To generate image files with `plotly`, using their server is currently the only way. Please see the [getting started page](https://plot.ly/python/getting-started/) for instructions on the `.credentials` file. +At the moment, the code must return a png image into the output cell. With modern versions of plotly (v5+), images are rendered locally using Kaleido - no authentication required. A Markdown comment can be added within triple quotes on the first line of the cell. (This currently doesn't work for `R` cells.) -You can render the images to `web/img/plots` by running `$ make`. Afterward, launch a local server (e.g. `python -m http.server`) from the `web` directory. \ No newline at end of file +#### Building the Site + +```bash +# Quick render (uses existing notebook without re-execution) +make qrender + +# Full build (executes notebook, then renders) +make render + +# Run tests +make test +``` + +After rendering, launch a local server from the `web` directory: +```bash +cd web && python -m http.server +``` + +#### CI/CD + +The project uses GitHub Actions for continuous integration and deployment: +- **Pushes to `master`**: Automatically deployed to production on Netlify +- **Other branches**: Create preview deployments on Netlify +- Requires secrets: `NETLIFY_AUTH_TOKEN` and `NETLIFY_SITE_ID` \ No newline at end of file diff --git a/TESTING_CHECKLIST.md b/TESTING_CHECKLIST.md new file mode 100644 index 0000000..90bf061 --- /dev/null +++ b/TESTING_CHECKLIST.md @@ -0,0 +1,111 @@ +# Testing Checklist for Modernized Build + +This checklist should be completed after setting up the modernized development environment. + +## Environment Setup Testing + +- [ ] **Install R** (if not already installed) + ```bash + # macOS + brew install r + + # Ubuntu/Debian + sudo apt-get install r-base r-base-dev + ``` + +- [ ] **Run R setup script** + ```bash + ./setup_r.sh + ``` + Expected: ggplot2 and mgcv packages installed successfully + +- [ ] **Install Python dependencies with uv** + ```bash + # Install uv if needed + curl -LsSf https://astral.sh/uv/install.sh | sh + + # Install dependencies + uv pip install -r requirements.txt + ``` + Expected: All packages install without errors + +- [ ] **Verify R integration** + ```bash + python -c "import rpy2.robjects as ro; ro.r('library(ggplot2)'); print('R integration works!')" + ``` + Expected: No errors, prints "R integration works!" + +## Build Testing + +- [ ] **Run tests** + ```bash + make test + ``` + Expected: All tests pass + +- [ ] **Quick render test (without notebook execution)** + ```bash + make qrender + ``` + Expected: Generates web/index.html from existing Examples.ipynb + +- [ ] **Full build test (with notebook execution)** + ```bash + make render + ``` + Expected: + - Executes notebook successfully + - Generates all plot images in web/img/plots/ + - Creates web/index.html + - No errors from any plotting library + +## Known Migration Issues to Watch For + +### Plotly v5 Changes +- [ ] Verify plotly plots render without authentication +- [ ] Check that Kaleido is used for static image export (not Orca) +- [ ] Ensure all plotly examples in notebook still work + +### Altair v5 Changes +- [ ] Verify altair plots render to PNG +- [ ] Confirm altair-saver is not needed (using native rendering) +- [ ] Check selenium/geckodriver works for image export + +### plotnine 0.13+ Changes +- [ ] Verify all ggplot-style plots render correctly +- [ ] Check for any deprecation warnings + +### statsmodels 0.14+ Changes +- [ ] Verify regression line plots work +- [ ] Check confidence interval calculations + +### rpy2 3.5+ Changes +- [ ] Verify %%R magic commands work +- [ ] Check data passing between Python and R +- [ ] Ensure all R ggplot2 examples render + +### Selenium 4+ Changes +- [ ] Verify geckodriver compatibility +- [ ] Check headless browser operations +- [ ] Test with xvfb if on Linux + +## CI/CD Testing + +- [ ] **GitHub Actions workflow** + - Push to a test branch + - Verify all workflow steps complete + - Check Netlify preview deploy works + - Merge to master and verify production deploy + +## Performance Checks + +- [ ] Note build time for comparison (old vs new) +- [ ] Check generated image file sizes +- [ ] Verify web/index.html loads correctly in browser + +## Rollback Plan + +If critical issues are found: +1. Keep Travis CI config (.travis.yml) until GitHub Actions is confirmed working +2. Document any notebook cells that fail with new library versions +3. Consider pinning problematic packages to older versions temporarily diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3fcd121 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,70 @@ +[project] +name = "pythonplot" +version = "0.1.0" +description = "An Introduction to Python Plotting for Exploratory Data Analysis" +authors = [ + {name = "Tim Hopper", email = "tim@tdhopper.com"} +] +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} + +dependencies = [ + # Core Jupyter/IPython + "ipython>=8.12", + "notebook>=7.0", + "jupyter>=1.0", + + # Data manipulation + "pandas>=2.0", + + # Plotting libraries + "matplotlib>=3.7", + "seaborn>=0.13", + "plotnine>=0.13", + "plotly>=5.24", + "altair>=5.0", + + # Image rendering + "kaleido>=0.2.1", + "pillow>=10.0", + "selenium>=4.15", + + # Statistical tools + "statsmodels>=0.14", + "scikit-misc>=0.3", + + # R integration + "rpy2>=3.5", + + # Template rendering + "jinja2>=3.1", + "jinja2-highlight>=0.6", + "markdown>=3.5", + + # Utilities + "psutil>=5.9", + "tzlocal>=5.0", + "simplegeneric>=0.8", + + # Cloud (if needed for deployment) + "boto3>=1.28", + + # Testing + "pytest>=7.4", +] + +[project.urls] +Homepage = "https://pythonplot.com" +Repository = "https://github.com/tdhopper/pythonplot.com" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.uv] +dev-dependencies = [] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] diff --git a/requirements.txt b/requirements.txt index b8b7027..154d0c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,42 @@ -selenium -pillow -ipython -pandas -jupyter -seaborn -plotnine<0.7.0 -jinja2 -jinja2-highlight -markdown -scikit-misc -boto3 -pytest -chart-studio -statsmodels==0.11.1 -rpy2 -psutil -plotly==4.10.0 -kaleido -tzlocal -simplegeneric -altair -altair-saver -selenium +# Core Jupyter/IPython +ipython>=8.12 +notebook>=7.0 +jupyter>=1.0 + +# Data manipulation +pandas>=2.0 + +# Plotting libraries +matplotlib>=3.7 +seaborn>=0.13 +plotnine>=0.13 +plotly>=5.24 +altair>=5.0 + +# Image rendering and export +kaleido>=0.2.1 +pillow>=10.0 +selenium>=4.15 + +# Statistical tools +statsmodels>=0.14 +scikit-misc>=0.3 + +# R integration +rpy2>=3.5 + +# Template rendering +jinja2>=3.1 +jinja2-highlight>=0.6 +markdown>=3.5 + +# Utilities +psutil>=5.9 +tzlocal>=5.0 +simplegeneric>=0.8 + +# Cloud services (for deployment) +boto3>=1.28 + +# Testing +pytest>=7.4 diff --git a/runtime.txt b/runtime.txt index d70c8f8..2c07333 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -3.6 +3.11 diff --git a/setup_r.sh b/setup_r.sh new file mode 100755 index 0000000..d2d0b1b --- /dev/null +++ b/setup_r.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Setup script for R and required R packages +# This script installs the R packages needed for pythonplot.com + +set -e # Exit on error + +echo "Installing R packages..." + +# Check if R is installed +if ! command -v R &> /dev/null; then + echo "Error: R is not installed." + echo "" + echo "Please install R first:" + echo " - macOS: brew install r" + echo " - Ubuntu/Debian: sudo apt-get install r-base r-base-dev" + echo " - Or download from: https://cran.r-project.org/" + exit 1 +fi + +echo "R version:" +R --version | head -n 1 + +# Install required R packages +R --quiet --no-save << 'EOF' +# Set CRAN mirror +options(repos = c(CRAN = "https://cran.rstudio.com/")) + +# List of required packages +packages <- c("ggplot2", "mgcv") + +# Install packages that aren't already installed +for (pkg in packages) { + if (!require(pkg, character.only = TRUE, quietly = TRUE)) { + cat(paste("Installing", pkg, "...\n")) + install.packages(pkg, quiet = FALSE) + } else { + cat(paste(pkg, "is already installed\n")) + } +} + +# Verify installations +cat("\nVerifying R package installations:\n") +for (pkg in packages) { + if (require(pkg, character.only = TRUE, quietly = TRUE)) { + cat(paste("✓", pkg, "version", as.character(packageVersion(pkg)), "\n")) + } else { + cat(paste("✗", pkg, "FAILED TO INSTALL\n")) + quit(status = 1) + } +} + +cat("\n✓ All R packages installed successfully!\n") +EOF + +echo "" +echo "R setup complete!" +echo "" +echo "Note: If you encounter issues with rpy2, make sure R_HOME is set correctly:" +echo " export R_HOME=\$(R RHOME)"