Improved documentation and preprocessing transformations

.github/workflows/main.yml

@@ -5,7 +5,33 @@ on:
     branches: [ main ]
 
 jobs:
+  Check-MDN-Changes:
+    runs-on: ubuntu-latest
+    outputs:
+      mdn_changed: ${{ steps.check.outputs.mdn_changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: Check for MDN-related file changes
+        id: check
+        run: |
+          # Get list of changed files in this push
+          CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD)
+          echo "Changed files:"
+          echo "$CHANGED_FILES"
+
+          # Check if any MDN-related files were changed
+          if echo "$CHANGED_FILES" | grep -qE "(mdn|MDN)"; then
+            echo "mdn_changed=true" >> $GITHUB_OUTPUT
+            echo "MDN-related files were changed"
+          else
+            echo "mdn_changed=false" >> $GITHUB_OUTPUT
+            echo "No MDN-related files were changed"
+          fi
+
   Test:
+    needs: Check-MDN-Changes
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -30,10 +56,14 @@ jobs:
         run: |
           sudo Rscript -e 'install.packages("StatMatch", repos="https://cloud.r-project.org")'
           sudo Rscript -e 'install.packages("clue", repos="https://cloud.r-project.org")'
-      - name: Install full dependencies (Python 3.13)
-        if: matrix.python-version == '3.13'
+      - name: Install full dependencies without MDN (Python 3.13)
+        if: matrix.python-version == '3.13' && needs.Check-MDN-Changes.outputs.mdn_changed != 'true'
         run: |
           uv pip install -e ".[dev,docs,matching,images]" --system
+      - name: Install full dependencies with MDN (Python 3.13)
+        if: matrix.python-version == '3.13' && needs.Check-MDN-Changes.outputs.mdn_changed == 'true'
+        run: |
+          uv pip install -e ".[dev,docs,matching,mdn,images]" --system
       - name: Install minimal dependencies (Python 3.12)
         if: matrix.python-version == '3.12'
         run: |

.github/workflows/pr_code_changes.yaml

@@ -20,7 +20,33 @@ jobs:
       - name: Check code formatting
         run: make check-format
 
+  Check-MDN-Changes:
+    runs-on: ubuntu-latest
+    outputs:
+      mdn_changed: ${{ steps.check.outputs.mdn_changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Check for MDN-related file changes
+        id: check
+        run: |
+          # Get list of changed files in this PR
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
+          echo "Changed files:"
+          echo "$CHANGED_FILES"
+
+          # Check if any MDN-related files were changed
+          if echo "$CHANGED_FILES" | grep -qE "(mdn|MDN)"; then
+            echo "mdn_changed=true" >> $GITHUB_OUTPUT
+            echo "MDN-related files were changed"
+          else
+            echo "mdn_changed=false" >> $GITHUB_OUTPUT
+            echo "No MDN-related files were changed"
+          fi
+
   Test:
+    needs: Check-MDN-Changes
     strategy:
       matrix:
         os: [ ubuntu-latest ]
@@ -49,10 +75,14 @@ jobs:
         run: |
           sudo Rscript -e 'install.packages("StatMatch", repos="https://cloud.r-project.org")'
           sudo Rscript -e 'install.packages("clue", repos="https://cloud.r-project.org")'
-      - name: Install full test dependencies (Python 3.13)
-        if: matrix.python-version == '3.13'
+      - name: Install full test dependencies without MDN (Python 3.13)
+        if: matrix.python-version == '3.13' && needs.Check-MDN-Changes.outputs.mdn_changed != 'true'
         run: |
           uv pip install -e ".[dev,matching]" --system
+      - name: Install full test dependencies with MDN (Python 3.13)
+        if: matrix.python-version == '3.13' && needs.Check-MDN-Changes.outputs.mdn_changed == 'true'
+        run: |
+          uv pip install -e ".[dev,matching,mdn]" --system
       - name: Install minimal test dependencies (Python 3.12)
         if: matrix.python-version == '3.12'
         run: |

.github/workflows/pr_docs_changes.yaml

@@ -36,7 +36,7 @@ jobs:
               sudo Rscript -e 'install.packages("clue", repos="https://cloud.r-project.org")'
           - name: Install dependencies
             run: |
-              uv pip install -e ".[dev,docs,matching,images]" --system
+              uv pip install -e ".[dev,docs,matching,mdn,images]" --system
           - name: Install JB
             run: uv pip install "jupyter-book>=2.0.0a0" --system
           - name: Test documentation builds

changelog_entry.yaml

@@ -0,0 +1,5 @@
+- bump: minor
+  changes:
+    added:
+    - Asinh transformation preprocessing option for numeric variables.
+    - Documentation for MDN model and preprocessing options.

docs/_config.yml

@@ -4,7 +4,7 @@ logo: logo.png
 
 execute:
   execute_notebooks: force
-  timeout: 120
+  timeout: 360
 
 repository:
   url: https://github.com/policyengine/microimpute

docs/_toc.yml

@@ -1,6 +1,6 @@
 format: jb-book
 root: index
-parts: 
+parts:
   - caption: Models
     chapters:
       - file: models/imputer/index
@@ -18,19 +18,25 @@ parts:
       - file: models/quantreg/index
         sections:
           - file: models/quantreg/quantreg-imputation
+      - file: models/mdn/index
+        sections:
+          - file: models/mdn/mdn-imputation
   - caption: Imputation and benchmarking
     chapters:
       - file: imputation-benchmarking/index
         sections:
+          - file: imputation-benchmarking/preprocessing
+          - file: imputation-benchmarking/cross-validation
+          - file: imputation-benchmarking/metrics
+          - file: imputation-benchmarking/visualizations
           - file: imputation-benchmarking/benchmarking-methods
-          - file: imputation-benchmarking/imputing-across-surveys
   - caption: AutoImpute
     chapters:
       - file: autoimpute/index
         sections:
           - file: autoimpute/autoimpute
-  - caption: SCF to CPS example
+  - caption: Use cases
     chapters:
-      - file: examples/scf_to_cps/index
+      - file: use_cases/index
         sections:
-          - file: examples/scf_to_cps/imputing-from-scf-to-cps
+          - file: use_cases/scf_to_cps/imputing-from-scf-to-cps