Move __doc__ crate to crates/doc (RustPython#6234)

* Add `__doc__` crate

* Base auto-generate ci

* Add dummy files

* Update docs

* Set codegen-units to 1 for doc db

* Mark `*.inc.rs` as auto generated

* Disable doctest

* Reset docs

Author: ShaharNaveh

.gitattributes

@@ -5,3 +5,4 @@ vm/src/stdlib/ast/gen.rs    linguist-generated -merge
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF
 *.pck       binary
+crates/rustpython_doc_db/src/*.inc.rs   linguist-generated=true

.github/workflows/ci.yaml

@@ -113,7 +113,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run rust tests
     runs-on:  ${{ matrix.os }}
-    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 35 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]
@@ -239,7 +239,7 @@ jobs:
       RUST_BACKTRACE: full
     name: Run snippets and cpython tests
     runs-on:  ${{ matrix.os }}
-    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 30 }}
+    timeout-minutes: ${{ contains(matrix.os, 'windows') && 45 || 35 }}
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-latest]

.github/workflows/update-doc-db.yml

@@ -0,0 +1,95 @@
+name: Update doc DB
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      python-version:
+        description: Target python version to generate doc db for
+        type: string
+        default: "3.13.9"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./crates/rustpython-doc
+
+jobs:
+  generate:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+          sparse-checkout: |
+            crates/rustpython-doc
+
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Generate docs
+        run: python ./generate.py
+
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: doc-db-${{ inputs.python-version }}-${{ matrix.os }}
+          path: "crates/rustpython-doc/generated/*.json"
+          if-no-files-found: error
+          retention-days: 7
+          overwrite: true
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: generate
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          persist-credentials: false
+          sparse-checkout: |
+            crates/rustpython-doc
+
+      - name: Download generated doc DBs
+        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
+        with:
+          pattern: "doc-db-${{ inputs.python-version }}-**"
+          path: crates/rustpython-doc/generated/
+          merge-multiple: true
+
+      - name: Transform JSON
+        run: |
+          # Merge all artifacts
+          jq -s "add" --sort-keys generated/*.json > generated/merged.json
+
+          # Format merged json for the phf macro
+          jq -r 'to_entries[] | "    \(.key | @json) => \(.value | @json),"' generated/merged.json > generated/raw_entries.txt
+
+          OUTPUT_FILE='src/data.inc.rs'
+
+          echo -n '' > $OUTPUT_FILE
+
+          echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE
+          echo "// CPython version: ${{ inputs.python-version }}" >> $OUTPUT_FILE
+          echo '// spell-checker: disable' >> $OUTPUT_FILE
+
+          echo '' >> $OUTPUT_FILE
+
+          echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" >> $OUTPUT_FILE
+          cat generated/raw_entries.txt >> $OUTPUT_FILE
+          echo '};' >> $OUTPUT_FILE
+
+      - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
+        with:
+          name: doc-db-${{ inputs.python-version }}
+          path: "crates/rustpython-doc/src/data.inc.rs"
+          if-no-files-found: error
+          retention-days: 7
+          overwrite: true

Cargo.lock

@@ -73,6 +73,10 @@ opt-level = 3
 # https://github.com/rust-lang/rust/issues/92869
 # lto = "thin"
 
+# Doesn't change often
+[profile.release.package.rustpython-doc]
+codegen-units = 1
+
 [profile.bench]
 lto = "thin"
 codegen-units = 1
@@ -132,6 +136,7 @@ members = [
     "derive-impl",
     "wtf8",
     "wasm/lib",
+    "crates/*",
 ]
 
 [workspace.package]
@@ -156,13 +161,14 @@ rustpython-pylib = { path = "pylib", version = "0.4.0" }
 rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" }
 rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" }
 rustpython-wtf8 = { path = "wtf8", version = "0.4.0" }
-rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" }
+rustpython-doc = { path = "crates/doc", version = "0.4.0" }
 
 ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.14.1" }
 
+phf = { version = "0.13.1", default-features = false, features = ["macros"]}
 ahash = "0.8.12"
 ascii = "1.1"
 bitflags = "2.9.4"

Cargo.toml

@@ -0,0 +1 @@
+generated/

crates/doc/.gitignore

@@ -0,0 +1,17 @@
+[package]
+name = "rustpython-doc"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license-file = "LICENSE"
+
+[dependencies]
+phf = { workspace = true }
+
+[lib]
+doctest = false # Crashes when true
+
+[lints]
+workspace = true