feat(release): Homebrew-tap delivery; drop PyPI; v0.9.0 target
- SHA
24fda51638a9bb353101724282678deeefa0bd10- Parents
-
d9b8ce8 - Tree
3b1214d
24fda51
24fda51638a9bb353101724282678deeefa0bd10d9b8ce8
3b1214d| Status | File | + | - |
|---|---|---|---|
| M |
.github/workflows/release.yml
|
119 | 89 |
| M |
CHANGELOG.md
|
7 | 17 |
| M |
CONTRIBUTING.md
|
56 | 45 |
| M |
README.md
|
27 | 20 |
| M |
pyproject.toml
|
1 | 1 |
| M |
src/dlm/export/vendoring.py
|
81 | 31 |
| M |
uv.lock
|
1 | 1 |
.github/workflows/release.ymlmodified@@ -1,12 +1,20 @@ | ||
| 1 | 1 | name: release |
| 2 | 2 | |
| 3 | -# Tag-driven release: push a `v*` tag, the workflow builds the wheel + | |
| 4 | -# sdist and publishes to PyPI via trusted-publisher OIDC (configure | |
| 5 | -# the publisher under the project's PyPI settings before the first | |
| 6 | -# real run). The docs site is deployed separately via `docs.yml`. | |
| 3 | +# Tag-driven release. We don't publish to PyPI — DLM ships via a | |
| 4 | +# Homebrew tap (https://github.com/tenseleyFlow/homebrew-tap). | |
| 7 | 5 | # |
| 8 | -# Dry-run to test.pypi.org by pushing a `v*-rc*` tag (e.g. `v1.0.0-rc1`). | |
| 9 | -# The publish step routes to the test index when the tag ends in `-rc*`. | |
| 6 | +# What this workflow does on a `v*` tag: | |
| 7 | +# | |
| 8 | +# 1. Gate on the full CI suite (ruff + format + mypy + non-slow pytest | |
| 9 | +# + mkdocs strict build). | |
| 10 | +# 2. Build a "fat" source tarball that bundles `vendor/llama.cpp/` | |
| 11 | +# (source only, no build artifacts) so the Homebrew formula can | |
| 12 | +# drop the convert scripts into libexec without cloning submodules | |
| 13 | +# at install time. | |
| 14 | +# 3. Create a GitHub release with the tarball + a computed sha256 | |
| 15 | +# pasted into the release notes so the tap formula can bump in one | |
| 16 | +# edit. | |
| 17 | +# 4. Deploy the docs site to gh-pages. | |
| 10 | 18 | |
| 11 | 19 | on: |
| 12 | 20 | push: |
@@ -14,10 +22,13 @@ on: | ||
| 14 | 22 | - "v*" |
| 15 | 23 | |
| 16 | 24 | permissions: |
| 17 | - # Required for trusted-publisher OIDC flow. | |
| 18 | - id-token: write | |
| 19 | - # Required so the workflow can read the repo on a tag push. | |
| 20 | - contents: read | |
| 25 | + contents: write # create release + upload asset | |
| 26 | + id-token: write # gh-pages OIDC deploy | |
| 27 | + pages: write | |
| 28 | + | |
| 29 | +concurrency: | |
| 30 | + group: release-${{ github.ref }} | |
| 31 | + cancel-in-progress: false | |
| 21 | 32 | |
| 22 | 33 | env: |
| 23 | 34 | UV_VERSION: "0.11.6" |
@@ -51,105 +62,124 @@ jobs: | ||
| 51 | 62 | run: uv run pytest -m "not slow and not online and not gpu" |
| 52 | 63 | |
| 53 | 64 | - name: Mkdocs build --strict |
| 54 | - # Audit-05 N14: block release on docs regressions (dead links, | |
| 55 | - # missing files in nav, etc.) before the publish step so a broken | |
| 56 | - # docs site can't ship alongside a real PyPI tag. | |
| 57 | 65 | run: uv run mkdocs build --strict --site-dir /tmp/mkdocs-check |
| 58 | 66 | |
| 59 | - build: | |
| 60 | - name: build wheel + sdist | |
| 67 | + build-release-tarball: | |
| 68 | + name: build fat source tarball | |
| 61 | 69 | needs: ci-gate |
| 62 | 70 | runs-on: ubuntu-latest |
| 71 | + outputs: | |
| 72 | + tarball_name: ${{ steps.build.outputs.tarball_name }} | |
| 73 | + sha256: ${{ steps.build.outputs.sha256 }} | |
| 63 | 74 | steps: |
| 64 | - - uses: actions/checkout@v4 | |
| 65 | - | |
| 66 | - - name: Install uv | |
| 67 | - uses: astral-sh/setup-uv@v4 | |
| 75 | + - name: Checkout with submodules | |
| 76 | + uses: actions/checkout@v4 | |
| 68 | 77 | with: |
| 69 | - version: ${{ env.UV_VERSION }} | |
| 70 | - | |
| 71 | - - name: Build | |
| 72 | - run: uv build | |
| 78 | + submodules: recursive | |
| 73 | 79 | |
| 74 | - - name: Upload artifacts | |
| 80 | + - name: Build tarball | |
| 81 | + id: build | |
| 82 | + run: | | |
| 83 | + set -euxo pipefail | |
| 84 | + TAG="${GITHUB_REF_NAME}" | |
| 85 | + NAME="dlm-${TAG}" | |
| 86 | + # Build the tarball with a top-level prefix matching NAME so | |
| 87 | + # `tar xzf` extracts into a clean subdir (Homebrew convention). | |
| 88 | + # Exclude CI noise + git metadata; KEEP `vendor/llama.cpp/` so | |
| 89 | + # the formula can use the vendored Python convert scripts. | |
| 90 | + tar czf "${NAME}.tar.gz" \ | |
| 91 | + --transform="s,^,${NAME}/," \ | |
| 92 | + --exclude=".git" \ | |
| 93 | + --exclude=".github" \ | |
| 94 | + --exclude=".pytest_cache" \ | |
| 95 | + --exclude=".mypy_cache" \ | |
| 96 | + --exclude=".ruff_cache" \ | |
| 97 | + --exclude="__pycache__" \ | |
| 98 | + --exclude="*.pyc" \ | |
| 99 | + --exclude="vendor/llama.cpp/.git" \ | |
| 100 | + --exclude="vendor/llama.cpp/build" \ | |
| 101 | + --exclude="vendor/llama.cpp/.cache" \ | |
| 102 | + --exclude="tests" \ | |
| 103 | + --exclude=".docs" \ | |
| 104 | + --exclude="site" \ | |
| 105 | + . | |
| 106 | + SHA=$(sha256sum "${NAME}.tar.gz" | awk '{print $1}') | |
| 107 | + echo "Tarball: ${NAME}.tar.gz (sha256=${SHA})" | |
| 108 | + echo "tarball_name=${NAME}.tar.gz" >> "$GITHUB_OUTPUT" | |
| 109 | + echo "sha256=${SHA}" >> "$GITHUB_OUTPUT" | |
| 110 | + | |
| 111 | + - name: Upload tarball artifact | |
| 75 | 112 | uses: actions/upload-artifact@v4 |
| 76 | 113 | with: |
| 77 | - name: dist | |
| 78 | - path: dist/ | |
| 79 | - | |
| 80 | - classify-tag: | |
| 81 | - # Audit-05 M8: route by `packaging.version.Version(tag).is_prerelease` | |
| 82 | - # rather than `contains('-rc')`. The substring check breaks on PEP 440 | |
| 83 | - # canonical prereleases (e.g. `v1.0.0rc1` without the hyphen) — those | |
| 84 | - # would silently publish to prod PyPI. | |
| 85 | - name: classify tag as release vs prerelease | |
| 86 | - needs: ci-gate | |
| 114 | + name: release-tarball | |
| 115 | + path: ${{ steps.build.outputs.tarball_name }} | |
| 116 | + | |
| 117 | + publish-github-release: | |
| 118 | + name: publish GitHub release | |
| 119 | + needs: build-release-tarball | |
| 87 | 120 | runs-on: ubuntu-latest |
| 88 | - outputs: | |
| 89 | - is_prerelease: ${{ steps.classify.outputs.is_prerelease }} | |
| 90 | 121 | steps: |
| 91 | - - name: Derive is_prerelease via packaging.version | |
| 92 | - id: classify | |
| 122 | + - uses: actions/download-artifact@v4 | |
| 123 | + with: | |
| 124 | + name: release-tarball | |
| 125 | + | |
| 126 | + - name: Create release | |
| 127 | + env: | |
| 128 | + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| 129 | + TAG: ${{ github.ref_name }} | |
| 130 | + TARBALL: ${{ needs.build-release-tarball.outputs.tarball_name }} | |
| 131 | + SHA256: ${{ needs.build-release-tarball.outputs.sha256 }} | |
| 93 | 132 | run: | |
| 94 | - TAG="${GITHUB_REF_NAME}" | |
| 95 | - # Strip leading 'v' if present; packaging accepts either form. | |
| 96 | - VERSION="${TAG#v}" | |
| 97 | - IS_PRE=$(python3 -c "import sys; from packaging.version import Version, InvalidVersion | |
| 133 | + set -euxo pipefail | |
| 134 | + # Route prerelease tags (PEP 440: rc / a / b / dev suffixes) | |
| 135 | + # to GitHub "prerelease" flag — surfaces as the yellow banner | |
| 136 | + # on the releases page instead of "latest". | |
| 137 | + PRERELEASE_FLAG="" | |
| 138 | + if python3 -c " | |
| 139 | + import sys | |
| 140 | + from packaging.version import Version, InvalidVersion | |
| 141 | + tag='${TAG}'.lstrip('v') | |
| 98 | 142 | try: |
| 99 | - v = Version('${VERSION}') | |
| 100 | - print('true' if v.is_prerelease else 'false') | |
| 143 | + sys.exit(0 if Version(tag).is_prerelease else 1) | |
| 101 | 144 | except InvalidVersion: |
| 102 | - # Unparseable tags → treat as prerelease (safer default: routes | |
| 103 | - # to test.pypi.org, where a bad tag can't stomp the real index). | |
| 104 | - print('true') | |
| 105 | - ") | |
| 106 | - echo "Tag ${TAG} → is_prerelease=${IS_PRE}" | |
| 107 | - echo "is_prerelease=${IS_PRE}" >> "$GITHUB_OUTPUT" | |
| 108 | - | |
| 109 | - publish-testpypi: | |
| 110 | - name: publish to test.pypi.org (prerelease tags only) | |
| 111 | - needs: [build, classify-tag] | |
| 112 | - if: needs.classify-tag.outputs.is_prerelease == 'true' | |
| 113 | - runs-on: ubuntu-latest | |
| 114 | - environment: | |
| 115 | - name: test-pypi | |
| 116 | - url: https://test.pypi.org/project/dlm/ | |
| 117 | - steps: | |
| 118 | - - uses: actions/download-artifact@v4 | |
| 119 | - with: | |
| 120 | - name: dist | |
| 121 | - path: dist/ | |
| 145 | + sys.exit(0) | |
| 146 | + "; then | |
| 147 | + PRERELEASE_FLAG="--prerelease" | |
| 148 | + fi | |
| 122 | 149 | |
| 123 | - - name: Publish | |
| 124 | - uses: pypa/gh-action-pypi-publish@release/v1 | |
| 125 | - with: | |
| 126 | - repository-url: https://test.pypi.org/legacy/ | |
| 150 | + cat > release-notes.md <<EOF | |
| 151 | + ## DLM ${TAG} | |
| 127 | 152 | |
| 128 | - publish-pypi: | |
| 129 | - name: publish to pypi.org (release tags) | |
| 130 | - needs: [build, classify-tag] | |
| 131 | - if: needs.classify-tag.outputs.is_prerelease == 'false' | |
| 132 | - runs-on: ubuntu-latest | |
| 133 | - environment: | |
| 134 | - name: pypi | |
| 135 | - url: https://pypi.org/project/dlm/ | |
| 136 | - steps: | |
| 137 | - - uses: actions/download-artifact@v4 | |
| 138 | - with: | |
| 139 | - name: dist | |
| 140 | - path: dist/ | |
| 153 | + Install via the Homebrew tap: | |
| 154 | + | |
| 155 | + \`\`\` | |
| 156 | + brew tap tenseleyFlow/tap | |
| 157 | + brew install dlm | |
| 158 | + \`\`\` | |
| 159 | + | |
| 160 | + ### Formula bump | |
| 161 | + | |
| 162 | + When bumping \`Formula/dlm.rb\` in \`tenseleyFlow/homebrew-tap\`: | |
| 163 | + | |
| 164 | + - \`url\` → https://github.com/tenseleyFlow/DocumentLanguageModel/releases/download/${TAG}/${TARBALL} | |
| 165 | + - \`sha256\` → \`${SHA256}\` | |
| 166 | + | |
| 167 | + ### Build | |
| 168 | + | |
| 169 | + Full changelog: see \`CHANGELOG.md\` in the tarball. | |
| 170 | + EOF | |
| 141 | 171 | |
| 142 | - - name: Publish | |
| 143 | - uses: pypa/gh-action-pypi-publish@release/v1 | |
| 172 | + gh release create "${TAG}" \ | |
| 173 | + --repo "${GITHUB_REPOSITORY}" \ | |
| 174 | + --title "DLM ${TAG}" \ | |
| 175 | + --notes-file release-notes.md \ | |
| 176 | + ${PRERELEASE_FLAG} \ | |
| 177 | + "${TARBALL}" | |
| 144 | 178 | |
| 145 | 179 | deploy-docs: |
| 146 | - name: deploy docs for release | |
| 147 | - needs: publish-pypi | |
| 180 | + name: deploy docs to gh-pages | |
| 181 | + needs: publish-github-release | |
| 148 | 182 | runs-on: ubuntu-latest |
| 149 | - permissions: | |
| 150 | - contents: read | |
| 151 | - pages: write | |
| 152 | - id-token: write | |
| 153 | 183 | environment: |
| 154 | 184 | name: github-pages |
| 155 | 185 | url: ${{ steps.deployment.outputs.page_url }} |
CHANGELOG.mdmodified@@ -6,27 +6,17 @@ the project targets [Semantic Versioning](https://semver.org/). | ||
| 6 | 6 | |
| 7 | 7 | ## [Unreleased] |
| 8 | 8 | |
| 9 | -### Added | |
| 9 | +## [0.9.0] — target | |
| 10 | 10 | |
| 11 | -- Sprint 16 (this entry) — MkDocs Material documentation site, | |
| 12 | - starter templates under `templates/`, release and docs deployment | |
| 13 | - workflows, `CHANGELOG.md` itself. | |
| 14 | - | |
| 15 | -### Pending v1.0 | |
| 16 | - | |
| 17 | -- Manual verification of the README quickstart on a fresh Linux + | |
| 18 | - macOS box. | |
| 19 | -- PyPI trusted-publisher configuration + dry-run to test.pypi.org | |
| 20 | - before tagging v1.0. | |
| 21 | - | |
| 22 | -## [1.0.0] — target | |
| 23 | - | |
| 24 | -First stable release. Covers Phase 0–3 of the sprint roadmap | |
| 25 | -(scaffolding through MVP release). | |
| 11 | +First tagged release. Ships via the | |
| 12 | +[tenseleyFlow/homebrew-tap](https://github.com/tenseleyFlow/homebrew-tap) | |
| 13 | +(`brew tap tenseleyFlow/tap && brew install dlm`). Below v1.0 on | |
| 14 | +purpose — a human still needs to train + export + `ollama run` a real | |
| 15 | +document end-to-end before we claim the stable number. | |
| 26 | 16 | |
| 27 | 17 | ### Highlights |
| 28 | 18 | |
| 29 | -- Full v1.0 CLI: `init`, `train`, `prompt`, `export`, `pack`, | |
| 19 | +- CLI: `init`, `train`, `prompt`, `export`, `pack`, | |
| 30 | 20 | `unpack`, `doctor`, `show`, `migrate`. |
| 31 | 21 | - Content-addressed store at `~/.dlm/store/<dlm_id>/` with atomic |
| 32 | 22 | manifest updates and exclusive locking. |
CONTRIBUTING.mdmodified@@ -106,31 +106,29 @@ A few things we actively don't want: | ||
| 106 | 106 | |
| 107 | 107 | ## Releasing |
| 108 | 108 | |
| 109 | -Tag-driven: pushing a `v*` tag triggers `.github/workflows/release.yml`, | |
| 110 | -which runs the full CI gate, builds wheel + sdist via `uv build`, and | |
| 111 | -publishes to PyPI via trusted-publisher OIDC. | |
| 112 | - | |
| 113 | -### One-time PyPI trusted-publisher setup | |
| 114 | - | |
| 115 | -Before the first real release: | |
| 116 | - | |
| 117 | -1. Create a PyPI account for the `dlm` project (someone with publish | |
| 118 | - rights has to own this). | |
| 119 | -2. Under project settings → **Publishing** → **Add a new pending | |
| 120 | - publisher**, fill in: | |
| 121 | - - Owner: `tenseleyFlow` | |
| 122 | - - Repository name: `DocumentLanguageModel` | |
| 123 | - - Workflow filename: `release.yml` | |
| 124 | - - Environment name: `pypi` | |
| 125 | -3. Repeat on test.pypi.org with environment name `test-pypi`. | |
| 126 | -4. In the GitHub repo settings → **Environments**, create both | |
| 127 | - `pypi` and `test-pypi` environments. Neither needs secrets; the | |
| 128 | - OIDC token is minted per run. | |
| 129 | - | |
| 130 | -### Pre-flight | |
| 131 | - | |
| 132 | -The CI gate runs the full check suite (ruff, mypy, non-slow pytest, | |
| 133 | -`mkdocs build --strict`). Before tagging, eyeball these locally: | |
| 109 | +Tag-driven. Pushing `v*` triggers `.github/workflows/release.yml`, | |
| 110 | +which runs the full CI gate, builds a "fat" source tarball (includes | |
| 111 | +`vendor/llama.cpp/` so the Homebrew formula can drop the convert | |
| 112 | +scripts into libexec without cloning submodules), creates a GitHub | |
| 113 | +release with the tarball + computed sha256, and deploys the docs to | |
| 114 | +gh-pages. | |
| 115 | + | |
| 116 | +We publish via our Homebrew tap — | |
| 117 | +[tenseleyFlow/homebrew-tap](https://github.com/tenseleyFlow/homebrew-tap). | |
| 118 | +**We do not publish to PyPI.** Rationale lives in the audit-05 / | |
| 119 | +release-mode discussion; the short version is: PyPI makes versions | |
| 120 | +permanent, requires us to maintain a ~5 GB transitive dep surface, | |
| 121 | +and signals "this is battle-tested" in a way we're not ready to back | |
| 122 | +yet. | |
| 123 | + | |
| 124 | +### Conservative versioning | |
| 125 | + | |
| 126 | +Stay below `v1.0.0` until a human has trained + exported + | |
| 127 | +`ollama run`'d an adapter end-to-end. That's the only contract v1.0 | |
| 128 | +actually owes users. Current target: `v0.9.0` for the first tagged | |
| 129 | +release. | |
| 130 | + | |
| 131 | +### Pre-flight (run locally before tagging) | |
| 134 | 132 | |
| 135 | 133 | ```sh |
| 136 | 134 | uv run ruff check . |
@@ -141,37 +139,50 @@ uv sync --group docs | ||
| 141 | 139 | uv run mkdocs build --strict |
| 142 | 140 | ``` |
| 143 | 141 | |
| 144 | -Then bump the version in `pyproject.toml`, update `CHANGELOG.md` | |
| 145 | -(move the `## [Unreleased]` entries under a new `## [X.Y.Z]` heading), | |
| 146 | -and land both in the same commit. | |
| 142 | +Bump the version in `pyproject.toml`, move `## [Unreleased]` entries | |
| 143 | +under a new `## [X.Y.Z]` heading in `CHANGELOG.md`, and land both in | |
| 144 | +one commit. | |
| 147 | 145 | |
| 148 | 146 | ### Tagging |
| 149 | 147 | |
| 150 | -`release.yml` classifies tags via `packaging.version.Version.is_prerelease`: | |
| 148 | +```sh | |
| 149 | +git tag v0.9.0 | |
| 150 | +git push origin v0.9.0 | |
| 151 | +``` | |
| 151 | 152 | |
| 152 | -- **Prerelease** (routes to `test.pypi.org`): any PEP 440 prerelease. | |
| 153 | - Canonical: `v1.0.0rc1`, `v1.0.0a2`, `v1.0.0b3`. Hyphenated also | |
| 154 | - works: `v1.0.0-rc1`. | |
| 155 | -- **Release** (routes to `pypi.org`): clean `vMAJOR.MINOR.PATCH`. | |
| 153 | +`release.yml` classifies the tag via | |
| 154 | +`packaging.version.Version.is_prerelease`: | |
| 156 | 155 | |
| 157 | -```sh | |
| 158 | -# Dry-run via test.pypi.org first | |
| 159 | -git tag v1.0.0rc1 | |
| 160 | -git push origin v1.0.0rc1 | |
| 156 | +- **Prerelease** (`v0.9.0rc1`, `v0.9.0a1`, `v0.9.0-rc1`): GitHub | |
| 157 | + release gets the `prerelease` flag so it doesn't show as "latest." | |
| 158 | +- **Release** (`v0.9.0`, `v0.9.1`): standard GitHub release. | |
| 159 | + | |
| 160 | +### Bumping the Homebrew formula | |
| 161 | + | |
| 162 | +After the release workflow finishes, it prints the fat-tarball sha256 | |
| 163 | +in the release notes. Bump `Formula/dlm.rb` in the tap: | |
| 161 | 164 | |
| 162 | -# Verify on https://test.pypi.org/project/dlm/, then: | |
| 163 | -git tag v1.0.0 | |
| 164 | -git push origin v1.0.0 | |
| 165 | +```ruby | |
| 166 | +url "https://github.com/tenseleyFlow/DocumentLanguageModel/releases/download/v0.9.0/dlm-v0.9.0.tar.gz" | |
| 167 | +sha256 "<copy from release notes>" | |
| 165 | 168 | ``` |
| 166 | 169 | |
| 167 | -The release workflow publishes, then the `deploy-docs` job builds the | |
| 168 | -MkDocs site and pushes it to `gh-pages`. | |
| 170 | +Then: | |
| 171 | + | |
| 172 | +```sh | |
| 173 | +cd ~/path/to/homebrew-tap | |
| 174 | +brew install --build-from-source ./Formula/dlm.rb # local smoke | |
| 175 | +brew test ./Formula/dlm.rb # runs the `test do` block | |
| 176 | +git commit -am "dlm: bump to v0.9.0" | |
| 177 | +git push | |
| 178 | +``` | |
| 169 | 179 | |
| 170 | 180 | ### Rollback |
| 171 | 181 | |
| 172 | -There's no unpublish on PyPI (trusted-publisher or otherwise). If a | |
| 173 | -release is bad, bump the patch version and cut a fixed release rather | |
| 174 | -than trying to yank the old one. | |
| 182 | +Homebrew rollback is straightforward: delete the bad GitHub release | |
| 183 | +(or mark it draft), revert the formula bump in the tap. Users who | |
| 184 | +already installed the bad version can `brew uninstall dlm && brew | |
| 185 | +install dlm` to pick up the revert. | |
| 175 | 186 | |
| 176 | 187 | Thanks again — reach out in issues if anything's unclear. |
| 177 | 188 | |
README.mdmodified@@ -7,10 +7,11 @@ on your machine. No telemetry, no uploads, no cloud. Built on PyTorch | ||
| 7 | 7 | + HuggingFace with a hardware-aware planner that picks precision, |
| 8 | 8 | attention, and batching for your box. |
| 9 | 9 | |
| 10 | -**Status:** v1.0 release candidate. All Phase 3 sprints are complete; | |
| 11 | -the CLI surface (`init`, `train`, `prompt`, `export`, `pack`, `unpack`, | |
| 12 | -`doctor`, `show`, `migrate`) is wired end-to-end. A PyPI dry-run on | |
| 13 | -`test.pypi.org` is the last box to tick before the `v1.0` tag. | |
| 10 | +**Status:** pre-1.0 — the Phase 3 CLI surface (`init`, `train`, | |
| 11 | +`prompt`, `export`, `pack`, `unpack`, `doctor`, `show`, `migrate`) is | |
| 12 | +wired end-to-end but hasn't been battle-tested by a human running a | |
| 13 | +full train-export-ollama-run cycle. Ship target is `v0.9.0` via the | |
| 14 | +Homebrew tap below; `v1.0` waits on a real end-to-end train. | |
| 14 | 15 | |
| 15 | 16 | ## Why |
| 16 | 17 | |
@@ -53,35 +54,41 @@ or Llama for production), deterministic retraining, Ollama export. | ||
| 53 | 54 | |
| 54 | 55 | ## Install |
| 55 | 56 | |
| 56 | -### From source (current) | |
| 57 | +### From the Homebrew tap (recommended) | |
| 57 | 58 | |
| 58 | 59 | ```sh |
| 59 | -# Python 3.11+ and uv (https://github.com/astral-sh/uv) | |
| 60 | -git clone https://github.com/tenseleyFlow/DocumentLanguageModel.git | |
| 61 | -cd DocumentLanguageModel | |
| 62 | -uv sync | |
| 63 | -uv run dlm --help | |
| 60 | +brew tap tenseleyFlow/tap | |
| 61 | +brew install dlm | |
| 62 | + | |
| 63 | +# Ollama is required for `dlm export` smoke runs: | |
| 64 | +brew install ollama | |
| 64 | 65 | ``` |
| 65 | 66 | |
| 66 | -### From PyPI (v1.0 target) | |
| 67 | +`brew install dlm` pulls in a vendored `llama.cpp` source tree for | |
| 68 | +GGUF conversion and declares `depends_on "llama.cpp"` for the | |
| 69 | +compiled `llama-quantize` / `llama-imatrix` binaries. On NVIDIA | |
| 70 | +hardware, unlock QLoRA 4-bit after install: | |
| 67 | 71 | |
| 68 | 72 | ```sh |
| 69 | -# Portable install — torch, transformers, peft, trl, datasets included. | |
| 70 | -pip install dlm | |
| 71 | - | |
| 72 | -# Add the CUDA extra for QLoRA 4-bit on Ampere+. | |
| 73 | -pip install "dlm[cuda]" | |
| 73 | +$(brew --prefix dlm)/libexec/venv/bin/pip install 'dlm[cuda]' | |
| 74 | 74 | ``` |
| 75 | 75 | |
| 76 | -For export: install [Ollama](https://ollama.com/) separately — minimum | |
| 77 | -version is pinned in the CLI; `dlm doctor` reports it. For GGUF | |
| 78 | -conversion, the repo vendors `llama.cpp` as a submodule; one-time | |
| 79 | -build: | |
| 76 | +### From source (contributors) | |
| 80 | 77 | |
| 81 | 78 | ```sh |
| 79 | +# Python 3.11+ and uv (https://github.com/astral-sh/uv). | |
| 80 | +git clone https://github.com/tenseleyFlow/DocumentLanguageModel.git | |
| 81 | +cd DocumentLanguageModel | |
| 82 | +uv sync | |
| 83 | +# One-time: build the vendored llama.cpp binaries for `dlm export`. | |
| 82 | 84 | scripts/bump-llama-cpp.sh build |
| 85 | +uv run dlm --help | |
| 83 | 86 | ``` |
| 84 | 87 | |
| 88 | +We deliberately don't publish to PyPI — too easy to ship unfinished | |
| 89 | +work to a permanent-file-archive with 5 GB of transitive deps. See | |
| 90 | +[CONTRIBUTING.md](./CONTRIBUTING.md) for the release flow. | |
| 91 | + | |
| 85 | 92 | ## First run |
| 86 | 93 | |
| 87 | 94 | ```sh |
pyproject.tomlmodified@@ -1,6 +1,6 @@ | ||
| 1 | 1 | [project] |
| 2 | 2 | name = "dlm" |
| 3 | -version = "0.1.0" | |
| 3 | +version = "0.9.0" | |
| 4 | 4 | description = "A text file with a .dlm extension becomes a local, trainable LLM." |
| 5 | 5 | readme = "README.md" |
| 6 | 6 | requires-python = ">=3.11" |
src/dlm/export/vendoring.pymodified@@ -12,13 +12,28 @@ Three primary artifacts: | ||
| 12 | 12 | - `llama-quantize` — compiled binary (built by cmake). Converts an |
| 13 | 13 | fp16 GGUF into one of the quant levels. |
| 14 | 14 | |
| 15 | +Lookup order for the llama.cpp source tree (convert scripts): | |
| 16 | + | |
| 17 | +1. `DLM_LLAMA_CPP_ROOT` env var — set by the Homebrew formula so | |
| 18 | + `brew install dlm` points at `libexec/vendor/llama.cpp/` without | |
| 19 | + needing an in-tree submodule. | |
| 20 | +2. `vendor/llama.cpp/` relative to the repo root — dev path. | |
| 21 | + | |
| 22 | +Binary lookup falls through to `shutil.which()` when the vendored | |
| 23 | +`build/bin/` isn't present, so `brew install llama.cpp`'s | |
| 24 | +`/opt/homebrew/bin/llama-quantize` satisfies the resolver on brew | |
| 25 | +installs. | |
| 26 | + | |
| 15 | 27 | Missing or unbuilt artifacts raise `VendoringError` with a remediation |
| 16 | -pointing at `scripts/bump-llama-cpp.sh`. The runner catches + reworks | |
| 17 | -the message for the CLI; test code can catch the bare typed error. | |
| 28 | +pointing at `scripts/bump-llama-cpp.sh` (source install) or | |
| 29 | +`brew install llama.cpp` (brew install). The runner catches + reworks | |
| 30 | +the message for the CLI. | |
| 18 | 31 | """ |
| 19 | 32 | |
| 20 | 33 | from __future__ import annotations |
| 21 | 34 | |
| 35 | +import os | |
| 36 | +import shutil | |
| 22 | 37 | from pathlib import Path |
| 23 | 38 | from typing import Final |
| 24 | 39 | |
@@ -26,6 +41,7 @@ from dlm.export.errors import VendoringError | ||
| 26 | 41 | |
| 27 | 42 | _REPO_ROOT: Final[Path] = Path(__file__).resolve().parents[3] |
| 28 | 43 | VENDOR_LLAMA_CPP: Final[Path] = _REPO_ROOT / "vendor" / "llama.cpp" |
| 44 | +_ENV_VAR: Final[str] = "DLM_LLAMA_CPP_ROOT" | |
| 29 | 45 | |
| 30 | 46 | CONVERT_HF_TO_GGUF: Final[str] = "convert_hf_to_gguf.py" |
| 31 | 47 | CONVERT_LORA_TO_GGUF: Final[str] = "convert_lora_to_gguf.py" |
@@ -54,18 +70,30 @@ _LLAMA_IMATRIX_CANDIDATES: Final[tuple[str, ...]] = ( | ||
| 54 | 70 | |
| 55 | 71 | |
| 56 | 72 | def llama_cpp_root(override: Path | None = None) -> Path: |
| 57 | - """Return the path to the vendored `llama.cpp` clone. | |
| 73 | + """Return the path to the llama.cpp source tree. | |
| 74 | + | |
| 75 | + Resolution order: | |
| 76 | + | |
| 77 | + 1. `override` kwarg (test hook; production code never passes it). | |
| 78 | + 2. `$DLM_LLAMA_CPP_ROOT` env var (set by the Homebrew formula). | |
| 79 | + 3. `vendor/llama.cpp/` at the repo root (source / dev install). | |
| 58 | 80 | |
| 59 | - `override` is a test hook — production code never passes it. | |
| 60 | - Raises `VendoringError` if the directory is missing OR is empty | |
| 61 | - (an uninitialized submodule). | |
| 81 | + Raises `VendoringError` if none of those resolve to a non-empty | |
| 82 | + directory. | |
| 62 | 83 | """ |
| 63 | - root = override or VENDOR_LLAMA_CPP | |
| 84 | + if override is not None: | |
| 85 | + root = override | |
| 86 | + elif env_override := os.environ.get(_ENV_VAR): | |
| 87 | + root = Path(env_override) | |
| 88 | + else: | |
| 89 | + root = VENDOR_LLAMA_CPP | |
| 64 | 90 | if not root.is_dir(): |
| 65 | 91 | raise VendoringError( |
| 66 | - f"vendor/llama.cpp is missing at {root}. " | |
| 67 | - "Run `git submodule update --init --recursive` and then " | |
| 68 | - "`scripts/bump-llama-cpp.sh build` to materialize the toolchain." | |
| 92 | + f"llama.cpp source tree missing at {root}. For source installs, " | |
| 93 | + "run `git submodule update --init --recursive` and then " | |
| 94 | + "`scripts/bump-llama-cpp.sh build`. For brew installs, " | |
| 95 | + f"ensure the {_ENV_VAR} env var points at a populated tree " | |
| 96 | + "(normally handled by the dlm formula)." | |
| 69 | 97 | ) |
| 70 | 98 | # An empty dir (uninitialized submodule) is the most common failure. |
| 71 | 99 | try: |
@@ -74,7 +102,9 @@ def llama_cpp_root(override: Path | None = None) -> Path: | ||
| 74 | 102 | raise VendoringError(f"cannot enumerate {root}: {exc}") from exc |
| 75 | 103 | if any_entry is None: |
| 76 | 104 | raise VendoringError( |
| 77 | - f"vendor/llama.cpp is empty at {root}. Run `git submodule update --init --recursive`." | |
| 105 | + f"llama.cpp source tree is empty at {root}. " | |
| 106 | + "Run `git submodule update --init --recursive` (source install) " | |
| 107 | + f"or unset {_ENV_VAR} and reinstall the dlm formula." | |
| 78 | 108 | ) |
| 79 | 109 | return root |
| 80 | 110 | |
@@ -89,40 +119,60 @@ def convert_lora_to_gguf_py(override: Path | None = None) -> Path: | ||
| 89 | 119 | return _resolve_script(CONVERT_LORA_TO_GGUF, override) |
| 90 | 120 | |
| 91 | 121 | |
| 92 | -def llama_quantize_bin(override: Path | None = None) -> Path: | |
| 93 | - """Path to the `llama-quantize` binary. | |
| 122 | +def _resolve_binary( | |
| 123 | + *, | |
| 124 | + name: str, | |
| 125 | + candidates: tuple[str, ...], | |
| 126 | + override: Path | None, | |
| 127 | +) -> Path: | |
| 128 | + """Find a llama.cpp binary, preferring the vendored build tree then $PATH. | |
| 94 | 129 | |
| 95 | - Checks several known build-layout locations since llama.cpp's | |
| 96 | - build output has moved between releases. If none of the | |
| 97 | - candidates exist, `VendoringError` points at the bump script's | |
| 98 | - build step. | |
| 130 | + When `override` is None and the env/vendor tree lacks a compiled | |
| 131 | + binary, fall back to `shutil.which(name)` — covers the common | |
| 132 | + `brew install llama.cpp` case where the binary lives under | |
| 133 | + `/opt/homebrew/bin/`. | |
| 99 | 134 | """ |
| 100 | 135 | root = llama_cpp_root(override) |
| 101 | - for candidate in _LLAMA_QUANTIZE_CANDIDATES: | |
| 136 | + for candidate in candidates: | |
| 102 | 137 | path = root / candidate |
| 103 | 138 | if path.is_file(): |
| 104 | 139 | return path |
| 140 | + # Fall through to PATH lookup (brew-installed llama.cpp). | |
| 141 | + on_path = shutil.which(name) | |
| 142 | + if on_path is not None: | |
| 143 | + return Path(on_path) | |
| 105 | 144 | raise VendoringError( |
| 106 | - f"llama-quantize binary not found under {root}. " | |
| 107 | - "Run `scripts/bump-llama-cpp.sh build` to compile it." | |
| 145 | + f"{name} binary not found under {root} and not on $PATH. For " | |
| 146 | + "source installs, run `scripts/bump-llama-cpp.sh build`. For " | |
| 147 | + "brew installs, `brew install llama.cpp`." | |
| 148 | + ) | |
| 149 | + | |
| 150 | + | |
| 151 | +def llama_quantize_bin(override: Path | None = None) -> Path: | |
| 152 | + """Path to the `llama-quantize` binary. | |
| 153 | + | |
| 154 | + Checks several known build-layout locations, then falls back to | |
| 155 | + `$PATH` — covers both the vendored `build/bin/llama-quantize` | |
| 156 | + (source install) and the brew `/opt/homebrew/bin/llama-quantize` | |
| 157 | + (brew install with `depends_on "llama.cpp"`). | |
| 158 | + """ | |
| 159 | + return _resolve_binary( | |
| 160 | + name="llama-quantize", | |
| 161 | + candidates=_LLAMA_QUANTIZE_CANDIDATES, | |
| 162 | + override=override, | |
| 108 | 163 | ) |
| 109 | 164 | |
| 110 | 165 | |
| 111 | 166 | def llama_imatrix_bin(override: Path | None = None) -> Path: |
| 112 | 167 | """Path to the `llama-imatrix` binary (Sprint 11.6). |
| 113 | 168 | |
| 114 | - Same resolver shape as `llama_quantize_bin` — checks several known | |
| 115 | - build-layout locations. `VendoringError` with the bump-script | |
| 116 | - pointer if the binary is absent. | |
| 169 | + Same resolver shape as `llama_quantize_bin` — checks vendored | |
| 170 | + build layouts, then `$PATH`. | |
| 117 | 171 | """ |
| 118 | - root = llama_cpp_root(override) | |
| 119 | - for candidate in _LLAMA_IMATRIX_CANDIDATES: | |
| 120 | - path = root / candidate | |
| 121 | - if path.is_file(): | |
| 122 | - return path | |
| 123 | - raise VendoringError( | |
| 124 | - f"llama-imatrix binary not found under {root}. " | |
| 125 | - "Run `scripts/bump-llama-cpp.sh build` to compile it." | |
| 172 | + return _resolve_binary( | |
| 173 | + name="llama-imatrix", | |
| 174 | + candidates=_LLAMA_IMATRIX_CANDIDATES, | |
| 175 | + override=override, | |
| 126 | 176 | ) |
| 127 | 177 | |
| 128 | 178 | |
uv.lockmodified@@ -598,7 +598,7 @@ wheels = [ | ||
| 598 | 598 | |
| 599 | 599 | [[package]] |
| 600 | 600 | name = "dlm" |
| 601 | -version = "0.1.0" | |
| 601 | +version = "0.9.0" | |
| 602 | 602 | source = { editable = "." } |
| 603 | 603 | dependencies = [ |
| 604 | 604 | { name = "cbor2" }, |