Skip to content

Commit

Permalink
Fix merge
Browse files Browse the repository at this point in the history
  • Loading branch information
NielsRogge committed Dec 27, 2024
2 parents 9e54274 + 11274e4 commit 2278ad3
Show file tree
Hide file tree
Showing 252 changed files with 6,013 additions and 1,673 deletions.
70 changes: 70 additions & 0 deletions .github/workflows/blob-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
name: Blob - Version and Release

on:
workflow_dispatch:
inputs:
newversion:
type: choice
description: "Semantic Version Bump Type"
default: patch
options:
- patch
- minor
- major

concurrency:
group: "push-to-main"

defaults:
run:
working-directory: packages/blob

jobs:
version_and_release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
# Needed to push the tag and the commit on the main branch, otherwise we get:
# > Run git push --follow-tags
# remote: error: GH006: Protected branch update failed for refs/heads/main.
# remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
token: ${{ secrets.BOT_ACCESS_TOKEN }}
- run: corepack enable
- uses: actions/setup-node@v3
with:
node-version: "20"
cache: "pnpm"
cache-dependency-path: |
packages/blob/pnpm-lock.yaml
# setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
registry-url: "https://registry.npmjs.org"
- run: pnpm install
- run: git config --global user.name machineuser
- run: git config --global user.email [email protected]
- run: |
PACKAGE_VERSION=$(node -p "require('./package.json').version")
BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
# Update package.json with the new version
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
git commit . -m "🔖 @huggingface/blob $BUMPED_VERSION"
git tag "blob-v$BUMPED_VERSION"
- run: pnpm publish --no-git-checks .
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- run: git pull --rebase && git push --follow-tags
# hack - reuse actions/setup-node@v3 just to set a new registry
- uses: actions/setup-node@v3
with:
node-version: "20"
registry-url: "https://npm.pkg.github.com"
# Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
# - run: pnpm publish --no-git-checks .
# env:
# NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Update Doc"
uses: peter-evans/repository-dispatch@v2
with:
event-type: doc-build
token: ${{ secrets.BOT_ACCESS_TOKEN }}
73 changes: 73 additions & 0 deletions .github/workflows/dduf-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: DDUF - Version and Release

on:
workflow_dispatch:
inputs:
newversion:
type: choice
description: "Semantic Version Bump Type"
default: patch
options:
- patch
- minor
- major

concurrency:
group: "push-to-main"

defaults:
run:
working-directory: packages/dduf

jobs:
version_and_release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
# Needed to push the tag and the commit on the main branch, otherwise we get:
# > Run git push --follow-tags
# remote: error: GH006: Protected branch update failed for refs/heads/main.
# remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
token: ${{ secrets.BOT_ACCESS_TOKEN }}
- run: corepack enable
- uses: actions/setup-node@v3
with:
node-version: "20"
cache: "pnpm"
cache-dependency-path: |
packages/dduf/pnpm-lock.yaml
# setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
registry-url: "https://registry.npmjs.org"
- run: pnpm install
- run: git config --global user.name machineuser
- run: git config --global user.email [email protected]
- run: |
PACKAGE_VERSION=$(node -p "require('./package.json').version")
BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
# Update package.json with the new version
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
git commit . -m "🔖 @huggingface/dduf $BUMPED_VERSION"
git tag "dduf-v$BUMPED_VERSION"
- name: "Check Deps are published before publishing this package"
run: pnpm -w check-deps blob

- run: pnpm publish --no-git-checks .
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- run: git pull --rebase && git push --follow-tags
# hack - reuse actions/setup-node@v3 just to set a new registry
- uses: actions/setup-node@v3
with:
node-version: "20"
registry-url: "https://npm.pkg.github.com"
# Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
# - run: pnpm publish --no-git-checks .
# env:
# NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Update Doc"
uses: peter-evans/repository-dispatch@v2
with:
event-type: doc-build
token: ${{ secrets.BOT_ACCESS_TOKEN }}
4 changes: 4 additions & 0 deletions .github/workflows/gguf-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ jobs:
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
git commit . -m "🔖 @huggingface/gguf $BUMPED_VERSION"
git tag "gguf-v$BUMPED_VERSION"
- name: "Check Deps are published before publishing this package"
run: pnpm -w check-deps tasks

- run: pnpm publish --no-git-checks .
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
Expand Down
38 changes: 2 additions & 36 deletions .github/workflows/hub-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,42 +53,8 @@ jobs:
git commit -m "🔖 @huggingface/hub $BUMPED_VERSION"
git tag "hub-v$BUMPED_VERSION"
- name: Make sure that the latest version of @huggingface/tasks is consistent with the local version
run: |
LOCAL_TASKS_VERSION=$(node -p "require('./package.json').version")
REMOTE_TASKS_VERSION=$(npm view @huggingface/tasks version)
# If the versions are different, error
if [ "$LOCAL_TASKS_VERSION" != "$REMOTE_TASKS_VERSION" ]; then
echo "Error: The local @huggingface/tasks package version ($LOCAL_TASKS_VERSION) differs from the remote version ($REMOTE_TASKS_VERSION). Release halted."
exit 1
fi
npm pack @huggingface/tasks
mv huggingface-tasks-$LOCAL_TASKS_VERSION.tgz tasks-local.tgz
npm pack @huggingface/tasks@$REMOTE_TASKS_VERSION
mv huggingface-tasks-$REMOTE_TASKS_VERSION.tgz tasks-remote.tgz
# Compute checksum of local tar. We need to extract both tar since the remote compression might be different
tar -xf tasks-local.tgz
LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Local package checksum: $LOCAL_CHECKSUM"
rm -Rf package
tar -xf tasks-remote.tgz
REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Remote package checksum: $REMOTE_CHECKSUM"
rm -Rf package
if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
echo "Checksum Verification Failed: The local @huggingface/tasks package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
exit 1
fi
echo "Checksum Verification Successful: The local and remote @huggingface/tasks packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
working-directory: packages/tasks
- name: "Check Deps are published before publishing this package"
run: pnpm -w check-deps tasks

- run: pnpm publish --no-git-checks .
env:
Expand Down
38 changes: 2 additions & 36 deletions .github/workflows/inference-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,42 +53,8 @@ jobs:
git commit -m "🔖 @huggingface/inference $BUMPED_VERSION"
git tag "inference-v$BUMPED_VERSION"
- name: Make sure that the latest version of @huggingface/tasks is consistent with the local version
run: |
LOCAL_TASKS_VERSION=$(node -p "require('./package.json').version")
REMOTE_TASKS_VERSION=$(npm view @huggingface/tasks version)
# If the versions are different, error
if [ "$LOCAL_TASKS_VERSION" != "$REMOTE_TASKS_VERSION" ]; then
echo "Error: The local @huggingface/tasks package version ($LOCAL_TASKS_VERSION) differs from the remote version ($REMOTE_TASKS_VERSION). Release halted."
exit 1
fi
npm pack @huggingface/tasks
mv huggingface-tasks-$LOCAL_TASKS_VERSION.tgz tasks-local.tgz
npm pack @huggingface/tasks@$REMOTE_TASKS_VERSION
mv huggingface-tasks-$REMOTE_TASKS_VERSION.tgz tasks-remote.tgz
# Compute checksum of local tar. We need to extract both tar since the remote compression might be different
tar -xf tasks-local.tgz
LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Local package checksum: $LOCAL_CHECKSUM"
rm -Rf package
tar -xf tasks-remote.tgz
REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Remote package checksum: $REMOTE_CHECKSUM"
rm -Rf package
if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
echo "Checksum Verification Failed: The local @huggingface/tasks package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
exit 1
fi
echo "Checksum Verification Successful: The local and remote @huggingface/tasks packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
working-directory: packages/tasks
- name: "Check Deps are published before publishing this package"
run: pnpm -w check-deps tasks

- run: pnpm publish --no-git-checks .
env:
Expand Down
37 changes: 0 additions & 37 deletions .github/workflows/tasks-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,43 +50,6 @@ jobs:
git commit . -m "🔖 @huggingface/tasks $BUMPED_VERSION"
git tag "tasks-v$BUMPED_VERSION"
- name: Make sure that the latest version of @huggingface/gguf is consistent with the local version
run: |
LOCAL_GGUF_VERSION=$(node -p "require('./package.json').version")
REMOTE_GGUF_VERSION=$(npm view @huggingface/gguf version)
# If the versions are different, error
if [ "$LOCAL_GGUF_VERSION" != "$REMOTE_GGUF_VERSION" ]; then
echo "Error: The local @huggingface/gguf package version ($LOCAL_GGUF_VERSION) differs from the remote version ($REMOTE_GGUF_VERSION). Release halted."
exit 1
fi
npm pack @huggingface/gguf
mv huggingface-gguf-$LOCAL_GGUF_VERSION.tgz gguf-local.tgz
npm pack @huggingface/gguf@$REMOTE_GGUF_VERSION
mv huggingface-gguf-$REMOTE_GGUF_VERSION.tgz gguf-remote.tgz
# Compute checksum of local tar. We need to extract both tar since the remote compression might be different
tar -xf gguf-local.tgz
LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Local package checksum: $LOCAL_CHECKSUM"
rm -Rf package
tar -xf gguf-remote.tgz
REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
echo "Remote package checksum: $REMOTE_CHECKSUM"
rm -Rf package
if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
echo "Checksum Verification Failed: The local @huggingface/gguf package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
exit 1
fi
echo "Checksum Verification Successful: The local and remote @huggingface/gguf packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
working-directory: packages/gguf

- run: pnpm publish --no-git-checks .
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ jobs:
working-directory: e2e
run: |
sleep 3
pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
pnpm --filter inference --filter hub --filter tasks --filter gguf publish --force --no-git-checks --registry http://localhost:4874/
pnpm i --filter root --filter inference... --filter hub... --filter tasks-gen --frozen-lockfile
pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
- name: E2E test - test yarn install
working-directory: e2e/ts
Expand Down
63 changes: 63 additions & 0 deletions .github/workflows/update-specs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: Tasks - Update specs

on:
workflow_dispatch:
schedule:
- cron: "0 3 * * *" # Every day at 3am

concurrency:
group: update-specs
cancel-in-progress: true

defaults:
run:
working-directory: packages/tasks-gen

jobs:
pull_request:
runs-on: ubuntu-latest
steps:
# Setup
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "20"
- name: Install pnpm
uses: pnpm/action-setup@v2
with:
run_install: true

# Generate specs
- run: pnpm run inference-tgi-import
- run: pnpm run inference-tei-import
- run: pnpm run inference-codegen

# Check changes
- run: git status

# Create or update Pull Request
- name: Create Pull Request
uses: peter-evans/create-pull-request@v7
with:
token: ${{ secrets.TOKEN_INFERENCE_SYNC_BOT }}
commit-message: Update tasks specs (automated commit)
branch: update-tasks-specs-automated-pr
delete-branch: true
title: "[Bot] Update tasks specs"
body: |
This PR updates the @huggingface/tasks specs. It has been generated by running:
```sh
pnpm run inference-tgi-import
pnpm run inference-tei-import
pnpm run inference-codegen
```
This PR was automatically created by the [Tasks - Update specs workflow](https://github.com/huggingface/huggingface.js/blob/main/.github/update-specs.yml).
Make sure the changes are correct before merging.
labels: |
tasks
specs
reviewers: |
Wauplin
hanouticelina
Loading

0 comments on commit 2278ad3

Please sign in to comment.