Skip to content

Commit

Permalink
feat: add scripts to mirror hf model to modelscope
Browse files Browse the repository at this point in the history
  • Loading branch information
wsxiaoys committed Sep 26, 2023
1 parent 88947aa commit 54b5e42
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 0 deletions.
2 changes: 2 additions & 0 deletions experimental/copy-to-modelscope/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
hf_model
ms_model
3 changes: 3 additions & 0 deletions experimental/copy-to-modelscope/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# copy-to-modelscope

Scripts to copy huggingface model to modelscope
54 changes: 54 additions & 0 deletions experimental/copy-to-modelscope/main.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash
set -e

MODEL_ID=$1
ACCESS_TOKEN=$2

usage() {
echo "Usage: $0 <model_id> <access_token>"
exit 1
}

if [ -z "${MODEL_ID}" ]; then
usage
fi

git clone https://huggingface.co/$MODEL_ID hf_model
git clone https://oauth2:${ACCESS_TOKEN}@www.modelscope.cn/$MODEL_ID.git ms_model

echo "Sync directory"
rsync -a --exclude '.git' hf_model/ ms_model/

echo "Create README.md"
cat <<EOF >ms_model/README.md
---
license: other
tasks:
- text-generation
---
# ${MODEL_ID}
This is an mirror of [${MODEL_ID}](https://huggingface.co/${MODEL_ID}).
EOF

echo "Create configuration.json"
cat <<EOF >ms_model/configuration.json
{
"framework": "pytorch",
"task": "text-generation",
"pipeline": {
"type": "text-generation-pipeline"
}
}
EOF

set -x
cd ms_model
git add .
git commit -m "sync with upstream"
git push origin

echo "Success!"
rm -rf hf_model
rm -rf ms_model

0 comments on commit 54b5e42

Please sign in to comment.