diff --git a/script/convert_params.sh b/script/convert_params.sh new file mode 100755 index 0000000000000000000000000000000000000000..e645d9ab9be0815ceb5423bc9c629184cca323f7 --- /dev/null +++ b/script/convert_params.sh @@ -0,0 +1,37 @@ + +#!/bin/sh +if [[ $# != 1 ]]; then + echo "usage: bash convert_params.sh " + exit 1 +fi + +if [[ -f $1/__palminfo__ ]]; then + echo "already converted." + exit 0 +fi + +echo "converting..." +if [[ -d $1/params ]]; then + cd $1/params +else + cd $1 +fi + +mkdir .palm.backup + +for file in $(ls *) + do cp $file .palm.backup; mv $file "__paddlepalm_"$file +done +tar -cf __rawmodel__ .palm.backup/* +rm .palm.backup/* +mv __rawmodel__ .palm.backup +# find . ! -name '__rawmodel__' -exec rm {} + +tar -cf __palmmodel__ __paddlepalm_* +touch __palminfo__ +ls __paddlepalm_* > __palminfo__ +rm __paddlepalm_* + +cd - >/dev/null + +echo "done!" + diff --git a/script/download_pretrain_backbone.sh b/script/download_pretrain_backbone.sh new file mode 100755 index 0000000000000000000000000000000000000000..bc64a428801cf08e3f184ad50955dd706187341e --- /dev/null +++ b/script/download_pretrain_backbone.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +if [[ $# != 1 ]]; then + echo "Usage: bash download_pretrain.sh " + exit 1 +fi + +if [[ $1 == 'bert' ]]; then + name="bert" + link="https://bert-models.bj.bcebos.com/uncased_L-24_H-1024_A-16.tar.gz" + packname="uncased_L-24_H-1024_A-16.tar.gz" + dirname="uncased_L-24_H-1024_A-16" +elif [[ $1 == 'ernie' ]]; then + name="ernie" + link="https://ernie.bj.bcebos.com/ERNIE_Large_en_stable-2.0.0.tar.gz" + packname="ERNIE_Large_en_stable-2.0.0.tar.gz" +else + echo "$1 is currently not supported." + exit 1 +fi + +if [[ ! -d pretrain_model ]]; then + mkdir pretrain_model +fi + +cd pretrain_model +mkdir $name +cd $name +echo "downloading ${name}..." +wget --no-check-certificate $link +echo "decompressing..." +tar -zxf $packname +rm -rf $packname +if [[ $dirname != "" ]]; then + mv $dirname/* . + rm -rf $dirname +fi + +cd ../.. + + diff --git a/script/recover_params.sh b/script/recover_params.sh new file mode 100755 index 0000000000000000000000000000000000000000..a99ceb500f5e9f70dfa8660d8c308ec1f0841c5b --- /dev/null +++ b/script/recover_params.sh @@ -0,0 +1,33 @@ + +#!/bin/sh +if [[ $# != 1 ]]; then + echo "usage: bash recover_params.sh " + exit 1 +fi + +if [[ ! -d $1 ]]; then + echo "$1 not found." + exit 1 +fi + +if [[ ! -f $1/__palmmodel__ ]]; then + echo "paddlepalm model not found." + exit 1 +fi + +echo "recovering..." +if [[ -d $1/params ]]; then + cd $1/params +else + cd $1 +fi +rm __palm* +mv .palm.backup/__rawmodel__ . +rm -rf .palm.backup +tar -xf __rawmodel__ +mv .palm.backup/* . +rm __rawmodel__ + +rm -rf .palm.backup +cd - >/dev/null +