diff --git a/paddlegan-wechaty-demo/README.md b/paddlegan-wechaty-demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4d3e15f56926235984d5ef7c43e6078c316452c4 --- /dev/null +++ b/paddlegan-wechaty-demo/README.md @@ -0,0 +1,131 @@ +# PaddleGAN-WeChaty-Demo + +本示例将展示如何在[Wechaty](https://github.com/Wechaty/wechaty)中使用PaddleGAN的多种能力。 + +基本原理：通过[Wechaty](https://github.com/Wechaty/wechaty)获取微信接收的消息，然后使用PaddleGAN中的人脸动作迁移算法`first order motion`模型，将静态照片转换成动态趣味视频，最终以微信消息的形式发送。 + +## 风险提示 + +本项目采用的api为第三方——Wechaty提供，**非微信官方api**，用户需承担来自微信方的使用风险。 +在运行项目的过程中，建议尽量选用**新注册的小号**进行测试，不要用自己的常用微信号。 + +## Wechaty + +关于Wechaty和python-wechaty，请查阅以下官方repo： +- [Wechaty](https://github.com/Wechaty/wechaty) +- [python-wechaty](https://github.com/wechaty/python-wechaty) +- [python-wechaty-getting-started](https://github.com/wechaty/python-wechaty-getting-started/blob/master/README.md) + + +## 环境准备 + +- 系统环境：Linux, MacOS, Windows +- python3.7+ + + +## 安装和使用 + +1. 安装PaddleGAN，详情请见[安装方式](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/install.md) + + ```shell + git clone https://github.com/PaddlePaddle/paddlegan + cd paddlegan-wechaty-demo + ``` + +2. 安装依赖 —— paddlepaddle, ppgan, wechaty + + ```shell + pip install -r requirements.txt + ``` + +3. 安装项目所需的PaddleGAN的module + + 此demo以`first order motion`为示例，其他module根据项目所需安装，更多的模型请查阅[PaddleGAN模型API接口说明](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/apis/apps.md)。 + +4. Set token for your bot + + 在当前系统的环境变量中，配置以下与`WECHATY_PUPPET`相关的两个变量。 + 关于其作用详情和TOKEN的获取方式，请查看[Wechaty Puppet Services](https://wechaty.js.org/docs/puppet-services/)。 + + ```shell + export WECHATY_PUPPET=wechaty-puppet-service + export WECHATY_PUPPET_SERVICE_TOKEN=your_token_at_here + ``` + + [Paimon](https://wechaty.js.org/docs/puppet-services/paimon/)的短期TOKEN经测试可用，比赛期间将提供选手一个可使用1个月的token，大家可自行使用。 + +4. Run the bot + + ```shell + python examples/paddleGAN_fom.py + ``` + 运行后，可以通过微信移动端扫码登陆，登陆成功后则可正常使用。 + +## 运行效果 + +在`examples/paddleGAN_fom.py`中，通过以下几行代码即可实例化一个`first order motion`的模型。 + +```python +# Initialize a PaddleGAN first order motion model +from ppgan.apps import FirstOrderPredictor +animate = FirstOrderPredictor(output="test_fom", filename="result.mp4",\ + relative=True, adapt_scale=True) +``` + +`on_message`方法是接收到消息时的回调函数，可以通过自定义的条件(譬如消息类型、消息来源、消息文字是否包含关键字、是否群聊消息等等)来判断是否回复信息，消息的更多属性和条件可以参考[Class Message](https://github.com/Wechaty/wechaty#3-class-message)。 + +本示例中的`on_message`方法的代码如下， + +```python +async def on_message(msg: Message): + """ + Message Handler for the Bot + """ + ### PaddleGAN fom + + global fom, source, driving + + if isinstance(msg.text(), str) and len(msg.text()) > 0 \ + and msg._payload.type == MessageType.MESSAGE_TYPE_TEXT \ + and "fom" in msg.text(): + bot_response = u"好嘞, 给我发个图片和驱动视频吧" + fom = True + await msg.say(bot_response) + + if fom and msg._payload.type == MessageType.MESSAGE_TYPE_IMAGE: + fileBox = await msg.to_file_box() + await fileBox.to_file("test_fom/source.jpg", True) + + bot_response = u"好嘞, 收到图片" + await msg.say(bot_response) + + source = True + + if fom and msg._payload.type == MessageType.MESSAGE_TYPE_VIDEO: + fileBox = await msg.to_file_box() + await fileBox.to_file("test_fom/driving.mp4", True) + + bot_response = u"好嘞, 收到驱动视频" + await msg.say(bot_response) + + driving = True + + if source and driving: + bot_response = u"都收到啦，稍等一下嘿嘿" + await msg.say(bot_response) + source = False + driving = False + fom = False + animate.run("test_fom/source.jpg", "test_fom/driving.mp4") + file_box = FileBox.from_file("test_fom/result.mp4") + await msg.say(file_box) + + ### + +``` + +脚本成功运行后，所登陆的账号即可作为一个Chatbot，下图左侧的内容由Chatbot生成和回复。 +
+ +
+ diff --git a/paddlegan-wechaty-demo/examples/paddleGAN_fom.py b/paddlegan-wechaty-demo/examples/paddleGAN_fom.py new file mode 100755 index 0000000000000000000000000000000000000000..614ed1b2655a1324d44b30710d317661e6062c7e --- /dev/null +++ b/paddlegan-wechaty-demo/examples/paddleGAN_fom.py @@ -0,0 +1,110 @@ +from collections import deque +import os +import asyncio + +from wechaty import ( + Contact, + FileBox, + Message, + Wechaty, + ScanStatus, +) +from wechaty_puppet import MessageType + +# Initialize a PaddleGAN fom model +from ppgan.apps import FirstOrderPredictor +animate = FirstOrderPredictor(output="test_fom", filename="result.mp4",\ + relative=True, adapt_scale=True) +fom = False +source = False +driving = False + + +async def on_message(msg: Message): + """ + Message Handler for the Bot + """ + ### PaddleGAN fom + + global fom, source, driving + + if isinstance(msg.text(), str) and len(msg.text()) > 0 \ + and msg._payload.type == MessageType.MESSAGE_TYPE_TEXT \ + and "fom" in msg.text(): + bot_response = u"好嘞, 给我发个图片和驱动视频吧" + fom = True + await msg.say(bot_response) + + if fom and msg._payload.type == MessageType.MESSAGE_TYPE_IMAGE: + fileBox = await msg.to_file_box() + await fileBox.to_file("test_fom/source.jpg", True) + + bot_response = u"好嘞, 收到图片" + await msg.say(bot_response) + + source = True + + if fom and msg._payload.type == MessageType.MESSAGE_TYPE_VIDEO: + fileBox = await msg.to_file_box() + await fileBox.to_file("test_fom/driving.mp4", True) + + bot_response = u"好嘞, 收到驱动视频" + await msg.say(bot_response) + + driving = True + + if source and driving: + bot_response = u"都收到啦，稍等一下嘿嘿" + await msg.say(bot_response) + source = False + driving = False + fom = False + animate.run("test_fom/source.jpg", "test_fom/driving.mp4") + file_box = FileBox.from_file("test_fom/result.mp4") + await msg.say(file_box) + + +async def on_scan( + qrcode: str, + status: ScanStatus, + _data, +): + """ + Scan Handler for the Bot + """ + print('Status: ' + str(status)) + print('View QR Code Online: https://wechaty.js.org/qrcode/' + qrcode) + + +async def on_login(user: Contact): + """ + Login Handler for the Bot + """ + print(user) + # TODO: To be written + + +async def main(): + """ + Async Main Entry + """ + # + # Make sure we have set WECHATY_PUPPET_SERVICE_TOKEN in the environment variables. + # + if 'WECHATY_PUPPET_SERVICE_TOKEN' not in os.environ: + print(''' + Error: WECHATY_PUPPET_SERVICE_TOKEN is not found in the environment variables + You need a TOKEN to run the Python Wechaty. Please goto our README for details + https://github.com/wechaty/python-wechaty-getting-started/#wechaty_puppet_service_token + ''') + + bot = Wechaty() + + bot.on('scan', on_scan) + bot.on('login', on_login) + bot.on('message', on_message) + + await bot.start() + + +asyncio.run(main()) diff --git a/paddlegan-wechaty-demo/requirements.txt b/paddlegan-wechaty-demo/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..af9ec4d750fd7a27ef4db63966c6e873be645661 --- /dev/null +++ b/paddlegan-wechaty-demo/requirements.txt @@ -0,0 +1,3 @@ +paddlepaddle >= 2.1.0 +ppgan >= 2.0.0 +wechaty ~= 0.7dev16 diff --git a/paddlegan-wechaty-demo/test_fom/=2.0.0 b/paddlegan-wechaty-demo/test_fom/=2.0.0 new file mode 100644 index 0000000000000000000000000000000000000000..936c280b3e48f37be8b1989650a4b7118b393f8b --- /dev/null +++ b/paddlegan-wechaty-demo/test_fom/=2.0.0 @@ -0,0 +1,34 @@ +Requirement already satisfied: ppgan in /workspace/paddle_gan_new/PaddleGAN (2.0.0) +Requirement already satisfied: tqdm in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (4.54.1) +Requirement already satisfied: PyYAML>=5.1 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (5.3.1) +Requirement already satisfied: scikit-image>=0.14.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (0.18.1) +Requirement already satisfied: scipy>=1.1.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (1.5.4) +Requirement already satisfied: opencv-python in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (4.2.0.32) +Requirement already satisfied: imageio-ffmpeg in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (0.4.3) +Requirement already satisfied: librosa==0.7.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (0.7.0) +Requirement already satisfied: numba==0.48 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (0.48.0) +Requirement already satisfied: easydict in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (1.9) +Requirement already satisfied: munch in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from ppgan) (2.5.0) +Requirement already satisfied: joblib>=0.12 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (1.0.0) +Requirement already satisfied: numpy>=1.15.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (1.20.2) +Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (0.23.2) +Requirement already satisfied: six>=1.3 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (1.16.0) +Requirement already satisfied: audioread>=2.0.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (2.1.9) +Requirement already satisfied: soundfile>=0.9.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (0.10.3.post1) +Requirement already satisfied: resampy>=0.2.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (0.2.2) +Requirement already satisfied: decorator>=3.0.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from librosa==0.7.0->ppgan) (4.4.2) +Requirement already satisfied: setuptools in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from numba==0.48->ppgan) (51.0.0.post20201207) +Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from numba==0.48->ppgan) (0.31.0) +Requirement already satisfied: pillow!=7.1.0,!=7.1.1,>=4.3.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (8.0.1) +Requirement already satisfied: imageio>=2.3.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (2.9.0) +Requirement already satisfied: PyWavelets>=1.1.1 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (1.1.1) +Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (3.3.3) +Requirement already satisfied: networkx>=2.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (2.5) +Requirement already satisfied: tifffile>=2019.7.26 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-image>=0.14.0->ppgan) (2020.12.8) +Requirement already satisfied: python-dateutil>=2.1 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.14.0->ppgan) (2.8.1) +Requirement already satisfied: cycler>=0.10 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.14.0->ppgan) (0.10.0) +Requirement already satisfied: kiwisolver>=1.0.1 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.14.0->ppgan) (1.3.1) +Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image>=0.14.0->ppgan) (2.4.7) +Requirement already satisfied: threadpoolctl>=2.0.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from scikit-learn!=0.19.0,>=0.14.0->librosa==0.7.0->ppgan) (2.1.0) +Requirement already satisfied: cffi>=1.0 in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from soundfile>=0.9.0->librosa==0.7.0->ppgan) (1.14.5) +Requirement already satisfied: pycparser in /root/miniconda3/envs/py37/lib/python3.7/site-packages (from cffi>=1.0->soundfile>=0.9.0->librosa==0.7.0->ppgan) (2.20)