fix(*): Support for send 25Mb+ files (#771)

* fix(*): Support for send 25Mb+ files

1. Upload a file larger than 25Mb.
2. Limit the upload file to no more than 100Mb.
3. According to webwxapp perfect upload processing.
4. Increase the upload and handling of upload exceptions.

* refactor(*): add support download file and send it again

1. add MediaMessage.saveFile() , used to save attachment
2. Refactor MediaMessage.forward(), support send to Room and Contact
array, add support download big file and send it again.

* fix(*): fix reuse data

* dev code

* fix(message): Remove support for foward 25Mb+ files in forward()

Because webWx restrictions, more than 25 MB of files can not be
downloaded, it can not be forwarded.

* fix: fix send files after restart

Closes 777

* feat(MediaMessage): support forward their own upload 25Mb+ file

* style: clean code

* docs: Modify forward support 25Mb+ file descrption

* fix: fix after upload file,display is cancel button

* clean code

* refactor(MediaMessage): Modify saveFile() return Promise<void>

* clean code

src/message.ts

@@ -64,6 +64,8 @@ export interface MsgRawObj {
   FileName:         string,  // FileName: '钢甲互联项目BP1108.pdf',
   FileSize:         number,  // FileSize: '2845701',
   MediaId:          string,  // MediaId: '@crypt_b1a45e3f_c21dceb3ac01349...
+  MMFileExt:        string,  // doc, docx ... 'undefined'?
+  Signature:        string,  // checkUpload return the signature used to upload large files
 
   MMAppMsgFileExt:      string,  // doc, docx ... 'undefined'?
   MMAppMsgFileSize:     string,  // '2.7MB',
@@ -141,7 +143,6 @@ export interface MsgRawObj {
   MsgIdBeforeTranspond?: string,  // oldMsg.MsgIdBeforeTranspond || oldMsg.MsgId,
   isTranspond?: boolean,
   MMSourceMsgId?: string,
-  sendByLocal?: boolean, // If transpond file, it must is false, not need to upload. And, can't to call createMessage(), it set to true
   MMSendContent?: string,
 
   MMIsChatRoom?: boolean,
@@ -713,7 +714,7 @@ export class Message implements Sayable {
 
     contactList = [].concat.apply([],
       mentionList.map(nameStr => room.memberAll(nameStr))
-      .filter(contact => !!contact),
+        .filter(contact => !!contact),
     )
 
     if (contactList.length === 0) {
@@ -743,12 +744,12 @@ export class Message implements Sayable {
       }
 
     } catch (e) {
-        log.error('Message', 'ready() exception: %s', e.stack)
-        Raven.captureException(e)
-        // console.log(e)
-        // this.dump()
-        // this.dumpRaw()
-        throw e
+      log.error('Message', 'ready() exception: %s', e.stack)
+      Raven.captureException(e)
+      // console.log(e)
+      // this.dump()
+      // this.dumpRaw()
+      throw e
     }
   }
 
@@ -1111,17 +1112,50 @@ export class MediaMessage extends Message {
       if (!this.obj.url) {
         throw new Error('no url')
       }
-      log.verbose('MediaMessage', 'stream() url: %s', this.obj.url)
+      log.verbose('MediaMessage', 'readyStream() url: %s', this.obj.url)
       return UtilLib.urlStream(this.obj.url, cookies)
     } catch (e) {
-      log.warn('MediaMessage', 'stream() exception: %s', e.stack)
+      log.warn('MediaMessage', 'readyStream() exception: %s', e.stack)
       Raven.captureException(e)
       throw e
     }
   }
 
-  public forward(room: Room): Promise<boolean>
-  public forward(contact: Contact): Promise<boolean>
+  /**
+   * save file
+   *
+   * @param filePath save file
+   */
+  public async saveFile(filePath: string): Promise<void> {
+    if (!filePath) {
+      throw new Error('saveFile() filePath is invalid')
+    }
+    log.silly('MediaMessage', `saveFile() filePath:'${filePath}'`)
+    if (fs.existsSync(filePath)) {
+      throw new Error('saveFile() file does exist!')
+    }
+    const writeStream = fs.createWriteStream(filePath)
+    let readStream
+    try {
+      readStream = await this.readyStream()
+    } catch (e) {
+      log.error('MediaMessage', `saveFile() call readyStream() error: ${e.message}`)
+      throw new Error(`saveFile() call readyStream() error: ${e.message}`)
+    }
+    await new Promise((resolve, reject) => {
+      readStream.pipe(writeStream)
+      readStream
+        .once('end', resolve)
+        .once('error', reject)
+    })
+      .catch(e => {
+        log.error('MediaMessage', `saveFile() error: ${e.message}`)
+        throw e
+      })
+  }
+
+  public forward(room: Room|Room[]): Promise<boolean>
+  public forward(contact: Contact|Contact[]): Promise<boolean>
   /**
    * Forward the received message.
    *
@@ -1150,30 +1184,52 @@ export class MediaMessage extends Message {
    *   EMOJI                    = 8,
    * }
    * ```
-   * But, it should be noted that when forwarding ATTACH type message, if the file size is greater than 25Mb, the forwarding will fail.
-   * The reason is that the server limits the forwarding of files above 25Mb. You need to download the file and use `new MediaMessage (file)` to send the file.
+   * It should be noted that when forwarding ATTACH type message, if the file size is greater than 25Mb, the forwarding will fail.
+   * The reason is that the server shields the web wx to download more than 25Mb files with a file size of 0.
+   *
+   * But if the file is uploaded by you using wechaty, you can forward it.
+   * You need to detect the following conditions in the message event, which can be forwarded if it is met.
+   *
+   * ```javasrcipt
+   * .on('message', async m => {
+   *   if (m.self() && m.rawObj && m.rawObj.Signature) {
+   *     // Filter the contacts you have forwarded
+   *     const msg = <MediaMessage> m
+   *     await msg.forward()
+   *   }
+   * })
+   * ```
    *
-   * @param {(Room | Contact)} sendTo
+   * @param {(Sayable | Sayable[])} sendTo Room or Contact, or array
    * The recipient of the message, the room, or the contact
    * @returns {Promise<boolean>}
    * @memberof MediaMessage
    */
-  public forward(sendTo: Room|Contact): Promise<boolean> {
+  public async forward(sendTo: Room|Room[]|Contact|Contact[]): Promise<boolean> {
     if (!this.rawObj) {
       throw new Error('no rawObj!')
     }
     let m = Object.assign({}, this.rawObj)
     const newMsg = <MsgRawObj>{}
-    const fileSizeLimit = 25 * 1024 * 1024
-    let id = ''
+    const largeFileSize = 25 * 1024 * 1024
+    let ret = false
     // if you know roomId or userId, you can use `Room.load(roomId)` or `Contact.load(userId)`
-    if (sendTo instanceof Room || sendTo instanceof Contact) {
-      id = sendTo.id
-    } else {
-      throw new Error('param must be Room or Contact!')
+    let sendToList: Contact[] = [].concat(sendTo as any || [])
+    sendToList = sendToList.filter(s => {
+      if ((s instanceof Room || s instanceof Contact) && s.id) {
+        return true
+      }
+      return false
+    }) as Contact[]
+    if (sendToList.length < 1) {
+      throw new Error('param must be Room or Contact and array')
+    }
+    if (m.FileSize >= largeFileSize && !m.Signature) {
+      // if has RawObj.Signature, can forward the 25Mb+ file
+      log.warn('MediaMessage', 'forward() Due to webWx restrictions, more than 25MB of files can not be downloaded and can not be forwarded.')
+      return false
     }
 
-    newMsg.ToUserName = id
     newMsg.FromUserName = config.puppetInstance().userId || ''
     newMsg.isTranspond = true
     newMsg.MsgIdBeforeTranspond = m.MsgIdBeforeTranspond || m.MsgId
@@ -1184,26 +1240,13 @@ export class MediaMessage extends Message {
 
     // The following parameters need to be overridden after calling createMessage()
 
-    // If you want to forward the file, would like to skip the duplicate upload, sendByLocal must be false.
-    // But need to pay attention to file.size> 25Mb, due to the server policy restrictions, need to re-upload
-    if (m.FileSize >= fileSizeLimit) {
-      log.warn('Message', 'forward() file size >= 25Mb,the message may fail to be forwarded due to server policy restrictions.')
-    }
-    newMsg.sendByLocal = false
-    newMsg.MMActualSender = config.puppetInstance().userId || ''
-    if (m.MMSendContent) {
-      newMsg.MMSendContent = m.MMSendContent.replace(/^@\w+:\s/, '')
-    }
-    if (m.MMDigest) {
-      newMsg.MMDigest = m.MMDigest.replace(/^@\w+:/, '')
-    }
-    if (m.MMActualContent) {
-      newMsg.MMActualContent = UtilLib.stripHtml(m.MMActualContent.replace(/^@\w+:<br\/>/, '')).replace(/^[\w\-]+:<br\/>/, '')
-    }
     m = Object.assign(m, newMsg)
-
-    return config.puppetInstance()
-      .forward(m, newMsg)
+    for (let i = 0; i < sendToList.length; i++) {
+      newMsg.ToUserName = sendToList[i].id
+      // all call success return true
+      ret = (i === 0 ? true : ret) && await config.puppetInstance().forward(m, newMsg)
+    }
+    return ret
   }
 }
 

src/puppet-web/bridge.ts

@@ -33,8 +33,9 @@ export interface MediaData {
   FileName:   string,
   FileSize:   number,
   FileMd5?:   string,
-  MMFileId:   string,
-  MMFileExt:  string,
+  FileType?:  number,
+  MMFileExt?: string,
+  Signature?: string,
 }
 
 export class Bridge {
@@ -398,6 +399,17 @@ export class Bridge {
     }
   }
 
+  public async getCheckUploadUrl(): Promise<string> {
+    log.verbose('PuppetWebBridge', 'getCheckUploadUrl()')
+
+    try {
+      return await this.proxyWechaty('getCheckUploadUrl')
+    } catch (e) {
+      log.silly('PuppetWebBridge', 'proxyWechaty(getCheckUploadUrl) exception: %s', e.message)
+      throw e
+    }
+  }
+
   public async getUploadMediaUrl(): Promise<string> {
     log.verbose('PuppetWebBridge', 'getUploadMediaUrl()')
 

src/puppet-web/puppet-web.ts

@@ -57,6 +57,7 @@ const enum UploadMediaType {
   AUDIO      = 3,
   ATTACHMENT = 4,
 }
+
 export interface PuppetWebSetting {
   head?:    HeadName,
   profile?: string,
@@ -409,44 +410,121 @@ export class PuppetWeb extends Puppet {
 
     // Sending video files is not allowed to exceed 20MB
     // https://github.com/Chatie/webwx-app-tracker/blob/7c59d35c6ea0cff38426a4c5c912a086c4c512b2/formatted/webwxApp.js#L1115
-    const videoMaxSize = 20 * 1024 * 1024
-    if (mediatype === 'video' && buffer.length > videoMaxSize)
-      throw new Error(`Sending video files is not allowed to exceed ${videoMaxSize / 1024 / 1024}MB`)
+    const maxVideoSize = 20 * 1024 * 1024
+    const largeFileSize = 25 * 1024 * 1024
+    const maxFileSize = 100 * 1024 * 1024
+    if (mediatype === 'video' && buffer.length > maxVideoSize)
+      throw new Error(`Sending video files is not allowed to exceed ${maxVideoSize / 1024 / 1024}MB`)
+    if (buffer.length > maxFileSize) {
+      throw new Error(`Sending files is not allowed to exceed ${maxFileSize / 1024 / 1024}MB`)
+    }
 
     const md5 = UtilLib.md5(buffer)
 
     const baseRequest = await this.getBaseRequest()
     const passTicket = await this.bridge.getPassticket()
     const uploadMediaUrl = await this.bridge.getUploadMediaUrl()
+    const checkUploadUrl = await this.bridge.getCheckUploadUrl()
     const cookie = await this.browser.readCookie()
     const first = cookie.find(c => c.name === 'webwx_data_ticket')
     const webwxDataTicket = first && first.value
     const size = buffer.length
+    const fromUserName = this.self().id
     const id = 'WU_FILE_' + this.fileId
     this.fileId++
 
     const hostname = await this.browser.hostname()
+    const headers = {
+      Referer: `https://${hostname}`,
+      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
+      Cookie: cookie.map(c => c.name + '=' + c.value).join('; '),
+    }
+
+    log.silly('PuppetWeb', 'uploadMedia() headers:%s', JSON.stringify(headers))
+
     const uploadMediaRequest = {
-      BaseRequest: baseRequest,
-      FileMd5: md5,
-      FromUserName: this.self().id,
-      ToUserName: toUserName,
-      UploadType: 2,
+      BaseRequest:   baseRequest,
+      FileMd5:       md5,
+      FromUserName:  fromUserName,
+      ToUserName:    toUserName,
+      UploadType:    2,
       ClientMediaId: +new Date,
-      MediaType: UploadMediaType.ATTACHMENT,
-      StartPos: 0,
-      DataLen: size,
-      TotalLen: size,
+      MediaType:     UploadMediaType.ATTACHMENT,
+      StartPos:      0,
+      DataLen:       size,
+      TotalLen:      size,
+      Signature:     '',
+      AESKey:        '',
+    }
+
+    const checkData = {
+      BaseRequest:  baseRequest,
+      FromUserName: fromUserName,
+      ToUserName:   toUserName,
+      FileName:     filename,
+      FileSize:     size,
+      FileMd5:      md5,
+      FileType:     7,              // If do not have this parameter, the api will fail
     }
 
-    const mediaData = <MediaData> {
+    const mediaData = <MediaData>{
       ToUserName: toUserName,
-      MediaId: '',
-      FileName: filename,
-      FileSize: size,
-      FileMd5: md5,
-      MMFileId: id,
-      MMFileExt: ext,
+      MediaId:    '',
+      FileName:   filename,
+      FileSize:   size,
+      FileMd5:    md5,
+      MMFileExt:  ext,
+    }
+
+    // If file size > 25M, must first call checkUpload to get Signature and AESKey, otherwise it will fail to upload
+    // https://github.com/Chatie/webwx-app-tracker/blob/7c59d35c6ea0cff38426a4c5c912a086c4c512b2/formatted/webwxApp.js#L1132 #1182
+    if (size > largeFileSize) {
+      let ret
+      try {
+        ret = <any> await new Promise((resolve, reject) => {
+          const r = {
+            url: `https://${hostname}${checkUploadUrl}`,
+            headers,
+            json: checkData,
+          }
+          request.post(r, function (err, res, body) {
+            try {
+              if (err) {
+                reject(err)
+              } else {
+                let obj = body
+                if (typeof body !== 'object') {
+                  obj = JSON.parse(body)
+                }
+                if (typeof obj !== 'object' || obj.BaseResponse.Ret !== 0) {
+                  const errMsg = obj.BaseResponse || 'api return err'
+                  log.silly('PuppetWeb', 'uploadMedia() checkUpload err:%s \nreq:%s\nret:%s', JSON.stringify(errMsg), JSON.stringify(r), body)
+                  reject(new Error('chackUpload err:' + JSON.stringify(errMsg)))
+                }
+                resolve({
+                  Signature: obj.Signature,
+                  AESKey: obj.AESKey,
+                })
+              }
+            } catch (e) {
+              reject(e)
+            }
+          })
+        })
+      } catch (e) {
+        log.error('PuppetWeb', 'uploadMedia() checkUpload exception: %s', e.message)
+        throw e
+      }
+      if (!ret.Signature) {
+        log.error('PuppetWeb', 'uploadMedia(): chackUpload failed to get Signature')
+        throw new Error('chackUpload failed to get Signature')
+      }
+      uploadMediaRequest.Signature = ret.Signature
+      uploadMediaRequest.AESKey    = ret.AESKey
+      mediaData.Signature          = ret.Signature
+    } else {
+      delete uploadMediaRequest.Signature
+      delete uploadMediaRequest.AESKey
     }
 
     log.verbose('PuppetWeb', 'uploadMedia() webwx_data_ticket: %s', webwxDataTicket)
@@ -461,7 +539,7 @@ export class PuppetWeb extends Puppet {
       mediatype,
       uploadmediarequest: JSON.stringify(uploadMediaRequest),
       webwx_data_ticket: webwxDataTicket,
-      pass_ticket: passTicket,
+      pass_ticket: passTicket || '',
       filename: {
         value: buffer,
         options: {
@@ -471,31 +549,37 @@ export class PuppetWeb extends Puppet {
         },
       },
     }
-
-    const mediaId = await new Promise<string>((resolve, reject) => {
-      request.post({
-        url: uploadMediaUrl + '?f=json',
-        headers: {
-          Referer: `https://${hostname}`,
-          'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
-        },
-        formData,
-      }, function (err, res, body) {
-        if (err) {
-          return reject(err)
-        }
+    let mediaId
+    try {
+      mediaId = <string>await new Promise((resolve, reject) => {
         try {
-          const obj = JSON.parse(body)
-          return resolve(obj.MediaId)
+          request.post({
+            url: uploadMediaUrl + '?f=json',
+            headers,
+            formData,
+          }, function (err, res, body) {
+            if (err) { reject(err) }
+            else {
+              let obj = body
+              if (typeof body !== 'object') {
+                obj = JSON.parse(body)
+              }
+              resolve(obj.MediaId || '')
+            }
+          })
         } catch (e) {
-          return reject(e)
+          reject(e)
         }
       })
-    })
+    } catch (e) {
+      log.error('PuppetWeb', 'uploadMedia() uploadMedia exception: %s', e.message)
+      throw new Error('uploadMedia err: ' + e.message)
+    }
     if (!mediaId) {
-      throw new Error('upload fail')
+      log.error('PuppetWeb', 'uploadMedia(): upload fail')
+      throw new Error('PuppetWeb.uploadMedia(): upload fail')
     }
-    return Object.assign(mediaData, { MediaId: mediaId })
+    return Object.assign(mediaData, { MediaId: mediaId as string })
   }
 
   public async sendMedia(message: MediaMessage): Promise<boolean> {
@@ -508,15 +592,39 @@ export class PuppetWeb extends Puppet {
       destinationId = room.id
     } else {
       if (!to) {
-        throw new Error('PuppetWeb.send(): message with neither room nor to?')
+        throw new Error('PuppetWeb.sendMedia(): message with neither room nor to?')
       }
       destinationId = to.id
     }
 
-    const mediaData = await this.uploadMedia(message, destinationId)
+    let mediaData: MediaData
+    const data = message.rawObj as MsgRawObj
+    if (!data.MediaId) {
+      try {
+        mediaData = await this.uploadMedia(message, destinationId)
+        message.rawObj = <MsgRawObj> Object.assign(data, mediaData)
+        log.silly('PuppetWeb', 'Upload completed, new rawObj:%s', JSON.stringify(message.rawObj))
+      } catch (e) {
+        log.error('PuppetWeb', 'sendMedia() exception: %s', e.message)
+        return false
+      }
+    } else {
+      // To support forward file
+      log.silly('PuppetWeb', 'skip upload file, rawObj:%s', JSON.stringify(data))
+      mediaData = {
+        ToUserName: destinationId,
+        MediaId: data.MediaId,
+        MsgType: data.MsgType,
+        FileName: data.FileName,
+        FileSize: data.FileSize,
+        MMFileExt: data.MMFileExt,
+      }
+      if (data.Signature) {
+        mediaData.Signature = data.Signature
+      }
+    }
     mediaData.MsgType = UtilLib.msgType(message.ext())
-
-    log.silly('PuppetWeb', 'send() destination: %s, mediaId: %s)',
+    log.silly('PuppetWeb', 'sendMedia() destination: %s, mediaId: %s)',
       destinationId,
       mediaData.MediaId,
     )
@@ -524,7 +632,7 @@ export class PuppetWeb extends Puppet {
     try {
       ret = await this.bridge.sendMedia(mediaData)
     } catch (e) {
-      log.error('PuppetWeb', 'send() exception: %s', e.message)
+      log.error('PuppetWeb', 'sendMedia() exception: %s', e.message)
       Raven.captureException(e)
       return false
     }
@@ -539,9 +647,6 @@ export class PuppetWeb extends Puppet {
     )
     let ret = false
     try {
-      // log.info('PuppetWeb', `forward() baseData: ${JSON.stringify(baseData)}\n`)
-      // log.info('PuppetWeb', `forward() patchData: ${JSON.stringify(patchData)}\n`)
-
       ret = await this.bridge.forward(baseData, patchData)
     } catch (e) {
       log.error('PuppetWeb', 'forward() exception: %s', e.message)

src/puppet-web/wechaty-bro.js

@@ -512,6 +512,11 @@
     return accountFactory.getPassticket()
   }
 
+  function getCheckUploadUrl() {
+    var confFactory = WechatyBro.glue.confFactory
+    return confFactory.API_checkupload
+  }
+
   function getUploadMediaUrl() {
     var confFactory = WechatyBro.glue.confFactory
     return confFactory.API_webwxuploadmedia
@@ -534,10 +539,18 @@
         FileName: data.FileName,
         FileSize: data.FileSize,
         MMFileExt: data.MMFileExt,
-        MMFileId: data.MMFileId
+      }
+
+      if (data.Signature) {
+        d.Signature = data.Signature
       }
 
       var m = chatFactory.createMessage(d)
+
+      m.MMFileStatus = confFactory.MM_SEND_FILE_STATUS_SUCCESS
+      m.MMStatus = confFactory.MSG_SEND_STATUS_SUCC
+      m.sendByLocal = false
+
       chatFactory.appendMessage(m)
       chatFactory.sendMessage(m)
     } catch (e) {
@@ -974,6 +987,7 @@
     , getUploadMediaUrl:   getUploadMediaUrl
     , sendMedia:           sendMedia
     , forward:             forward
+    , getCheckUploadUrl:   getCheckUploadUrl
 
     // for Wechaty Contact Class
     , contactFindAsync:   contactFindAsync