{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Change working directory to the project root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/yang/PaddleDetection\n"
     ]
    }
   ],
   "source": [
    "%cd .."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Now let's take a look at the input image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 637180 characters]\n",
      "text/plain": [
       "<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x480 at 0x7F2B7E54C550>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "\n",
    "image_path = 'demo/000000570688.jpg'\n",
    "img = Image.open(image_path)\n",
    "img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "For inference, preprocess only involves decoding, normalization and transposing to CHW.\n",
    "\n",
    "**NOTE:** in most cases, one should use the configuration based [data feed](../docs/DATA.md) API which greatly simplifies the data pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from ppdet.data.transform.operators import DecodeImage, NormalizeImage, Permute\n",
    "\n",
    "sample = {'im_file': image_path}\n",
    "decode = DecodeImage(to_rgb=True)\n",
    "normalize = NormalizeImage(\n",
    "    mean=[0.485, 0.456, 0.406],\n",
    "    std=[0.229, 0.224, 0.225],\n",
    "    is_scale=True,\n",
    "    is_channel_first=False)\n",
    "permute = Permute(to_bgr=False, channel_first=True)\n",
    "\n",
    "sample = permute(normalize(decode(sample)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Some extra effort is needed to massage data into the desired format. \n",
    "\n",
    "**NOTE:** Again, if the data feed API is used, these are handled automatically."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "h = sample['h']\n",
    "w = sample['w']\n",
    "im_info = np.array((h, w, 1), dtype=np.float32)\n",
    "\n",
    "sample['im_info'] = im_info\n",
    "sample['im_shape'] = im_info\n",
    "\n",
    "# we don't need these\n",
    "for key in ['im_file', 'h', 'w']:\n",
    "    del sample[key]\n",
    "\n",
    "# batch of a single sample\n",
    "sample = {k: v[np.newaxis, ...] for k, v in sample.items()}\n",
    "\n",
    "feed_var_def = [\n",
    "    {'name': 'image',  'shape': (h, w, 3)},\n",
    "    {'name': 'im_info',  'shape': [3]},\n",
    "    {'name': 'im_shape', 'shape': [3]},\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Next, build the [Mask R-CNN](https://arxiv.org/abs/1703.06870) model and associated fluid programs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from paddle import fluid\n",
    "from ppdet.modeling import (MaskRCNN, ResNet, ResNetC5, RPNHead, RoIAlign,\n",
    "                            BBoxHead, MaskHead, BBoxAssigner, MaskAssigner)\n",
    "\n",
    "roi_size = 14\n",
    "\n",
    "model = MaskRCNN(\n",
    "    ResNet(feature_maps=4),\n",
    "    RPNHead(),\n",
    "    BBoxHead(ResNetC5()),\n",
    "    BBoxAssigner(),\n",
    "    RoIAlign(resolution=roi_size),\n",
    "    MaskAssigner(),\n",
    "    MaskHead())\n",
    "\n",
    "startup_prog = fluid.Program()\n",
    "infer_prog = fluid.Program()\n",
    "with fluid.program_guard(infer_prog, startup_prog):\n",
    "    with fluid.unique_name.guard():\n",
    "        feed_vars = {\n",
    "            var['name']: fluid.layers.data(\n",
    "                name=var['name'],\n",
    "                shape=var['shape'],\n",
    "                dtype='float32',\n",
    "                lod_level=0) for var in feed_var_def\n",
    "        }\n",
    "        test_fetches = model.test(feed_vars)\n",
    "infer_prog = infer_prog.clone(for_test=True)\n",
    "\n",
    "# use GPU if available\n",
    "if fluid.core.get_cuda_device_count() > 0:\n",
    "    place = fluid.CUDAPlace(0)\n",
    "else:\n",
    "    place = fluid.CPUPlace()\n",
    "\n",
    "exe = fluid.Executor(place)\n",
    "_ = exe.run(startup_prog)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Load the checkpoint weights, just wait a couple of minutes for it to be downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 140690/140690 [00:12<00:00, 10843.70KB/s]\n"
     ]
    }
   ],
   "source": [
    "from ppdet.utils import checkpoint\n",
    "\n",
    "ckpt_url = 'https://paddlemodels.bj.bcebos.com/object_detection/mask_rcnn_r50_1x.tar'\n",
    "checkpoint.load_checkpoint(exe, infer_prog, ckpt_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Run the program and fetch the result. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "output = exe.run(infer_prog, feed=sample,\n",
    "                 fetch_list=[t.name for t in test_fetches.values()],\n",
    "                 return_numpy=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Again, we need to massage the result a bit for visualization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "res = {\n",
    "    k: (np.array(v), v.recursive_sequence_lengths())\n",
    "    for k, v in zip(test_fetches.keys(), output)\n",
    "}\n",
    "# fake image id\n",
    "res['im_id'] = [[[0] for _ in range(res['bbox'][1][0][0])]]\n",
    "res['im_shape'] = [[im_info]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "Now overlay the bboxes and masks onto the image...\n",
    "\n",
    "And voila, we've successully built and run the Mask R-CNN inference pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "autoscroll": false,
    "ein.hycell": false,
    "ein.tags": "worksheet-0",
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 541696 characters]\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=640x480 at 0x7F2B2046D190>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ppdet.utils.coco_eval import bbox2out, mask2out, coco17_category_info\n",
    "from ppdet.utils.visualizer import visualize_results\n",
    "\n",
    "cls2cat, cat2name = coco17_category_info()\n",
    "bboxes = bbox2out([res], cls2cat)\n",
    "masks = mask2out([res], cls2cat, roi_size)\n",
    "\n",
    "visualize_results(img, 0, cat2name, 0.5, bboxes, masks)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  },
  "name": "mask_rcnn_demo.ipynb"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
