enable data unittest (#216)

* enable data unittest

.travis/requirements.txt

-# add python requirements for unittests here, note install Cython
-# and pycocotools directly is not supported in travis ci.
+# add python requirements for unittests here, note install pycocotools
+# directly is not supported in travis ci, it is installed by compiling
+# from source files in unittest.sh
 tqdm
+cython

.travis/unittest.sh

@@ -12,7 +12,7 @@ unittest(){
     if [ $? != 0 ]; then
         exit 1
     fi
-    find "./ppdet/modeling" -name 'tests' -type d -print0 | \
+    find "./ppdet" -name 'tests' -type d -print0 | \
         xargs -0 -I{} -n1 bash -c \
         'python -m unittest discover -v -s {}'
 }
@@ -20,11 +20,26 @@ unittest(){
 trap 'abort' 0
 set -e
 
-# install travis python dependencies
+# install travis python dependencies exclude pycocotools
 if [ -f ".travis/requirements.txt" ]; then
     pip install -r .travis/requirements.txt
 fi
 
+# install pycocotools
+if [ `pip list | grep pycocotools | wc -l` -eq 0 ]; then
+  # install git if needed
+  if [ -n  `which git` ]; then
+    apt-get update
+    apt-get install -y git
+  fi;
+  git clone https://github.com/cocodataset/cocoapi.git
+  cd cocoapi/PythonAPI
+  make install
+  python setup.py install --user
+  cd ../..
+  rm -rf cocoapi
+fi
+
 export PYTHONPATH=`pwd`:$PYTHONPATH
 
 unittest .

ppdet/data/parallel_map.py

@@ -70,10 +70,11 @@ class ParallelMap(object):
                         "multi-process reader on Windows.")
             self._use_process = False
         if self._use_process and type(memsize) is str:
-            assert memsize[-1].lower() == 'g', \
-                "invalid param for memsize[%s], should be ended with 'G' or 'g'" % (memsize)
-            gb = memsize[:-1]
-            self._memsize = int(gb) * 1024**3
+            assert memsize[-1].lower() in ['g', 'm'], \
+                "invalid param for memsize[%s], should be " \
+                "ended with 'G' or 'g' or 'M' or 'm'" % (memsize)
+            power = 3 if memsize[-1].lower() == 'g' else 2
+            self._memsize = int(memsize[:-1]) * (1024**power)
         self._started = False
         self._source = source
         self._worker = worker

ppdet/data/tests/test_dataset.py

@@ -109,7 +109,7 @@ class TestDataset(unittest.TestCase):
             return 2 * sample
 
         test_worker = ParallelMap(
-            mem_sc, _worker, worker_num=2, use_process=True)
+            mem_sc, _worker, worker_num=2, use_process=True, memsize='2M')
 
         ct = 0
         for i, d in enumerate(test_worker):
@@ -131,7 +131,7 @@ class TestDataset(unittest.TestCase):
             return 2 * sample
 
         test_worker = ParallelMap(
-            mem_sc, _worker, worker_num=2, use_process=True)
+            mem_sc, _worker, worker_num=2, use_process=True, memsize='2M')
 
         ct = 0
         for i, d in enumerate(test_worker):