From f6d414d2ace0dfe4ef2575b5b49b28429dce339b Mon Sep 17 00:00:00 2001
From: Kaipeng Deng <dengkaipeng@baidu.com>
Date: Mon, 10 Feb 2020 09:41:18 +0800
Subject: [PATCH] enable data unittest (#216)

* enable data unittest
---
 .travis/requirements.txt         |  6 ++++--
 .travis/unittest.sh              | 19 +++++++++++++++++--
 ppdet/data/parallel_map.py       |  9 +++++----
 ppdet/data/tests/test_dataset.py |  4 ++--
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/.travis/requirements.txt b/.travis/requirements.txt
index c04a1d495..1bcad9874 100644
--- a/.travis/requirements.txt
+++ b/.travis/requirements.txt
@@ -1,3 +1,5 @@
-# add python requirements for unittests here, note install Cython
-# and pycocotools directly is not supported in travis ci.
+# add python requirements for unittests here, note install pycocotools
+# directly is not supported in travis ci, it is installed by compiling
+# from source files in unittest.sh
 tqdm
+cython
diff --git a/.travis/unittest.sh b/.travis/unittest.sh
index 58fee7fe8..e71833134 100755
--- a/.travis/unittest.sh
+++ b/.travis/unittest.sh
@@ -12,7 +12,7 @@ unittest(){
     if [ $? != 0 ]; then
         exit 1
     fi
-    find "./ppdet/modeling" -name 'tests' -type d -print0 | \
+    find "./ppdet" -name 'tests' -type d -print0 | \
         xargs -0 -I{} -n1 bash -c \
         'python -m unittest discover -v -s {}'
 }
@@ -20,11 +20,26 @@ unittest(){
 trap 'abort' 0
 set -e
 
-# install travis python dependencies
+# install travis python dependencies exclude pycocotools
 if [ -f ".travis/requirements.txt" ]; then
     pip install -r .travis/requirements.txt
 fi
 
+# install pycocotools
+if [ `pip list | grep pycocotools | wc -l` -eq 0 ]; then
+  # install git if needed
+  if [ -n  `which git` ]; then
+    apt-get update
+    apt-get install -y git
+  fi;
+  git clone https://github.com/cocodataset/cocoapi.git
+  cd cocoapi/PythonAPI
+  make install
+  python setup.py install --user
+  cd ../..
+  rm -rf cocoapi
+fi
+
 export PYTHONPATH=`pwd`:$PYTHONPATH
 
 unittest .
diff --git a/ppdet/data/parallel_map.py b/ppdet/data/parallel_map.py
index 1d1db20ab..9e55f7fda 100644
--- a/ppdet/data/parallel_map.py
+++ b/ppdet/data/parallel_map.py
@@ -70,10 +70,11 @@ class ParallelMap(object):
                         "multi-process reader on Windows.")
             self._use_process = False
         if self._use_process and type(memsize) is str:
-            assert memsize[-1].lower() == 'g', \
-                "invalid param for memsize[%s], should be ended with 'G' or 'g'" % (memsize)
-            gb = memsize[:-1]
-            self._memsize = int(gb) * 1024**3
+            assert memsize[-1].lower() in ['g', 'm'], \
+                "invalid param for memsize[%s], should be " \
+                "ended with 'G' or 'g' or 'M' or 'm'" % (memsize)
+            power = 3 if memsize[-1].lower() == 'g' else 2
+            self._memsize = int(memsize[:-1]) * (1024**power)
         self._started = False
         self._source = source
         self._worker = worker
diff --git a/ppdet/data/tests/test_dataset.py b/ppdet/data/tests/test_dataset.py
index 6b35b3620..2e490551e 100644
--- a/ppdet/data/tests/test_dataset.py
+++ b/ppdet/data/tests/test_dataset.py
@@ -109,7 +109,7 @@ class TestDataset(unittest.TestCase):
             return 2 * sample
 
         test_worker = ParallelMap(
-            mem_sc, _worker, worker_num=2, use_process=True)
+            mem_sc, _worker, worker_num=2, use_process=True, memsize='2M')
 
         ct = 0
         for i, d in enumerate(test_worker):
@@ -131,7 +131,7 @@ class TestDataset(unittest.TestCase):
             return 2 * sample
 
         test_worker = ParallelMap(
-            mem_sc, _worker, worker_num=2, use_process=True)
+            mem_sc, _worker, worker_num=2, use_process=True, memsize='2M')
 
         ct = 0
         for i, d in enumerate(test_worker):
-- 
GitLab