diff --git a/.travis/requirements.txt b/.travis/requirements.txt index c04a1d4952e3452987bb53d103f0ac4801d4a15f..1bcad987485b8d73bded95c746309f8833cc1aa4 100644 --- a/.travis/requirements.txt +++ b/.travis/requirements.txt @@ -1,3 +1,5 @@ -# add python requirements for unittests here, note install Cython -# and pycocotools directly is not supported in travis ci. +# add python requirements for unittests here, note install pycocotools +# directly is not supported in travis ci, it is installed by compiling +# from source files in unittest.sh tqdm +cython diff --git a/.travis/unittest.sh b/.travis/unittest.sh index 58fee7fe8afcf24f4f297c736745d689011a5fcb..e71833134fe62db0b3eddfe2806951e1880624d9 100755 --- a/.travis/unittest.sh +++ b/.travis/unittest.sh @@ -12,7 +12,7 @@ unittest(){ if [ $? != 0 ]; then exit 1 fi - find "./ppdet/modeling" -name 'tests' -type d -print0 | \ + find "./ppdet" -name 'tests' -type d -print0 | \ xargs -0 -I{} -n1 bash -c \ 'python -m unittest discover -v -s {}' } @@ -20,11 +20,26 @@ unittest(){ trap 'abort' 0 set -e -# install travis python dependencies +# install travis python dependencies exclude pycocotools if [ -f ".travis/requirements.txt" ]; then pip install -r .travis/requirements.txt fi +# install pycocotools +if [ `pip list | grep pycocotools | wc -l` -eq 0 ]; then + # install git if needed + if [ -n `which git` ]; then + apt-get update + apt-get install -y git + fi; + git clone https://github.com/cocodataset/cocoapi.git + cd cocoapi/PythonAPI + make install + python setup.py install --user + cd ../.. + rm -rf cocoapi +fi + export PYTHONPATH=`pwd`:$PYTHONPATH unittest . diff --git a/ppdet/data/parallel_map.py b/ppdet/data/parallel_map.py index 1d1db20ab32d42374fe33ecb2b784264b944fee4..9e55f7fdabad5d32022c93204ec2414e52a136d5 100644 --- a/ppdet/data/parallel_map.py +++ b/ppdet/data/parallel_map.py @@ -70,10 +70,11 @@ class ParallelMap(object): "multi-process reader on Windows.") self._use_process = False if self._use_process and type(memsize) is str: - assert memsize[-1].lower() == 'g', \ - "invalid param for memsize[%s], should be ended with 'G' or 'g'" % (memsize) - gb = memsize[:-1] - self._memsize = int(gb) * 1024**3 + assert memsize[-1].lower() in ['g', 'm'], \ + "invalid param for memsize[%s], should be " \ + "ended with 'G' or 'g' or 'M' or 'm'" % (memsize) + power = 3 if memsize[-1].lower() == 'g' else 2 + self._memsize = int(memsize[:-1]) * (1024**power) self._started = False self._source = source self._worker = worker diff --git a/ppdet/data/tests/test_dataset.py b/ppdet/data/tests/test_dataset.py index 6b35b36207c05daaa5077b9e23929db8a9000058..2e490551ed429c9d88ba3c7f34b8259540a34c8e 100644 --- a/ppdet/data/tests/test_dataset.py +++ b/ppdet/data/tests/test_dataset.py @@ -109,7 +109,7 @@ class TestDataset(unittest.TestCase): return 2 * sample test_worker = ParallelMap( - mem_sc, _worker, worker_num=2, use_process=True) + mem_sc, _worker, worker_num=2, use_process=True, memsize='2M') ct = 0 for i, d in enumerate(test_worker): @@ -131,7 +131,7 @@ class TestDataset(unittest.TestCase): return 2 * sample test_worker = ParallelMap( - mem_sc, _worker, worker_num=2, use_process=True) + mem_sc, _worker, worker_num=2, use_process=True, memsize='2M') ct = 0 for i, d in enumerate(test_worker):