diff --git a/paddle/cuda/src/hl_warpctc_wrap.cc b/paddle/cuda/src/hl_warpctc_wrap.cc index 55b940ca67acce2c7ee7c1ee286ab96240652274..f57efb2b46797c303d99a5468ad96163a3e74972 100644 --- a/paddle/cuda/src/hl_warpctc_wrap.cc +++ b/paddle/cuda/src/hl_warpctc_wrap.cc @@ -54,22 +54,26 @@ DYNAMIC_LOAD_WARPCTC_WRAP(get_workspace_size) #define WARPCTC_GET_VERSION dynload::get_warpctc_version #define WARPCTC_GET_STATUS_STRING dynload::ctcGetStatusString +static int g_warpctcVersion = -1; #ifndef PADDLE_TYPE_DOUBLE #define WARPCTC_COMPUTE_LOSS dynload::compute_ctc_loss #define WARPCTC_GET_WORKSPACE_SIZE dynload::get_workspace_size #else -#define WARPCTC_LOG_FATAL \ - LOG(FATAL) << "warp-ctc [version " << g_warpctcVersion \ - << "] Error: not support double precision." -#define WARPCTC_COMPUTE_LOSS(...) WARPCTC_LOG_FATAL(__VA_ARGS__) -#define WARPCTC_GET_WORKSPACE_SIZE(...) WARPCTC_LOG_FATAL(__VA_ARGS__) +hl_warpctc_status_t fatal(...) { + LOG(FATAL) << "warp-ctc [version " << g_warpctcVersion + << "] Error: not support double precision."; + // both of get_warpctc_version() and get_workspace_size() return an ctcStatus + // type value + return CTC_STATUS_EXECUTION_FAILED; +} +#define WARPCTC_COMPUTE_LOSS fatal +#define WARPCTC_GET_WORKSPACE_SIZE fatal #endif /** * Check build-in warp-ctc function using glog and it also * support << operator for more details error info. */ -static int g_warpctcVersion = -1; #define CHECK_WARPCTC(warpctcStat) \ CHECK_EQ(CTC_STATUS_SUCCESS, warpctcStat) \ << "warp-ctc [version " << g_warpctcVersion \ diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index 5fc799e61dab954b9993321c4e816f2f5abce448..9f4234358f469b9741dfef96aab3853d5972b5f4 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['buffered', 'compose', 'chain', 'shuffle', 'ComposeNotAligned'] +__all__ = [ + 'map_readers', 'buffered', 'compose', 'chain', 'shuffle', + 'ComposeNotAligned' +] from Queue import Queue from threading import Thread @@ -20,20 +23,38 @@ import itertools import random +def map_readers(func, *readers): + """ + Creates a data reader that outputs return value of function using + output of each data readers as arguments. + + :param func: function to use. + :param *readers: readers whose outputs will be used as arguments of func. + :returns: the created data reader. + """ + + def reader(): + rs = [] + for r in readers: + rs.append(r()) + for e in itertools.imap(func, *rs): + yield e + + return reader + + def shuffle(reader, buf_size): - """Creates a data reader whose data output is suffled. + """ + Creates a data reader whose data output is suffled. Output from the iterator that created by original reader will be buffered into shuffle buffer, and then shuffled. The size of shuffle buffer is determined by argument buf_size. - Args: - reader: the original reader whose output will be - shuffled. - buf_size: shuffle buffer size. + :param reader: the original reader whose output will be shuffled. + :param buf_size: shuffle buffer size. - Returns: - the new reader whose output is shuffled. + :returns:the new reader whose output is shuffled. """ def data_reader(): @@ -55,8 +76,9 @@ def shuffle(reader, buf_size): def chain(*readers): - """Creates a data reader whose output is the outputs of input data - readers chained together. + """ + Creates a data reader whose output is the outputs of input data + readers chained together. If input readers output following data entries: [0, 0, 0] @@ -65,11 +87,8 @@ def chain(*readers): The chained reader will output: [0, 0, 0, 1, 1, 1, 2, 2, 2] - Args: - readers: input readers. - - Returns: - the new data reader. + :param readers: input readers. + :returns: the new data reader. """ def reader(): @@ -88,25 +107,23 @@ class ComposeNotAligned(ValueError): def compose(*readers, **kwargs): - """Creates a data reader whose output is the combination of input readers. + """ + Creates a data reader whose output is the combination of input readers. If input readers output following data entries: (1, 2) 3 (4, 5) The composed reader will output: (1, 2, 3, 4, 5) - Args: - *readers: readers that will be composed together. - check_alignment: If True, will check if input readers are aligned - correctly. If False, will not check alignment and trailing outputs - will be discarded. Defaults to True. + :*readers: readers that will be composed together. + :check_alignment: if True, will check if input readers are aligned + correctly. If False, will not check alignment and trailing outputs + will be discarded. Defaults to True. - Returns: - the new data reader. + :returns: the new data reader. - Raises: - ComposeNotAligned: outputs of readers are not aligned. - Will not raise when check_alignment is set to False. + :raises ComposeNotAligned: outputs of readers are not aligned. + Will not raise when check_alignment is set to False. """ check_alignment = kwargs.pop('check_alignment', True) @@ -136,18 +153,17 @@ def compose(*readers, **kwargs): def buffered(reader, size): - """Creates a buffered data reader. + """ + Creates a buffered data reader. The buffered data reader will read and save data entries into a buffer. Reading from the buffered data reader will proceed as long as the buffer is not empty. - Args: - reader: the data reader to read from. - size: max buffer size. + :param reader: the data reader to read from. + :param size: max buffer size. - Returns: - The buffered data reader. + :returns: the buffered data reader. """ class EndSignal(): diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py index 46eec44158cee5f8c70a0e6197e856e485a7d40c..0396a61786539bf57be4cab9ebbd108ade9e7c83 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/reader/tests/decorator_test.py @@ -26,6 +26,22 @@ def reader_creator_10(dur): return reader +class TestMap(unittest.TestCase): + def test_map(self): + d = {"h": 0, "i": 1} + + def tokenize(x): + return d[x] + + def read(): + yield "h" + yield "i" + + r = paddle.reader.map_readers(tokenize, read) + for i, e in enumerate(r()): + self.assertEqual(e, i) + + class TestBuffered(unittest.TestCase): def test_read(self): for size in range(20):