From d8f30da7c18fde6b93c0f1f37994980d6bc9c772 Mon Sep 17 00:00:00 2001 From: yuyang18 Date: Wed, 31 Aug 2016 06:50:33 +0000 Subject: [PATCH] Add PyDataProvider2 DataConverter to swig api. * fix recommendation prediction also. ISSUE=4561941 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1449 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- demo/recommendation/.gitignore | 1 + demo/recommendation/prediction.py | 12 +++++------ demo/recommendation/trainer_config.py | 4 +--- paddle/api/Arguments.cpp | 26 ++++++++++++++++++++-- paddle/api/PaddleAPI.h | 3 +++ paddle/py_paddle/__init__.py | 10 +++++---- paddle/py_paddle/util.py | 31 +++++++++++++++++---------- 7 files changed, 60 insertions(+), 27 deletions(-) diff --git a/demo/recommendation/.gitignore b/demo/recommendation/.gitignore index aeae0f189d..fd27ef62a8 100644 --- a/demo/recommendation/.gitignore +++ b/demo/recommendation/.gitignore @@ -7,3 +7,4 @@ data/train.list data/test.list dataprovider_copy_1.py *.pyc +output diff --git a/demo/recommendation/prediction.py b/demo/recommendation/prediction.py index 1a6cfce58f..f8044a3195 100755 --- a/demo/recommendation/prediction.py +++ b/demo/recommendation/prediction.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from py_paddle import swig_paddle, DataProviderWrapperConverter +from py_paddle import swig_paddle, DataProviderConverter from common_utils import * from paddle.trainer.config_parser import parse_config @@ -31,11 +31,11 @@ if __name__ == '__main__': network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) - with open('meta.bin', 'rb') as f: + with open('./data/meta.bin', 'rb') as f: meta = pickle.load(f) headers = list(meta_to_header(meta, 'movie')) headers.extend(list(meta_to_header(meta, 'user'))) - cvt = DataProviderWrapperConverter(True, map(lambda x: x[1], headers)) + cvt = DataProviderConverter(headers) while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) @@ -45,7 +45,5 @@ if __name__ == '__main__': data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) - data = map(lambda (header, val): val if header[0] else [val], - zip(headers, data)) - print "Prediction Score is %.2f" % ((network.forwardTest(cvt([ - data]))[0]['value'][0][0] + 5) / 2) + print "Prediction Score is %.2f" % ((network.forwardTest( + cvt.convert([data]))[0]['value'][0][0] + 5) / 2) diff --git a/demo/recommendation/trainer_config.py b/demo/recommendation/trainer_config.py index 69b9aa7a77..624c22ec96 100755 --- a/demo/recommendation/trainer_config.py +++ b/demo/recommendation/trainer_config.py @@ -61,9 +61,7 @@ def construct_feature(name): slot_dim = each_meta['max'] embedding = embedding_layer(input=data_layer(slot_name, size=slot_dim), - size=256, - param_attr=ParamAttr( - sparse_update=True)) + size=256) fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'embedding': diff --git a/paddle/api/Arguments.cpp b/paddle/api/Arguments.cpp index e030f16b42..f9f191c711 100644 --- a/paddle/api/Arguments.cpp +++ b/paddle/api/Arguments.cpp @@ -102,8 +102,23 @@ static inline void doCopyFromSafely(std::shared_ptr& dest, IVector* Arguments::getSlotSequenceStartPositions(size_t idx) const throw(RangeError) { auto& a = m->getArg(idx); - return IVector::createByPaddleVectorPtr( - &a.sequenceStartPositions->getMutableVector(false)); + if (a.sequenceStartPositions) { + return IVector::createByPaddleVectorPtr( + &a.sequenceStartPositions->getMutableVector(false)); + } else { + return nullptr; + } +} + +IVector*Arguments::getSlotSubSequenceStartPositions(size_t idx) const + throw (RangeError){ + auto& a = m->getArg(idx); + if (a.subSequenceStartPositions) { + return IVector::createByPaddleVectorPtr( + &a.subSequenceStartPositions->getMutableVector(false)); + } else { + return nullptr; + } } void Arguments::setSlotSequenceStartPositions(size_t idx, @@ -113,6 +128,13 @@ void Arguments::setSlotSequenceStartPositions(size_t idx, a.sequenceStartPositions = std::make_shared(v); } +void Arguments::setSlotSubSequenceStartPositions( + size_t idx, IVector *vec) throw (RangeError) { + auto& a = m->getArg(idx); + auto& v = m->cast(vec->getSharedPtr()); + a.subSequenceStartPositions = std::make_shared(v); +} + IVector* Arguments::getSlotSequenceDim(size_t idx) const throw(RangeError) { auto& a = m->getArg(idx); return IVector::createByPaddleVectorPtr(&a.cpuSequenceDims); diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index 7f0c668716..c92ed205d5 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -374,6 +374,7 @@ public: IVector* getSlotIds(size_t idx) const throw(RangeError); Matrix* getSlotIn(size_t idx) const throw(RangeError); IVector* getSlotSequenceStartPositions(size_t idx) const throw(RangeError); + IVector* getSlotSubSequenceStartPositions(size_t idx) const throw(RangeError); IVector* getSlotSequenceDim(size_t idx) const throw(RangeError); // End Of get functions of Arguments @@ -390,6 +391,8 @@ public: void setSlotIds(size_t idx, IVector* vec) throw(RangeError); void setSlotSequenceStartPositions(size_t idx, IVector* vec) throw(RangeError); + void setSlotSubSequenceStartPositions(size_t idx, + IVector* vec) throw (RangeError); void setSlotSequenceDim(size_t idx, IVector* vec) throw(RangeError); private: diff --git a/paddle/py_paddle/__init__.py b/paddle/py_paddle/__init__.py index 970b4e64ae..f372068942 100644 --- a/paddle/py_paddle/__init__.py +++ b/paddle/py_paddle/__init__.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import swig_paddle -import util from util import DataProviderWrapperConverter +from dataprovider_converter import DataProviderConverter -__all__ = ['paddle', 'DataProviderWrapperConverter', 'loadParameterFile'] - +__all__ = ['paddle', + 'DataProviderConverter', + 'DataProviderWrapperConverter', # for deprecated usage. + 'loadParameterFile'] util.monkeypatches() + diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index eb148c9c66..e6cf2710ef 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -209,6 +209,7 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.getLayerOutputs = getLayerOutputs + def loadGradientMachine(config_filename, model_dir=None): """ Load a gradient machine from config file name/path. @@ -229,6 +230,7 @@ def loadGradientMachine(config_filename, model_dir=None): network.loadParameters(model_dir) return network + def loadParameterFile(fn): """ Load Paddle Parameter file to numpy.ndarray @@ -257,6 +259,7 @@ def loadParameterFile(fn): else: raise swig_paddle.UnsupportError() + class DataProviderWrapperConverter(object): """ A class convert DataFormat from PyDataProvider Wrapper to @@ -278,8 +281,8 @@ class DataProviderWrapperConverter(object): def __call__(self, slot_idx, arg): mat = swig_paddle.Matrix.createDense(self.buf, - len(self.buf) / self.__dim__, - self.__dim__) + len(self.buf) / self.__dim__, + self.__dim__) arg.setSlotValue(slot_idx, mat) class IdValueConverter(object): @@ -312,8 +315,9 @@ class DataProviderWrapperConverter(object): self.cols += other def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, self.dim, - len(self.cols), True) + mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, + self.dim, + len(self.cols), True) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols) self.putIntoArg(slot_idx, arg, mat) @@ -337,8 +341,9 @@ class DataProviderWrapperConverter(object): self.values += map(lambda x: x[1], other) def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, self.dim, - len(self.cols), False) + mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, + self.dim, + len(self.cols), False) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols, self.values) self.putIntoArg(slot_idx, arg, mat) @@ -373,7 +378,7 @@ class DataProviderWrapperConverter(object): """ if argument is None: argument = swig_paddle.Arguments.createArguments(0) - assert isinstance(argument,swig_paddle.Arguments) + assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.__header__)) values = map(lambda x: @@ -394,10 +399,12 @@ class DataProviderWrapperConverter(object): seq_dim[slot_idx].append(len(sequence)) for slot_idx in xrange(len(self.__header__)): - argument.setSlotSequenceDim(slot_idx, swig_paddle.IVector.create( - seq_dim[slot_idx])) + argument.setSlotSequenceDim(slot_idx, + swig_paddle.IVector.create( + seq_dim[slot_idx])) argument.setSlotSequenceStartPositions( - slot_idx, swig_paddle.IVector.create(seq_start_pos[slot_idx])) + slot_idx, + swig_paddle.IVector.create(seq_start_pos[slot_idx])) else: for each_sample in wrapper_data: for raw_data, value in zip(each_sample, values): @@ -415,6 +422,7 @@ class DataProviderWrapperConverter(object): return self.convert(wrapper_data, argument) + def __monkey_patch_protobuf_objects__(): def ParameterConfig_toProto(self): """ @@ -451,7 +459,8 @@ def __monkey_patch_protobuf_objects__(): :return: paddle.OptimizationConfig """ - assert isinstance(protoObj, paddle.proto.TrainerConfig_pb2.OptimizationConfig) + assert isinstance(protoObj, + paddle.proto.TrainerConfig_pb2.OptimizationConfig) return swig_paddle.OptimizationConfig.createFromProtoString( protoObj.SerializeToString()) -- GitLab