// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/reader.h" namespace paddle { namespace operators { namespace reader { static constexpr char kFileFormatSeparator[] = ":"; using FileReaderCreator = std::function&)>; std::unordered_map& FileReaderRegistry(); template int RegisterFileReader(const std::string& filetype) { FileReaderRegistry()[filetype] = []( const std::string& fn, const std::vector& dims) { return new Reader(fn, dims); }; return 0; } std::unique_ptr CreateReaderByFileName( const std::string& file_name, const std::vector& dims) { size_t separator_pos = file_name.find(kFileFormatSeparator); PADDLE_ENFORCE_NE(separator_pos, std::string::npos, "File name illegal! A legal file name should be like: " "[file_format]:[file_name] (e.g., 'recordio:data_file')."); std::string filetype = file_name.substr(0, separator_pos); std::string f_name = file_name.substr(separator_pos + 1); auto itor = FileReaderRegistry().find(filetype); PADDLE_ENFORCE(itor != FileReaderRegistry().end(), "No file reader registered for '%s' format.", filetype); framework::ReaderBase* reader = (itor->second)(f_name, dims); return std::unique_ptr(reader); } extern std::vector RestoreShapes( const std::vector& shape_concat, const std::vector& ranks); class FileReaderMakerBase : public framework::OpProtoAndCheckerMaker { public: FileReaderMakerBase(OpProto* op_proto, OpAttrChecker* op_checker); }; class FileReaderInferShape : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext* ctx) const override; }; class FileReaderInferVarType : public framework::VarTypeInference { public: void operator()(const framework::OpDesc& op_desc, framework::BlockDesc* block) const override; }; // general infershape for decorated reader class DecoratedReaderInferShape : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext* ctx) const override; }; // general var type inference for decorated reader class DecoratedReaderInferVarType : public framework::VarTypeInference { public: void operator()(const framework::OpDesc& op_desc, framework::BlockDesc* block) const override; }; class DecoratedReaderMakerBase : public framework::OpProtoAndCheckerMaker { public: DecoratedReaderMakerBase(OpProto* op_proto, OpAttrChecker* op_checker); }; } // namespace reader } // namespace operators } // namespace paddle #define REGISTER_FILE_READER_OPERATOR(op_name, ...) \ REGISTER_OPERATOR(op_name, __VA_ARGS__, \ paddle::operators::reader::FileReaderInferShape, \ paddle::framework::EmptyGradOpMaker, \ paddle::operators::reader::FileReaderInferVarType) #define REGISTER_DECORATED_READER_OPERATOR(op_name, ...) \ REGISTER_OPERATOR(op_name, __VA_ARGS__, \ paddle::operators::reader::DecoratedReaderInferShape, \ paddle::framework::EmptyGradOpMaker, \ paddle::operators::reader::DecoratedReaderInferVarType) #define REGISTER_FILE_READER(_filetype, _reader) \ STATIC_ASSERT_GLOBAL_NAMESPACE( \ _reg_file_reader_##_filetype, \ "Must use REGISTER_FILE_READER in global namespace"); \ int TouchFileReader##_filetype() { return 0; } \ int _reg_file_reader_entry_##filetype = \ paddle::operators::reader::RegisterFileReader<_reader>(#_filetype) #define USE_FILE_READER(filetype) \ extern int TouchFileReader##filetype(); \ static int _use_##filetype = TouchFileReader##filetype()