diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt
index fbeacb66cefab84c342ee11bcb7bf0fd55da0645..c14a2b7786f9f7c06d59479d3bbce9c5d542e495 100644
--- a/paddle/fluid/operators/CMakeLists.txt
+++ b/paddle/fluid/operators/CMakeLists.txt
@@ -270,6 +270,11 @@ foreach(src ${READER_LIBRARY})
     set(OP_LIBRARY ${src} ${OP_LIBRARY})
 endforeach()
 
+add_subdirectory(detection)
+foreach(src ${DETECTION_LIBRARY})
+    set(OP_LIBRARY ${src} ${OP_LIBRARY})
+endforeach()
+
 set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library")
 
 cc_test(gather_test SRCS gather_test.cc DEPS tensor)
diff --git a/paddle/fluid/operators/detection/CMakeLists.txt b/paddle/fluid/operators/detection/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a5bb58c2f4047a3bf2f8592b605772b4fa166c57
--- /dev/null
+++ b/paddle/fluid/operators/detection/CMakeLists.txt
@@ -0,0 +1,29 @@
+set(LOCAL_DETECTION_LIBS)
+
+function(detection_library TARGET_NAME)
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    set(options "")
+    set(common_deps op_registry)
+    set(pybind_flag 0)
+    cmake_parse_arguments(detection_library "${options}" "${oneValueArgs}"
+            "${multiValueArgs}" ${ARGN})
+    op_library(${TARGET_NAME} SRCS ${detection_library_SRCS} DEPS ${common_deps} ${detection_library_DEPS})
+    set(LOCAL_DETECTION_LIBS
+            ${TARGET_NAME}
+            ${LOCAL_DETECTION_LIBS}
+        PARENT_SCOPE)
+endfunction()
+
+detection_library(bipartite_match_op SRCS bipartite_match_op.cc)
+detection_library(box_coder_op SRCS box_coder_op.cc box_coder_op.cu)
+detection_library(iou_similarity_op SRCS iou_similarity_op.cc
+iou_similarity_op.cu)
+detection_library(mine_hard_examples_op SRCS mine_hard_examples_op.cc)
+detection_library(multiclass_nms_op SRCS multiclass_nms_op.cc)
+detection_library(prior_box_op SRCS prior_box_op.cc prior_box_op.cu)
+detection_library(target_assign_op SRCS target_assign_op.cc
+target_assign_op.cu)
+
+# Export local libraries to parent
+set(DETECTION_LIBRARY ${LOCAL_DETECTION_LIBS} PARENT_SCOPE)
diff --git a/paddle/fluid/operators/bipartite_match_op.cc b/paddle/fluid/operators/detection/bipartite_match_op.cc
similarity index 100%
rename from paddle/fluid/operators/bipartite_match_op.cc
rename to paddle/fluid/operators/detection/bipartite_match_op.cc
diff --git a/paddle/fluid/operators/box_coder_op.cc b/paddle/fluid/operators/detection/box_coder_op.cc
similarity index 99%
rename from paddle/fluid/operators/box_coder_op.cc
rename to paddle/fluid/operators/detection/box_coder_op.cc
index ce9bf10dc3c4718c5adfc0d5034d06c22f9d54e1..74848005d0bea6e5c818ff999727aa2b8ad51d84 100644
--- a/paddle/fluid/operators/box_coder_op.cc
+++ b/paddle/fluid/operators/detection/box_coder_op.cc
@@ -9,7 +9,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/box_coder_op.h"
+#include "paddle/fluid/operators/detection/box_coder_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/box_coder_op.cu b/paddle/fluid/operators/detection/box_coder_op.cu
similarity index 99%
rename from paddle/fluid/operators/box_coder_op.cu
rename to paddle/fluid/operators/detection/box_coder_op.cu
index 708c7a5fa96c2f9ce6a2d913ca5f30126bb6192f..8cef8e03439df4ca5b0fa94176a21a36f9eb9f70 100644
--- a/paddle/fluid/operators/box_coder_op.cu
+++ b/paddle/fluid/operators/detection/box_coder_op.cu
@@ -9,7 +9,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/box_coder_op.h"
+#include "paddle/fluid/operators/detection/box_coder_op.h"
 #include "paddle/fluid/platform/cuda_primitives.h"
 
 namespace paddle {
diff --git a/paddle/fluid/operators/box_coder_op.h b/paddle/fluid/operators/detection/box_coder_op.h
similarity index 100%
rename from paddle/fluid/operators/box_coder_op.h
rename to paddle/fluid/operators/detection/box_coder_op.h
diff --git a/paddle/fluid/operators/iou_similarity_op.cc b/paddle/fluid/operators/detection/iou_similarity_op.cc
similarity index 98%
rename from paddle/fluid/operators/iou_similarity_op.cc
rename to paddle/fluid/operators/detection/iou_similarity_op.cc
index 007e0af7a5a9fd8c562218e641e1867388b081f1..8e58605fcea04f9ffa97ce8cca53c073e7068aaf 100644
--- a/paddle/fluid/operators/iou_similarity_op.cc
+++ b/paddle/fluid/operators/detection/iou_similarity_op.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/iou_similarity_op.h"
+#include "paddle/fluid/operators/detection/iou_similarity_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/iou_similarity_op.cu b/paddle/fluid/operators/detection/iou_similarity_op.cu
similarity index 92%
rename from paddle/fluid/operators/iou_similarity_op.cu
rename to paddle/fluid/operators/detection/iou_similarity_op.cu
index f40a388d62e66a110656ebb71094d46b5ac147eb..8342b4138c87e6ea1803146bac6d6954a569ef5f 100644
--- a/paddle/fluid/operators/iou_similarity_op.cu
+++ b/paddle/fluid/operators/detection/iou_similarity_op.cu
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/iou_similarity_op.h"
+#include "paddle/fluid/operators/detection/iou_similarity_op.h"
 
 namespace ops = paddle::operators;
 REGISTER_OP_CUDA_KERNEL(
diff --git a/paddle/fluid/operators/iou_similarity_op.h b/paddle/fluid/operators/detection/iou_similarity_op.h
similarity index 100%
rename from paddle/fluid/operators/iou_similarity_op.h
rename to paddle/fluid/operators/detection/iou_similarity_op.h
diff --git a/paddle/fluid/operators/mine_hard_examples_op.cc b/paddle/fluid/operators/detection/mine_hard_examples_op.cc
similarity index 100%
rename from paddle/fluid/operators/mine_hard_examples_op.cc
rename to paddle/fluid/operators/detection/mine_hard_examples_op.cc
diff --git a/paddle/fluid/operators/multiclass_nms_op.cc b/paddle/fluid/operators/detection/multiclass_nms_op.cc
similarity index 100%
rename from paddle/fluid/operators/multiclass_nms_op.cc
rename to paddle/fluid/operators/detection/multiclass_nms_op.cc
diff --git a/paddle/fluid/operators/prior_box_op.cc b/paddle/fluid/operators/detection/prior_box_op.cc
similarity index 99%
rename from paddle/fluid/operators/prior_box_op.cc
rename to paddle/fluid/operators/detection/prior_box_op.cc
index a0b069da0dda59c769723211533df8c33511fe3f..4e35c38e4e03d4d0f00601812fdc4803519b89ae 100644
--- a/paddle/fluid/operators/prior_box_op.cc
+++ b/paddle/fluid/operators/detection/prior_box_op.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/prior_box_op.h"
+#include "paddle/fluid/operators/detection/prior_box_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/prior_box_op.cu b/paddle/fluid/operators/detection/prior_box_op.cu
similarity index 99%
rename from paddle/fluid/operators/prior_box_op.cu
rename to paddle/fluid/operators/detection/prior_box_op.cu
index 0ea8909296f8f52d252b0ec258666cf32d69a8bb..f67e6ca91c0852b5a3be35d23246884d1157caa4 100644
--- a/paddle/fluid/operators/prior_box_op.cu
+++ b/paddle/fluid/operators/detection/prior_box_op.cu
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/prior_box_op.h"
+#include "paddle/fluid/operators/detection/prior_box_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/prior_box_op.h b/paddle/fluid/operators/detection/prior_box_op.h
similarity index 100%
rename from paddle/fluid/operators/prior_box_op.h
rename to paddle/fluid/operators/detection/prior_box_op.h
diff --git a/paddle/fluid/operators/target_assign_op.cc b/paddle/fluid/operators/detection/target_assign_op.cc
similarity index 98%
rename from paddle/fluid/operators/target_assign_op.cc
rename to paddle/fluid/operators/detection/target_assign_op.cc
index 9fce216e880f2af3b665202f4c4d2777995db003..367001939251114a9cf442fd85c734958ccb2da8 100644
--- a/paddle/fluid/operators/target_assign_op.cc
+++ b/paddle/fluid/operators/detection/target_assign_op.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/target_assign_op.h"
+#include "paddle/fluid/operators/detection/target_assign_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/target_assign_op.cu b/paddle/fluid/operators/detection/target_assign_op.cu
similarity index 97%
rename from paddle/fluid/operators/target_assign_op.cu
rename to paddle/fluid/operators/detection/target_assign_op.cu
index 24664f99b20f92108220d27ec58e8fdf3ba6193c..ddf6889942355457fb281b6c33430ab8337db3ed 100644
--- a/paddle/fluid/operators/target_assign_op.cu
+++ b/paddle/fluid/operators/detection/target_assign_op.cu
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
-#include "paddle/fluid/operators/target_assign_op.h"
+#include "paddle/fluid/operators/detection/target_assign_op.h"
 
 namespace paddle {
 namespace operators {
diff --git a/paddle/fluid/operators/target_assign_op.h b/paddle/fluid/operators/detection/target_assign_op.h
similarity index 100%
rename from paddle/fluid/operators/target_assign_op.h
rename to paddle/fluid/operators/detection/target_assign_op.h