Merge pull request #187 from yinhaofeng/dssm

change dssm and match readme

models/match/dssm/config.yaml

@@ -17,50 +17,52 @@ workspace: "models/match/dssm"
 
 dataset:
 - name: dataset_train
-  batch_size: 4
-  type: QueueDataset
+  batch_size: 8
+  type: DataLoader # or QueueDataset
   data_path: "{workspace}/data/train" 
   data_converter: "{workspace}/synthetic_reader.py"
 - name: dataset_infer
   batch_size: 1
-  type: QueueDataset
-  data_path: "{workspace}/data/train"
+  type: DataLoader # or QueueDataset
+  data_path: "{workspace}/data/test"
   data_converter: "{workspace}/synthetic_evaluate_reader.py"
 
 hyper_parameters:
   optimizer:
     class: sgd
-    learning_rate: 0.01
+    learning_rate: 0.001
     strategy: async
-  trigram_d: 1000
-  neg_num: 4
+  trigram_d: 1439
+  neg_num: 1
   fc_sizes: [300, 300, 128]
   fc_acts: ['tanh', 'tanh', 'tanh']
 
-mode: train_runner
+mode: [train_runner,infer_runner]
 # config of each runner.
 # runner is a kind of paddle training class, which wraps the train/infer process.
 runner:
 - name: train_runner
   class: train
   # num of epochs
-  epochs: 4
+  epochs: 3
   # device to run training or infer
   device: cpu
-  save_checkpoint_interval: 2 # save model interval of epochs
-  save_inference_interval: 4 # save inference
+  save_checkpoint_interval: 1 # save model interval of epochs
+  save_inference_interval: 1 # save inference
   save_checkpoint_path: "increment" # save checkpoint path
   save_inference_path: "inference" # save inference path
   save_inference_feed_varnames: ["query", "doc_pos"] # feed vars of save inference
   save_inference_fetch_varnames: ["cos_sim_0.tmp_0"] # fetch vars of save inference
   init_model_path: "" # load model path
   print_interval: 2
+  phases: phase1
 - name: infer_runner
   class: infer
   # device to run training or infer
   device: cpu
   print_interval: 1
   init_model_path: "increment/2" # load model path
+  phases: phase2
 
 # runner will run all the phase in each epoch
 phase:
@@ -68,7 +70,7 @@ phase:
   model: "{workspace}/model.py" # user-defined model
   dataset_name: dataset_train # select dataset by name
   thread_num: 1
-#- name: phase2
-#  model: "{workspace}/model.py" # user-defined model
-#  dataset_name: dataset_infer # select dataset by name
-#  thread_num: 1
+- name: phase2
+  model: "{workspace}/model.py" # user-defined model
+  dataset_name: dataset_infer # select dataset by name
+  thread_num: 1

models/match/dssm/data/preprocess.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#encoding=utf-8
+
+import os
+import sys
+import numpy as np
+import random
+
+f = open("./zhidao", "r")
+lines = f.readlines()
+f.close()
+
+#建立字典
+word_dict = {}
+for line in lines:
+    line = line.strip().split("\t")
+    text = line[0].split(" ") + line[1].split(" ")
+    for word in text:
+        if word in word_dict:
+            continue
+        else:
+            word_dict[word] = len(word_dict) + 1
+
+f = open("./zhidao", "r")
+lines = f.readlines()
+f.close()
+
+lines = [line.strip().split("\t") for line in lines]
+
+#建立以query为key，以负例为value的字典
+neg_dict = {}
+for line in lines:
+    if line[2] == "0":
+        if line[0] in neg_dict:
+            neg_dict[line[0]].append(line[1])
+        else:
+            neg_dict[line[0]] = [line[1]]
+
+#建立以query为key，以正例为value的字典
+pos_dict = {}
+for line in lines:
+    if line[2] == "1":
+        if line[0] in pos_dict:
+            pos_dict[line[0]].append(line[1])
+        else:
+            pos_dict[line[0]] = [line[1]]
+
+#划分训练集和测试集
+query_list = list(pos_dict.keys())
+#print(len(query))
+random.shuffle(query_list)
+train_query = query_list[:90]
+test_query = query_list[90:]
+
+#获得训练集
+train_set = []
+for query in train_query:
+    for pos in pos_dict[query]:
+        if query not in neg_dict:
+            continue
+        for neg in neg_dict[query]:
+            train_set.append([query, pos, neg])
+random.shuffle(train_set)
+
+#获得测试集
+test_set = []
+for query in test_query:
+    for pos in pos_dict[query]:
+        test_set.append([query, pos, 1])
+    if query not in neg_dict:
+        continue
+    for neg in neg_dict[query]:
+        test_set.append([query, neg, 0])
+random.shuffle(test_set)
+
+#训练集中的query,pos,neg转化为词袋
+f = open("train.txt", "w")
+for line in train_set:
+    query = line[0].strip().split(" ")
+    pos = line[1].strip().split(" ")
+    neg = line[2].strip().split(" ")
+    query_token = [0] * (len(word_dict) + 1)
+    for word in query:
+        query_token[word_dict[word]] = 1
+    pos_token = [0] * (len(word_dict) + 1)
+    for word in pos:
+        pos_token[word_dict[word]] = 1
+    neg_token = [0] * (len(word_dict) + 1)
+    for word in neg:
+        neg_token[word_dict[word]] = 1
+    f.write(','.join([str(x) for x in query_token]) + "\t" + ','.join([
+        str(x) for x in pos_token
+    ]) + "\t" + ','.join([str(x) for x in neg_token]) + "\n")
+f.close()
+
+#测试集中的query和pos转化为词袋
+f = open("test.txt", "w")
+fa = open("label.txt", "w")
+for line in test_set:
+    query = line[0].strip().split(" ")
+    pos = line[1].strip().split(" ")
+    label = line[2]
+    query_token = [0] * (len(word_dict) + 1)
+    for word in query:
+        query_token[word_dict[word]] = 1
+    pos_token = [0] * (len(word_dict) + 1)
+    for word in pos:
+        pos_token[word_dict[word]] = 1
+    f.write(','.join([str(x) for x in query_token]) + "\t" + ','.join(
+        [str(x) for x in pos_token]) + "\n")
+    fa.write(str(label) + "\n")
+f.close()
+fa.close()

models/match/dssm/model.py

@@ -73,6 +73,7 @@ class Model(ModelBase):
 
         query_fc = fc(inputs[0], self.hidden_layers, self.hidden_acts,
                       ['query_l1', 'query_l2', 'query_l3'])
+
         doc_pos_fc = fc(inputs[1], self.hidden_layers, self.hidden_acts,
                         ['doc_pos_l1', 'doc_pos_l2', 'doc_pos_l3'])
         R_Q_D_p = fluid.layers.cos_sim(query_fc, doc_pos_fc)
@@ -93,7 +94,7 @@ class Model(ModelBase):
         prob = fluid.layers.softmax(concat_Rs, axis=1)
 
         hit_prob = fluid.layers.slice(
-            prob, axes=[0, 1], starts=[0, 0], ends=[4, 1])
+            prob, axes=[0, 1], starts=[0, 0], ends=[8, 1])
         loss = -fluid.layers.reduce_sum(fluid.layers.log(hit_prob))
         avg_cost = fluid.layers.mean(x=loss)
         self._cost = avg_cost

models/match/dssm/readme.md

+# DSSM文本匹配模型
+
+以下是本例的简要目录结构及说明： 
+
+```
+├── data #样例数据
+	├── train
+		├── train.txt #训练数据样例
+	├── test
+    	├── test.txt #测试数据样例
+	├── preprocess.py #数据处理程序
+├── __init__.py
+├── README.md #文档
+├── model.py #模型文件
+├── config.yaml #配置文件
+├── synthetic_reader.py #读取训练集的程序
+├── synthetic_evaluate_reader.py #读取测试集的程序
+├── transform.py #将数据整理成合适的格式方便计算指标
+├── run.sh #全量数据集中的训练脚本，从训练到预测并计算指标
+
+```
+
+注：在阅读该示例前，建议您先了解以下内容：
+
+[paddlerec入门教程](https://github.com/PaddlePaddle/PaddleRec/blob/master/README.md)
+
+## 内容
+
+- [模型简介](#模型简介)
+- [数据准备](#数据准备)
+- [运行环境](#运行环境)
+- [快速开始](#快速开始)
+- [效果复现](#效果复现)
+- [进阶使用](#进阶使用)
+- [FAQ](#FAQ)
+
+
+## 模型简介
+DSSM是Deep Structured Semantic Model的缩写，即我们通常说的基于深度网络的语义模型，其核心思想是将query和doc映射到到共同维度的语义空间中，通过最大化query和doc语义向量之间的余弦相似度，从而训练得到隐含语义模型，达到检索的目的。DSSM有很广泛的应用，比如：搜索引擎检索，广告相关性，问答系统，机器翻译等。    
+DSSM 的输入采用 BOW（Bag of words）的方式，相当于把字向量的位置信息抛弃了，整个句子里的词都放在一个袋子里了。将一个句子用这种方式转化为一个向量输入DNN中。  
+Query 和 Doc 的语义相似性可以用这两个向量的 cosine 距离表示，然后通过softmax 函数选出与Query语义最相似的样本 Doc 。  
+
+模型的具体细节可以阅读论文[DSSM](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/cikm2013_DSSM_fullversion.pdf):
+<p align="center">
+<img align="center" src="../../../doc/imgs/dssm.png">
+<p>
+
+## 数据准备
+我们公开了自建的测试集，包括百度知道、ECOM、QQSIM、UNICOM 四个数据集。这里我们选取百度知道数据集来进行训练。执行以下命令可以获取上述数据集。
+```
+wget --no-check-certificate https://baidu-nlp.bj.bcebos.com/simnet_dataset-1.0.0.tar.gz
+tar xzf simnet_dataset-1.0.0.tar.gz
+rm simnet_dataset-1.0.0.tar.gz
+```
+
+## 运行环境
+PaddlePaddle>=1.7.2
+
+python 2.7/3.5/3.6/3.7
+
+PaddleRec >=0.1
+
+os : windows/linux/macos
+
+## 快速开始
+本文提供了样例数据可以供您快速体验，在paddlerec目录下执行下面的命令即可快速启动训练： 
+
+```
+python -m paddlerec.run -m models/match/dssm/config.yaml
+```   
+
+输出结果示例：
+```
+PaddleRec: Runner train_runner Begin
+Executor Mode: train
+processor_register begin
+Running SingleInstance.
+Running SingleNetwork.
+file_list : ['models/match/dssm/data/train/train.txt']
+Running SingleStartup.
+Running SingleRunner.
+!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.
+CPU_NUM indicates that how many CPUPlace are used in the current task.
+And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.
+export CPU_NUM=32 # for example, set CPU_NUM as number of physical CPU core which is 32.
+!!! The default number of CPU_NUM=1.
+I0821 06:56:26.224299 31061 parallel_executor.cc:440] The Program will be executed on CPU using ParallelExecutor, 1 cards are used, so 1 programs are executed in parallel.
+I0821 06:56:26.231163 31061 build_strategy.cc:365] SeqOnlyAllReduceOps:0, num_trainers:1
+I0821 06:56:26.237023 31061 parallel_executor.cc:307] Inplace strategy is enabled, when build_strategy.enable_inplace = True
+I0821 06:56:26.240788 31061 parallel_executor.cc:375] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
+batch: 2, LOSS: [4.538238]
+batch: 4, LOSS: [4.16424]
+batch: 6, LOSS: [3.8121371]
+batch: 8, LOSS: [3.4250507]
+batch: 10, LOSS: [3.2285979]
+batch: 12, LOSS: [3.2116117]
+batch: 14, LOSS: [3.1406002]
+epoch 0 done, use time: 0.357971906662, global metrics: LOSS=[3.0968776]
+batch: 2, LOSS: [2.6843479]
+batch: 4, LOSS: [2.546976]
+batch: 6, LOSS: [2.4103594]
+batch: 8, LOSS: [2.301374]
+batch: 10, LOSS: [2.264183]
+batch: 12, LOSS: [2.315862]
+batch: 14, LOSS: [2.3409634]
+epoch 1 done, use time: 0.22123003006, global metrics: LOSS=[2.344321]
+batch: 2, LOSS: [2.0882485]
+batch: 4, LOSS: [2.006743]
+batch: 6, LOSS: [1.9231766]
+batch: 8, LOSS: [1.8850241]
+batch: 10, LOSS: [1.8829436]
+batch: 12, LOSS: [1.9336565]
+batch: 14, LOSS: [1.9784685]
+epoch 2 done, use time: 0.212922096252, global metrics: LOSS=[1.9934461]
+PaddleRec Finish
+```
+## 效果复现
+为了方便使用者能够快速的跑通每一个模型，我们在每个模型下都提供了样例数据。如果需要复现readme中的效果,请按如下步骤依次操作即可。  
+1. 确认您当前所在目录为PaddleRec/models/match/dssm
+2. 在data目录下载并解压数据集，命令如下：  
+``` 
+cd data
+wget --no-check-certificate https://baidu-nlp.bj.bcebos.com/simnet_dataset-1.0.0.tar.gz
+tar xzf simnet_dataset-1.0.0.tar.gz
+rm simnet_dataset-1.0.0.tar.gz
+```
+3. 本文提供了快速将数据集中的汉字数据处理为可训练格式数据的脚本，您在解压数据集后，可以看见目录中存在一个名为zhidao的文件。然后能可以在python3环境下运行我们提供的preprocess.py文件。即可生成可以直接用于训练的数据目录test.txt,train.txt和label.txt。将其放入train和test目录下以备训练时调用。命令如下：
+```
+mv data/zhidao ./
+rm -rf data
+python3 preprocess.py
+rm -f ./train/train.txt
+mv train.txt ./train
+rm -f ./test/test.txt
+mv test.txt test
+cd ..
+```
+经过预处理的格式：  
+训练集为三个稀疏的BOW方式的向量：query,pos,neg  
+测试集为两个稀疏的BOW方式的向量：query,pos  
+label.txt中对应的测试集中的标签
+
+4. 退回dssm目录中，打开文件config.yaml,更改其中的参数  
+
+将workspace改为您当前的绝对路径。（可用pwd命令获取绝对路径）  
+将dataset_train中的batch_size从8改为128
+将文件model.py中的 hit_prob = fluid.layers.slice(prob, axes=[0, 1], starts=[0, 0], ends=[8, 1])  
+    改为hit_prob = fluid.layers.slice(prob, axes=[0, 1], starts=[0, 0], ends=[128, 1]).当您需要改变batchsize的时候，end中第一个参数也需要随之变化
+
+5.  执行脚本，开始训练.脚本会运行python -m paddlerec.run -m ./config.yaml启动训练，并将结果输出到result文件中。然后启动transform.py整合数据，最后计算出正逆序指标：
+```
+sh run.sh
+```
+
+输出结果示例：
+```
+................run.................
+!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.
+CPU_NUM indicates that how many CPUPlace are used in the current task.
+And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.
+
+export CPU_NUM=32 # for example, set CPU_NUM as number of physical CPU core which is 32.
+
+!!! The default number of CPU_NUM=1.
+I0821 07:16:04.512531 32200 parallel_executor.cc:440] The Program will be executed on CPU using ParallelExecutor, 1 cards are used, so 1 programs are executed in parallel.
+I0821 07:16:04.515708 32200 build_strategy.cc:365] SeqOnlyAllReduceOps:0, num_trainers:1
+I0821 07:16:04.518872 32200 parallel_executor.cc:307] Inplace strategy is enabled, when build_strategy.enable_inplace = True
+I0821 07:16:04.520995 32200 parallel_executor.cc:375] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
+75
+pnr: 2.25581395349
+query_num: 11
+pair_num: 184 184
+equal_num: 44
+正序率： 0.692857142857
+97 43
+```
+6. 提醒：因为采取较小的数据集进行训练和测试，得到指标的浮动程度会比较大。如果得到的指标不合预期，可以多次执行步骤5，即可获得合理的指标。
+
+## 进阶使用
+  
+## FAQ

models/match/dssm/run.sh

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#!/bin/bash
+echo "................run................."
+python -m paddlerec.run -m ./config.yaml >result1.txt
+grep -i "query_doc_sim" ./result1.txt >./result2.txt
+sed '$d' result2.txt >result.txt
+rm -f result1.txt
+rm -f result2.txt
+python transform.py
+sort -t $'\t' -k1,1 -k 2nr,2 pair.txt >result.txt
+rm -f pair.txt
+python ../../../tools/cal_pos_neg.py result.txt

models/match/dssm/transform.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import numpy as np
+import sklearn.metrics
+
+label = []
+filename = './data/label.txt'
+f = open(filename, "r")
+f.readline()
+num = 0
+for line in f.readlines():
+    num = num + 1
+    line = line.strip()
+    label.append(line)
+f.close()
+print(num)
+
+filename = './result.txt'
+sim = []
+for line in open(filename):
+    line = line.strip().split(",")
+    line[1] = line[1].split(":")
+    line = line[1][1].strip(" ")
+    line = line.strip("[")
+    line = line.strip("]")
+    sim.append(float(line))
+
+filename = './data/test/test.txt'
+f = open(filename, "r")
+f.readline()
+query = []
+for line in f.readlines():
+    line = line.strip().split("\t")
+    query.append(line[0])
+f.close()
+
+filename = 'pair.txt'
+f = open(filename, "w")
+for i in range(len(sim)):
+    f.write(str(query[i]) + "\t" + str(sim[i]) + "\t" + str(label[i]) + "\n")
+f.close()

models/match/readme.md

 # 匹配模型库
 
 ## 简介
-我们提供了常见的匹配任务中使用的模型算法的PaddleRec实现, 单机训练&预测效果指标以及分布式训练&预测性能指标等。实现的模型包括 [DSSM](http://gitlab.baidu.com/tangwei12/paddlerec/tree/develop/models/match/dssm)、[MultiView-Simnet](http://gitlab.baidu.com/tangwei12/paddlerec/tree/develop/models/match/multiview-simnet)。
+我们提供了常见的匹配任务中使用的模型算法的PaddleRec实现, 单机训练&预测效果指标以及分布式训练&预测性能指标等。实现的模型包括 [DSSM](http://gitlab.baidu.com/tangwei12/paddlerec/tree/develop/models/match/dssm)、[MultiView-Simnet](http://gitlab.baidu.com/tangwei12/paddlerec/tree/develop/models/match/multiview-simnet)、[match-pyramid](https://github.com/PaddlePaddle/PaddleRec/tree/master/models/match/match-pyramid)。
 
 模型算法库在持续添加中，欢迎关注。
 
@@ -18,6 +18,8 @@
 | :------------------: | :--------------------: | :---------: |
 | DSSM | Deep Structured Semantic Models | [CIKM 2013][Learning Deep Structured Semantic Models for Web Search using Clickthrough Data](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/cikm2013_DSSM_fullversion.pdf) |
 | MultiView-Simnet | Multi-view Simnet for Personalized recommendation | [WWW 2015][A Multi-View Deep Learning Approach for Cross Domain User Modeling in Recommendation Systems](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/frp1159-songA.pdf) |
+| match-pyramid | Text Matching as Image Recognition | [arXiv W2016][Text Matching as Image Recognition](https://arxiv.org/pdf/1602.06359.pdf) |
+
 
 下面是每个模型的简介（注：图片引用自链接中的论文）
 
@@ -31,24 +33,26 @@
 <img align="center" src="../../doc/imgs/multiview-simnet.png">
 <p>
 
-## 使用教程(快速开始)
-### 训练
-```shell
-git clone https://github.com/PaddlePaddle/PaddleRec.git paddle-rec
-cd paddle-rec
+[match-pyramid](https://arxiv.org/pdf/1602.06359.pdf):
+<p align="center">
+<img align="center" src="../../doc/imgs/match-pyramid.png">
+<p>
 
+## 使用教程(快速开始)
+### 训练&预测
+每个模型都提供了样例数据可以供您快速体验，在paddlerec目录下直接执行下面的命令即可启动训练：
+```
 python -m paddlerec.run -m models/match/dssm/config.yaml # dssm
 python -m paddlerec.run -m models/match/multiview-simnet/config.yaml # multiview-simnet
+python -m paddlerec.run -m models/contentunderstanding/match-pyramid/config.yaml #match-pyramid
 ```
+### 效果复现
+每个模型下的readme中都有详细的效果复现的教程，您可以进入模型的目录中详细查看  
 
-### 预测
-```shell
-# 修改对应模型的config.yaml, workspace配置为当前目录的绝对路径
-# 修改对应模型的config.yaml，mode配置infer_runner
-# 示例: mode: train_runner -> mode: infer_runner
-# infer_runner中 class配置为 class: infer
-# 修改phase阶段为infer的配置，参照config注释
+### 模型效果 (测试)
 
-# 修改完config.yaml后 执行:
-python -m paddlerec.run -m ./config.yaml # 以dssm为例
-```
+|       数据集        |       模型       |      正逆序比          |       map       |  
+| :------------------: | :--------------------: | :---------: |:---------: |
+|       zhidao       |       DSSM       |       2.25        |       --          | 
+|       Letor07        |       match-pyramid       |       --        |      0.42          | 
+|       zhidao        |       multiview-simnet       |       1.72        |       --          |

tools/cal_pos_neg.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#!/usr/bin/python  
+#-*- coding:utf-8 -*-  
+############################  
+#File Name: cal_pos_neg.py
+#Author: youqiheng 
+#Mail: youqiheng@baidu.com
+#Created Time: 2018-04-15 21:59:45
+############################ 
+"""
+docstring
+"""
+
+import os
+import sys
+
+if len(sys.argv) < 2:
+    print "usage:python %s input" % (sys.argv[0])
+    sys.exit(-1)
+
+fin = file(sys.argv[1])
+pos_num = 0
+neg_num = 0
+
+score_list = []
+label_list = []
+last_query = "-1"
+
+#0       12.786960       1
+#0       -1.480890       0
+cnt = 0
+query_num = 0
+pair_num = 0
+equal_num = 0
+for line in fin:
+    cols = line.strip().split("\t")
+    cnt += 1
+    if cnt % 500000 == 0:
+        print "cnt:", cnt, 1.0 * pos_num / neg_num
+    if len(cols) != 3:
+        continue
+
+    cur_query = cols[0]
+    if cur_query != last_query:
+        query_num += 1
+        for i in xrange(0, len(score_list)):
+            for j in xrange(i + 1, len(score_list)):
+                if label_list[i] == label_list[j]:
+                    continue
+                pair_num += 1
+                if (score_list[i] - score_list[j]) * (
+                        label_list[i] - label_list[j]) < 0:
+                    neg_num += 1
+                elif (score_list[i] - score_list[j]) * (
+                        label_list[i] - label_list[j]) > 0:
+                    pos_num += 1
+                else:
+                    equal_num += 1
+        score_list = []
+        label_list = []
+
+    last_query = cur_query
+
+    label = int(cols[2])
+
+    score_list.append(round(float(cols[1]), 6))
+    label_list.append(int(cols[2]))
+
+fin.close()
+
+for i in xrange(0, len(score_list)):
+    for j in xrange(i + 1, len(score_list)):
+        if label_list[i] == label_list[j]:
+            continue
+        pair_num += 1
+        if (score_list[i] - score_list[j]) * (label_list[i] - label_list[j]
+                                              ) < 0:
+            neg_num += 1
+        elif (score_list[i] - score_list[j]) * (label_list[i] - label_list[j]
+                                                ) > 0:
+            pos_num += 1
+        else:
+            equal_num += 1
+
+if neg_num > 0:
+    print "pnr:", 1.0 * pos_num / neg_num
+    print "query_num:", query_num
+    print "pair_num:", pos_num + neg_num + equal_num, pair_num
+    print "equal_num:", equal_num
+    print "正序率：", 1.0 * pos_num / (pos_num + neg_num)
+print pos_num, neg_num