标签爬虫去重

src/tag_source/stackoverflow.py

@@ -42,12 +42,17 @@ class TagPipeline(object):
         self.file = open('dataset/stackoverflow.tag.json', 'w')
         self.file.write('[\n')
         self.count = 0
+        self.tags = {}
 
     def close_spider(self, spider):
         self.file.write('\n]')
         self.file.close()
 
     def process_item(self, item, spider):
+        if self.tags.get(item['name']) is not None:
+            return
+        self.tags[item['name']] = True
+
         words = []
         if self.count>0:
             words.append(',\n')