Skip to content

N
NewsRecommendSystem

小刘同学liu / NewsRecommendSystem
与 Fork 源项目一致

从无法访问的项目Fork

通知 1
Star 0
Fork 0
N
NewsRecommendSystem

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

提交 cd5a83f1 编写于 作者: T Tom Qian
浏览文件
操作

First submit of all files

上级 2065f794
隐藏空白更改
内联 并排
Showing with 2470 addition and 0 deletion
.classpath 0 → 100644
浏览文件 @ cd5a83f1
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/jfinal-2.2-all/jfinal-2.2-lib/log4j/log4j-1.2.16.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/jfinal-2.2-all/jfinal-2.2-lib/jackson/jackson-core-2.5.3.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/jfinal-2.2-all/jfinal-2.2-lib/c3p0/c3p0-0.9.5.1.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/数据库驱动/mysql-connector-java-5.1.40-bin.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/分词/ansj/ansj_seg-5.0.3.jar" sourcepath="/Users/hanbo/Desktop/边城/开源好工具/分词/ansj/ansj_seg-5.0.3-sources.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/分词/ansj/tree_split-1.4.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/分词/ansj/nlp-lang-1.7.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/日志类工具/slf4j-api-1.7.10.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-flink_2.10-0.12.2.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2-dependency-reduced.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-math-0.12.2.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-mr-0.12.2.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-mr-0.12.2-job.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/quartz-2.2.2/lib/quartz-2.2.2.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/开源好工具/数据库驱动/postgresql-9.4.1207.jre6.jar"/>
<classpathentry kind="lib" path="/Users/hanbo/Desktop/边城/apache-mahout-0.12.2/mahout-integration-0.12.2.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>
.project 0 → 100644
浏览文件 @ cd5a83f1
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>RecommandSystem</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
.settings/org.eclipse.core.resources.prefs 0 → 100644
浏览文件 @ cd5a83f1
eclipse.preferences.version=1
encoding//src/log4j.properties=UTF-8
.settings/org.eclipse.jdt.core.prefs 0 → 100644
浏览文件 @ cd5a83f1
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.source=1.8
RecommandSystem.iml 0 → 100644
浏览文件 @ cd5a83f1
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="EclipseModuleManager">
<libelement value="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-3.14-20160307.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/log4j/log4j-1.2.16.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-ooxml-3.14-20160307.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/xmlbeans-2.6.0.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-excelant-3.14-20160307.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-ooxml-schemas-3.14-20160307.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/jackson/jackson-core-2.5.3.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-flink_2.10-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2-dependency-reduced.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-hdfs-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-integration-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-math-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-math-scala_2.10-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-mr-0.12.2.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/c3p0/c3p0-0.9.5.1.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/c3p0/mchange-commons-java-0.2.10.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/数据库驱动/mysql-connector-java-5.1.40-bin.jar!/" />
<libelement value="jar://$MODULE_DIR$/../../开源好工具/分词/ansj_seg-2.0.8-min.jar!/" />
<src_description expected_position="0">
<src_folder value="file://$MODULE_DIR$/src" expected_position="0" />
</src_description>
</component>
<component name="NewModuleRootManager" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/bin" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="jdk" jdkName="JavaSE-1.8" jdkType="JavaSDK" />
<orderEntry type="module-library">
<library name="poi-3.14-20160307.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-3.14-20160307.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-3.14-20160307.jar!/" />
</SOURCES>
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="log4j-1.2.16.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/log4j/log4j-1.2.16.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="poi-ooxml-3.14-20160307.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-ooxml-3.14-20160307.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="xmlbeans-2.6.0.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/xmlbeans-2.6.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="poi-excelant-3.14-20160307.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-excelant-3.14-20160307.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="poi-ooxml-schemas-3.14-20160307.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/poi-3.14/poi-ooxml-schemas-3.14-20160307.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="jackson-core-2.5.3.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/jackson/jackson-core-2.5.3.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-flink_2.10-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-flink_2.10-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-h2o_2.10-0.12.2-dependency-reduced.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2-dependency-reduced.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-h2o_2.10-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-h2o_2.10-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-hdfs-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-hdfs-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-integration-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-integration-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-math-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-math-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-math-scala_2.10-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-math-scala_2.10-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mahout-mr-0.12.2.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../apache-mahout-0.12.2/mahout-mr-0.12.2.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="c3p0-0.9.5.1.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/c3p0/c3p0-0.9.5.1.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mchange-commons-java-0.2.10.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../jfinal-2.2-all/jfinal-2.2-lib/c3p0/mchange-commons-java-0.2.10.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="mysql-connector-java-5.1.40-bin.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/数据库驱动/mysql-connector-java-5.1.40-bin.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
<orderEntry type="module-library">
<library name="ansj_seg-2.0.8-min.jar">
<CLASSES>
<root url="jar://$MODULE_DIR$/../../开源好工具/分词/ansj_seg-2.0.8-min.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
</component>
</module>
\ No newline at end of file
bin/.DS_Store 0 → 100644
浏览文件 @ cd5a83f1
文件已添加
bin/log4j.properties 0 → 100644
浏览文件 @ cd5a83f1
###设置###
log4j.rootLogger = stdout,D,E
### 输出信息到控制台###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
###输出DEBUG级别日志到/Users/hanbo/Desktop/debug.log ###
log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
log4j.appender.D.File = /Users/hanbo/Desktop/debug.log
log4j.appender.D.Append = true
log4j.appender.D.Threshold = DEBUG
log4j.appender.D.layout = org.apache.log4j.PatternLayout
log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
###输出ERROR级别日志到/Users/hanbo/Desktop/error.log###
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =/Users/hanbo/Desktop/error.log
log4j.appender.E.Append = true
log4j.appender.E.Threshold = ERROR
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
\ No newline at end of file
bin/top/.DS_Store 0 → 100644
浏览文件 @ cd5a83f1
文件已添加
res/dbconfig.properties 0 → 100644
浏览文件 @ cd5a83f1
#url=jdbc:mysql://121.42.36.199/recommand_system
#user=root
#password=123456
#dbname=recommand_system
#ip=121.42.36.199
url = jdbc:postgresql://202.114.234.171:25432/mcipdb
user = postgres
password = n749fj
dbname=mcipdb
#url = jdbc:postgresql://115.28.212.146:5432/mcipdb
#user = postgres
#password = n749fj
#dbname=mcipdb
\ No newline at end of file
res/paraConfig.properties 0 → 100644
浏览文件 @ cd5a83f1
#---Recommend System Parameter---
#Recommend Executing Moment(Quartz's CronExpression)
startAt=0 0 0 ? * *
#---Specific Algorithm Parameter---
#-RecommendKit-
#Valid specific day the news published after which are still valuable
beforeDays=-30
#Valid specific day after which those who has browsed news can be regarded "active"
activeDay=-30
#-Collaborative Filtering Recommendation-
#Recommend Num from CF(Collaborative Filtering)
CFRecNum=5
#Valid specific day after which users' browsing history would be calculated
CFValidDay=-30
#-Content-Based Recommendation-
#Recommend Num from CB(Content-Based Recommend)
CBRecNum=5
#Set a previous day after which the browsing history will be calculated with, namely, is still in time
previousDays=-30
#TF-IDF's extracting numbers of keywords from every news
TFIDFKeywordsNum=10
#-Hot Recommendation-
src/.DS_Store 0 → 100644
浏览文件 @ cd5a83f1
文件已添加
src/log4j.properties 0 → 100644
浏览文件 @ cd5a83f1
###设置###
log4j.rootLogger = stdout,D,E
### 输出信息到控制台###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
###输出DEBUG级别日志到/Users/hanbo/Desktop/debug.log ###
log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
log4j.appender.D.File = /Users/hanbo/Desktop/debug.log
log4j.appender.D.Append = true
log4j.appender.D.Threshold = DEBUG
log4j.appender.D.layout = org.apache.log4j.PatternLayout
log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
###输出ERROR级别日志到/Users/hanbo/Desktop/error.log###
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =/Users/hanbo/Desktop/error.log
log4j.appender.E.Append = true
log4j.appender.E.Threshold = ERROR
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
\ No newline at end of file
src/top/.DS_Store 0 → 100644
浏览文件 @ cd5a83f1
文件已添加
src/top/qianxinyao/Main/DataProcessor.java 0 → 100644
浏览文件 @ cd5a83f1
///**
// *
// */
//package top.qianxinyao.Main;
//
//import java.io.File;
//import java.io.FileInputStream;
//import java.io.FileNotFoundException;
//import java.io.IOException;
//import java.text.DecimalFormat;
//import java.util.HashMap;
//import java.util.Iterator;
//
//import org.apache.log4j.Logger;
//import org.apache.poi.hssf.usermodel.HSSFCell;
//import org.apache.poi.hssf.usermodel.HSSFRow;
//import org.apache.poi.hssf.usermodel.HSSFSheet;
//import org.apache.poi.hssf.usermodel.HSSFWorkbook;
//import org.apache.poi.ss.usermodel.Cell;
//import org.apache.poi.ss.usermodel.Row;
//
//
///**
// * @author qianxinyao
// * @email tomqianmaple@gmail.com
// * @github https://github.com/bluemapleman
// * @date 2016年10月16日
// */
//
//public class DataProcessor
//{
//
// private static Logger logger = Logger.getLogger(DataProcessor.class);
//
// private static HashMap<String,HashMap<String,Integer>> userLikes;
//
// /**
// * 读取数据表，获得用户的喜好数据
// */
// public static void readDataTable(){
//
// }
//
// /**
// * 读取excel文件，获得用户的喜好数据
// *
// * @param file
// */
//
// public static void readExcelData(File file)
// {
// userLikes=new HashMap<String,HashMap<String,Integer>>();
//
// HSSFWorkbook wb=null;
//
// try
// {
// wb = new HSSFWorkbook(new FileInputStream(file));
//
// HSSFSheet sheet = wb.getSheetAt(0);
//
// // Iterate over each row in the sheet
//
// Iterator<Row> rows = sheet.rowIterator();
//
// while (rows.hasNext())
// {
// HashMap<String,Integer> likes=new HashMap<String,Integer>();
//
// HSSFRow row = (HSSFRow) rows.next();
//
// // Iterate over each cell in the row and print out the cell"s
//
// // content
//
// Iterator<Cell> cells = row.cellIterator();
//
//
// while (cells.hasNext())
// {
// Cell cell=cells.next();
//
// String cellValue=getCellValue((HSSFCell) cell);
//
//
// logger.info(cellValue);
//
// }
// }
// }
// catch (FileNotFoundException fe)
// {
// logger.error("Exception:" + fe.toString());
// }
// catch (Exception e)
// {
// logger.error("Exception:" + e.toString());
// }
// finally
// {
// if(null!=wb){
// try
// {
// wb.close();
// }
// catch (IOException e)
// {
// // TODO Auto-generated catch block
// logger.error("XSSFWorkbook close failed!");
// }
// }
// logger.info("程序出现异常！请检查！");
// }
// }
//
//
// /**
// * 返回各种类型单元格值的字符串形式的方法
// * @param cell
// * @return
// */
// private static String getCellValue(HSSFCell cell) {
// String cellValue = "";
// DecimalFormat df = new DecimalFormat("#");
// switch (cell.getCellType()) {
// case HSSFCell.CELL_TYPE_STRING:
// cellValue = cell.getRichStringCellValue().getString().trim();
// break;
// case HSSFCell.CELL_TYPE_NUMERIC:
// cellValue = df.format(cell.getNumericCellValue()).toString();
// break;
// case HSSFCell.CELL_TYPE_BOOLEAN:
// cellValue = String.valueOf(cell.getBooleanCellValue()).trim();
// break;
// case HSSFCell.CELL_TYPE_FORMULA:
// cellValue = cell.getCellFormula();
// break;
// default:
// cellValue = "";
// }
// return cellValue;
// }
//
//
//
//
//
// /**
// * 分隔用户所喜好新闻数据的方法，获得用户喜好的所有新闻信息。
// * @param likes
// * @param seperator 新闻数据分隔符
// * @return
// */
//// private static String[] seperateLikes(String likes,String seperator){
//// return likes.split(seperator);
//// }
//
//}
src/top/qianxinyao/Main/Main.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.Main;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import top.qianxinyao.UserBasedCollaborativeRecommender.MahoutUserBasedCollaborativeRecommender;
import top.qianxinyao.algorithms.PropGetKit;
import top.qianxinyao.algorithms.RecommendKit;
import top.qianxinyao.contentbasedrecommend.ContentBasedRecommender;
import top.qianxinyao.hotrecommend.HotRecommender;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月20日
*/
public class Main
{
static ResultSet rs=null;
/**
* @param args
* @throws SQLException
*/
public static void main(String[] args) throws SQLException
{
//加载系统配置文件
PropGetKit.loadProperties("paraConfig");
//仅给最近一个月有活动的用户进行推荐动作
List<String> users=RecommendKit.getActiveUsers();
//设定推荐任务每天的执行时间
// String cronExpression=PropGetKit.getString("startAt");
// try
// {
// new CFCronTriggerRunner().task(users,cronExpression);
// new CBCronTriggerRunner().task(users,cronExpression);
// new HRCronTriggerRunner().task(users,cronExpression);
// }
// catch (SchedulerException e)
// {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
HotRecommender.formTodayTopHotNewsList();
// new MahoutUserBasedCollaborativeRecommender().recommend(users);
// new ContentBasedRecommender().recommend(users);
new HotRecommender().recommend(users);
}
}
src/top/qianxinyao/UserBasedCollaborativeRecommender/MahoutUserBasedCollaborativeRecommender.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.UserBasedCollaborativeRecommender;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.log4j.Logger;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.model.jdbc.PostgreSQLBooleanPrefJDBCDataModel;
import org.apache.mahout.cf.taste.impl.neighborhood.NearestNUserNeighborhood;
import org.apache.mahout.cf.taste.impl.recommender.GenericUserBasedRecommender;
import org.apache.mahout.cf.taste.impl.similarity.LogLikelihoodSimilarity;
import org.apache.mahout.cf.taste.neighborhood.UserNeighborhood;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.apache.mahout.cf.taste.recommender.Recommender;
import org.apache.mahout.cf.taste.similarity.UserSimilarity;
import top.qianxinyao.algorithms.PropGetKit;
import top.qianxinyao.algorithms.RecommendAlgorithm;
import top.qianxinyao.algorithms.RecommendKit;
import top.qianxinyao.dbconnection.ConnectionFactory;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月18日 协同过滤
*/
/**
* Collaborative-Based Filter 基于用户的协同过滤
*
*/
public class MahoutUserBasedCollaborativeRecommender implements RecommendAlgorithm
{
public static final Logger logger = Logger.getLogger(MahoutUserBasedCollaborativeRecommender.class);
/**
* 对应计算相似度时的时效天数
*/
private static int inRecDays = PropGetKit.getInt("CFValidDay");
/**
* 给每个用户推荐的新闻的条数
*/
public static int N =PropGetKit.getInt("CFRecNum");
/**
* 给特定的一批用户进行新闻推荐
*
* @param 目标用户的id列表
*/
@SuppressWarnings("unused")
@Override
public void recommend(List<String> users)
{
int count=0;
try
{
System.out.println("CF start at "+new Date());
PostgreSQLBooleanPrefJDBCDataModel dataModel = ConnectionFactory.getPostgreSQLBooleanPrefJDBCDataModel();
Statement stmt = ConnectionFactory.getNewStatement();
ResultSet rs1 = stmt.executeQuery("select " + ConnectionFactory.PREF_TABLE_USERID + ","
+ ConnectionFactory.PREF_TABLE_NEWSID + "," + ConnectionFactory.PREF_TABLE_TIME + " from newslogs");
// 移除过期的用户浏览新闻行为，这些行为对计算用户相似度不再具有较大价值
while (rs1.next())
{
if (rs1.getTimestamp(3).before(RecommendKit.getInRecTimestamp(inRecDays)))
{
dataModel.removePreference(Long.parseLong(rs1.getString(1)), Long.parseLong(rs1.getString(2)));
}
}
UserSimilarity similarity = new LogLikelihoodSimilarity(dataModel);
// NearestNeighborhood的数量有待考察
UserNeighborhood neighborhood = new NearestNUserNeighborhood(5, similarity, dataModel);
Recommender recommender = new GenericUserBasedRecommender(dataModel, neighborhood, similarity);
for (String user : users)
{
long start = System.currentTimeMillis();
Long userid = Long.parseLong(user);
List<RecommendedItem> recItems = recommender.recommend(userid, N);
Set<String> hs = new HashSet<String>();
for (RecommendedItem recItem : recItems)
{
hs.add(String.valueOf(recItem.getItemID()));
}
// 过滤掉已推荐新闻和已过期新闻
RecommendKit.filterOutDateNews(hs, String.valueOf(userid));
RecommendKit.filterReccedNews(hs, String.valueOf(userid));
// 无可推荐新闻
if (hs == null)
{
continue;
}
if(hs.size()>N){
RecommendKit.removeOverNews(hs, N);
}
RecommendKit.insertRecommend(String.valueOf(userid), hs.iterator(),RecommendAlgorithm.CF);
count+=hs.size();
}
}
catch (TasteException e)
{
logger.error("CB算法构造偏好对象失败！");
e.printStackTrace();
}
catch (SQLException e)
{
logger.error("CB算法数据库操作失败！");
e.printStackTrace();
}
System.out.println("CF has contributed " + (count/users.size()) + " recommending news on average");
System.out.println("CF finish at "+new Date());
return;
}
public int getRecNums()
{
return N;
}
public void setRecNums(int recNums)
{
N = recNums;
}
}
src/top/qianxinyao/UserBasedCollaborativeRecommender/quartz/CFCronTriggerRunner.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.UserBasedCollaborativeRecommender.quartz;
import java.util.List;
import org.quartz.CronExpression;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.JobDetailImpl;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.triggers.CronTriggerImpl;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
*/
public class CFCronTriggerRunner
{
public void task(List<String> users,String cronExpression) throws SchedulerException
{
// Initiate a Schedule Factory
SchedulerFactory schedulerFactory = new StdSchedulerFactory();
// Retrieve a scheduler from schedule factory
Scheduler scheduler = schedulerFactory.getScheduler();
// Initiate JobDetail with job name, job group, and executable job class
JobDetailImpl jobDetailImpl =
new JobDetailImpl();
jobDetailImpl.setJobClass(CFJob.class);
jobDetailImpl.setKey(new JobKey("CFJob1"));
jobDetailImpl.getJobDataMap().put("users", users);
// Initiate CronTrigger with its name and group name
CronTriggerImpl cronTriggerImpl = new CronTriggerImpl();
cronTriggerImpl.setName("CFCronTrigger1");
try {
// setup CronExpression
CronExpression cexp = new CronExpression(cronExpression);
// Assign the CronExpression to CronTrigger
cronTriggerImpl.setCronExpression(cexp);
} catch (Exception e) {
e.printStackTrace();
}
// schedule a job with JobDetail and Trigger
scheduler.scheduleJob(jobDetailImpl, cronTriggerImpl);
// start the scheduler
scheduler.start();
}
}
src/top/qianxinyao/UserBasedCollaborativeRecommender/quartz/CFJob.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.UserBasedCollaborativeRecommender.quartz;
import java.util.List;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import top.qianxinyao.UserBasedCollaborativeRecommender.MahoutUserBasedCollaborativeRecommender;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
* 每天定时根据用户当日的新闻浏览记录来更新用户的喜好关键词列表
*/
public class CFJob implements Job
{
@SuppressWarnings("unchecked")
@Override
public void execute(JobExecutionContext arg0) throws JobExecutionException
{
List<String> users=(List<String>) arg0.getJobDetail().getJobDataMap().get("users");
new MahoutUserBasedCollaborativeRecommender().recommend(users);
}
}
src/top/qianxinyao/algorithms/JsonKit.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.algorithms;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import top.qianxinyao.contentbasedrecommend.CustomizedHashMap;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月21日
*/
public class JsonKit
{
public static String test(){
String json=null;
try {
Map<Integer, Object> moduleidMap = new HashMap<Integer, Object>();
Map<String, Double> keywordRateMap = new HashMap<String, Double>();
keywordRateMap.put("政治", 123.1);
keywordRateMap.put("金融", 35.2);
moduleidMap.put(1,keywordRateMap);
keywordRateMap.put("电影", 351.1);
moduleidMap.put(2,keywordRateMap);
ObjectMapper objectMapper=new ObjectMapper();
json=objectMapper.writeValueAsString(moduleidMap);
String test="{\"1\":{},\"2\":{},\"3\":{},\"4\":{}}";
return test;
} catch (IOException e) {
e.printStackTrace();
}
return json;
}
/**
* 获取用户所关注的模板的id的set
* @param srcJson
* @return
*/
public static Set<Integer> getUserModuleIdSet(String srcJson){
//java的擦除机制不允许直接获取泛型类的class,但是这样会使得jackson的readValue自动将键转换为String，于是需要使用jackson提供的TypeReference来解决这个问题
Map<Integer, Object> map=null;
try
{
ObjectMapper objectMapper=new ObjectMapper();
map = objectMapper.readValue(srcJson, new TypeReference<Map<Integer,Object>>(){});
}
catch (JsonParseException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (JsonMappingException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
return map.keySet();
}
/**
* 获得用户对应module的喜好关键词列表的map
* @param srcJson
* @param moduleId
* @return
*/
@SuppressWarnings("unchecked")
public static LinkedHashMap<String,Double> getModulePrefMap(String srcJson,int moduleId){
LinkedHashMap<String,Double> keyWordsRateMap=null;
try
{
ObjectMapper objectMapper=new ObjectMapper();
//java的擦除机制不允许直接获取泛型类的class,但是这样会使得jackson的readValue自动将键转换为String，于是需要使用jackson提供的TypeReference来解决这个问题
Map<Integer,Object> map=objectMapper.readValue(srcJson, new TypeReference<Map<Integer,Object>>(){});
keyWordsRateMap=(LinkedHashMap<String,Double>) map.get(moduleId);
}
catch (JsonParseException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (JsonMappingException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
return keyWordsRateMap;
}
/**
* 将用户的喜好关键词列表字符串转换为map
* @param srcJson
* @return
*/
public static CustomizedHashMap<Integer,CustomizedHashMap<String,Double>> jsonPrefListtoMap(String srcJson){
ObjectMapper objectMapper=new ObjectMapper();
CustomizedHashMap<Integer,CustomizedHashMap<String,Double>> map=null;
try
{
map=objectMapper.readValue(srcJson, new TypeReference<CustomizedHashMap<Integer,CustomizedHashMap<String,Double>>>(){});
}
catch (JsonParseException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (JsonMappingException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
return map;
}
}
src/top/qianxinyao/algorithms/PropGetKit.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.algorithms;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Properties;
import org.apache.log4j.Logger;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月30日 用以读取配置文件，获取对应属性
*/
public class PropGetKit
{
private static final Logger logger = Logger.getLogger(PropGetKit.class);
public static Properties propGetKit = new Properties();;
public static void loadProperties(String configFileName)
{
try
{
propGetKit.load(new FileInputStream(System.getProperty("user.dir") + "/res/" + configFileName + ".properties"));
}
catch (FileNotFoundException e)
{
logger.error("读取属性文件--->失败！- 原因：文件路径错误或者文件不存在");
}
catch (IOException e)
{
logger.error("装载文件--->失败!");
}
}
public static String getString(String key)
{
return propGetKit.getProperty(key);
}
public static int getInt(String key)
{
return Integer.valueOf(propGetKit.getProperty(key));
}
}
src/top/qianxinyao/algorithms/RecommendAlgorithm.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.algorithms;
import java.util.List;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月20日
*/
public interface RecommendAlgorithm
{
/**
* 推荐算法的int表示
*/
//协同过滤
public static final int CF=0;
//基于内容的推荐
public static final int CB=1;
//基于热点新闻的推荐
public static final int HR=2;
/**
* 针对所有用户返回推荐结果
*/
public default void recommend(){
recommend(RecommendKit.getUserList());
}
/**
* 针对特定用户返回推荐结果
*/
public void recommend(List<String> users);
}
src/top/qianxinyao/algorithms/RecommendKit.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.algorithms;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import top.qianxinyao.contentbasedrecommend.CustomizedHashMap;
import top.qianxinyao.dbconnection.ConnectionFactory;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月21日 提供推荐算法通用的一些方法
*/
public class RecommendKit
{
/**
* 推荐新闻的时效性天数，即从推荐当天开始到之前beforeDays天的新闻属于仍具有时效性的新闻，予以推荐。
*/
private static int beforeDays = PropGetKit.getInt("beforeDays");
/**
* @return the inRecDate 返回时效时间的"year-month-day"的格式表示，方便数据库的查询
*/
public static String getInRecDate()
{
return getSpecificDayFormat(beforeDays);
}
/**
* @return the inRecDate 返回时效时间的"year-month-day"的格式表示，方便数据库的查询
*/
public static String getInRecDate(int beforeDays)
{
return getSpecificDayFormat(beforeDays);
}
/**
* @return the inRecDate 返回时效时间timestamp形式表示，方便其他推荐方法在比较时间先后时调用
*/
public static Timestamp getInRecTimestamp(int before_Days)
{
Calendar calendar = Calendar.getInstance(); // 得到日历
calendar.add(Calendar.DAY_OF_MONTH, before_Days); // 设置为前beforeNum天
return new Timestamp(calendar.getTime().getTime());
}
/**
* 过滤方法filterOutDateNews() 过滤掉失去时效性的新闻（由beforeDays属性控制）
*/
public static void filterOutDateNews(Collection<String> col, String userId)
{
try
{
String newsids = getInQueryString(col.iterator());
if (!newsids.equals("()"))
{
ResultSet rs = ConnectionFactory.getStatement()
.executeQuery("select newsid,ntime from news where newsid in " + newsids);
while (rs.next())
{
if (rs.getTimestamp(2).before(getInRecTimestamp(beforeDays)))
{
col.remove(rs.getString(1));
}
}
}
}
catch (SQLException e)
{
e.printStackTrace();
}
}
/**
* 过滤方法filterBrowsedNews() 过滤掉已经用户已经看过的新闻
*/
public static void filterBrowsedNews(Collection<String> col, String userId)
{
try
{
Statement stmt = ConnectionFactory.getNewStatement();
ResultSet rs;
rs = stmt.executeQuery("select nlnewsid from newslogs where nluserid='" + userId + "'");
while (rs.next())
{
if (col.contains(rs.getString(1)))
{
col.remove(rs.getString(1));
}
}
}
catch (SQLException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* 过滤方法filterReccedNews() 过滤掉已经推荐过的新闻（在recommend表中查找）
*/
public static void filterReccedNews(Collection<String> col, String userId)
{
try
{
Statement stmt = ConnectionFactory.getNewStatement();
ResultSet rs;
rs = stmt.executeQuery("select rnewsid from recommend where ruserid='" + userId + "' and rrectime>"+getInRecDate());
while (rs.next())
{
if (col.contains(rs.getString(1)))
{
col.remove(rs.getString(1));
}
}
}
catch (SQLException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* 获取所有用户的Id列表
*
* @return
*/
public static ArrayList<String> getUserList()
{
ArrayList<String> users = new ArrayList<String>();
try
{
ResultSet rs = ConnectionFactory.getNewStatement().executeQuery("select userid from users");
while (rs.next())
{
users.add(rs.getString(1));
}
}
catch (SQLException e)
{
e.printStackTrace();
}
return users;
}
public static int getbeforeDays()
{
return beforeDays;
}
public static void setbeforeDays(int beforeDays)
{
RecommendKit.beforeDays = beforeDays;
}
public static String getSpecificDayFormat(int before_Days)
{
SimpleDateFormat date_format = new SimpleDateFormat("yyyy-MM-dd");
Calendar calendar = Calendar.getInstance(); // 得到日历
calendar.add(Calendar.DAY_OF_MONTH, before_Days); // 设置为前beforeNum天
Date d = calendar.getTime();
return "'" + date_format.format(d) + "'";
}
/**
* 获取所有用户的喜好关键词列表
*
* @return
*/
public static HashMap<String, CustomizedHashMap<Integer, CustomizedHashMap<String, Double>>> getUserPrefListMap(
Collection<String> userSet)
{
ResultSet rs = null;
HashMap<String, CustomizedHashMap<Integer, CustomizedHashMap<String, Double>>> userPrefListMap = null;
try
{
String userPrefListQuery = getInQueryStringWithSingleQuote(userSet.iterator());
if (!userPrefListQuery.equals("()"))
{
rs = ConnectionFactory.getNewStatement()
.executeQuery("select userid,upreflist from users where userid in " + userPrefListQuery);
userPrefListMap = new HashMap<String, CustomizedHashMap<Integer, CustomizedHashMap<String, Double>>>();
while (rs.next())
{
userPrefListMap.put(rs.getString(1), JsonKit.jsonPrefListtoMap(rs.getString(2)));
}
}
}
catch (SQLException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
return userPrefListMap;
}
/**
* 用以select语句中使用in (n1，n2,n3...)范围查询的字符串拼接
*
* @param ite
* 待查询对象集合的迭代器
* @return 若迭代集合不为空:"(n1,n2,n3)"，若为空："()"
*/
public static <T> String getInQueryString(Iterator<T> ite)
{
String inQuery = "(";
while (ite.hasNext())
{
inQuery += ite.next() + ",";
}
if (inQuery.length() > 1)
{
inQuery = inQuery.substring(0, inQuery.length() - 1);
}
inQuery += ")";
return inQuery;
}
public static <T> String getInQueryStringWithSingleQuote(Iterator<T> ite)
{
String inQuery = "(";
while (ite.hasNext())
{
inQuery += "'" + ite.next() + "',";
}
if (inQuery.length() > 1)
{
inQuery = inQuery.substring(0, inQuery.length() - 1);
}
inQuery += ")";
return inQuery;
}
/**
* 将推荐结果插入recommend表
*
* @param userId
* 推荐目标用户id
* @param newsIte
* 待推荐新闻集合的迭代器
* @param recAlgo
* 标明推荐结果来自哪个推荐算法(RecommendAlgorithm.XX)
*/
public static void insertRecommend(String userId, Iterator<String> newsIte, int recAlgo)
{
try
{
String insertValues = "";
while (newsIte.hasNext())
{
insertValues += "(" + userId + "," + newsIte.next() + ",'" + new Timestamp(System.currentTimeMillis())
+ "'," + recAlgo + "),";
}
if (insertValues.length() > 0)
{
insertValues = insertValues.substring(0, insertValues.length() - 1);
ConnectionFactory.getNewStatement()
.execute("insert into recommend (ruserid,rnewsid,rrectime,rrecalgo) values " + insertValues);
}
}
catch (SQLException e)
{
e.printStackTrace();
}
}
/**
* Acquire list of "active" users' ids
* "Active" means who use app recently(determined by method getInRecDate())
*
* @return
*/
public static List<String> getActiveUsers()
{
try
{
int activeDay=PropGetKit.getInt("activeDay");
ResultSet rs1;
rs1 = ConnectionFactory.getNewStatement()
.executeQuery("select distinct nluserid from newslogs where nltime>" + getInRecDate(activeDay));
List<String> users = new ArrayList<String>();
while (rs1.next())
{
users.add(rs1.getString(1));
}
return users;
}
catch (SQLException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("return null");
return null;
}
/**
* 去除数量上超过为算法设置的推荐结果上限值的推荐结果
*
* @param set
* @param N
* @return
*/
public static void removeOverNews(Set<String> set, int N)
{
int i = 0;
Iterator<String> ite = set.iterator();
while (ite.hasNext())
{
if (i >= N)
{
ite.remove();
ite.next();
}
else
{
ite.next();
}
i++;
}
}
}
src/top/qianxinyao/contentbasedrecommend/ContentBasedRecommender.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.ansj.app.keyword.Keyword;
import org.apache.log4j.Logger;
import top.qianxinyao.algorithms.PropGetKit;
import top.qianxinyao.algorithms.RecommendAlgorithm;
import top.qianxinyao.algorithms.RecommendKit;
import top.qianxinyao.dbconnection.ConnectionFactory;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月20日 基于内容的推荐 Content-Based
*
* 思路：提取抓取进来的新闻的关键词列表（tf-idf），与每个用户的喜好关键词列表，做关键词相似度计算，取最相似的N个新闻推荐给用户。
*
* Procedure: 1、Every time that the recommendation is started up(according
* to quartz framework), the current day's coming in news will be
* processed by class TF-IDF's getTFIDF() method to obtain their key words
* list.And then the system go over every user and calculate the
* similarity between every news's key words list with user's preference
* list.After that, rank the news according to the similarities and
* recommend them to users.
*/
public class ContentBasedRecommender implements RecommendAlgorithm
{
public static final Logger logger = Logger.getLogger(ContentBasedRecommender.class);
// TFIDF提取关键词数
private static final int KEY_WORDS_NUM = PropGetKit.getInt("TFIDFKeywordsNum");
// 推荐新闻数
private static final int N = PropGetKit.getInt("CBRecNum");
@Override
public void recommend(List<String> users)
{
try
{
int count=0;
System.out.println("CB start at "+ new Date());
// 首先进行用户喜好关键词列表的衰减更新+用户当日历史浏览记录的更新
new UserPrefRefresher().refresh(users);
// 新闻及对应关键词列表的Map
HashMap<String, List<Keyword>> newsKeyWordsMap = new HashMap<String, List<Keyword>>();
HashMap<String, Integer> newsModuleMap = new HashMap<String, Integer>();
// 用户喜好关键词列表
HashMap<String, CustomizedHashMap<Integer, CustomizedHashMap<String, Double>>> userPrefListMap = RecommendKit
.getUserPrefListMap(users);
ResultSet rs = ConnectionFactory.getStatement()
.executeQuery("select newsid,ntitle,ncontent,nmoduleid from news where ntime>"
+ RecommendKit.getInRecDate() + " and ncontent not like '<meta%'");
while (rs.next())
{
newsKeyWordsMap.put(rs.getString(1), TFIDF.getTFIDE(rs.getString(2), rs.getString(3), KEY_WORDS_NUM));
newsModuleMap.put(rs.getString(1), rs.getInt(4));
}
for (String userId : users)
{
Map<String, Double> tempMatchMap = new HashMap<String, Double>();
Iterator<String> ite = newsKeyWordsMap.keySet().iterator();
while (ite.hasNext())
{
String newsId = ite.next();
int moduleId = newsModuleMap.get(newsId);
if (null != userPrefListMap.get(userId).get(moduleId))
tempMatchMap.put(newsId,
getMatchValue(userPrefListMap.get(userId).get(moduleId), newsKeyWordsMap.get(newsId)));
else
continue;
}
// 去除匹配值为0的项目
removeZeroItem(tempMatchMap);
if (!(tempMatchMap.toString().equals("{}")))
{
tempMatchMap = sortMapByValue(tempMatchMap);
Set<String> toBeRecommended=tempMatchMap.keySet();
//过滤掉已经推荐过的新闻
RecommendKit.filterReccedNews(toBeRecommended,userId);
//过滤掉用户已经看过的新闻
RecommendKit.filterBrowsedNews(toBeRecommended, userId);
if(toBeRecommended.size()>N){
RecommendKit.removeOverNews(toBeRecommended,N);
}
RecommendKit.insertRecommend(userId, toBeRecommended.iterator(),RecommendAlgorithm.CB);
count+=toBeRecommended.size();
}
}
System.out.println("CB has contributed " + (count/users.size()) + " recommending news on average");
System.out.println("CB finished at "+new Date());
}
catch (Exception e)
{
e.printStackTrace();
}
return;
}
/**
* 获得用户的关键词列表和新闻关键词列表的匹配程度
*
* @return
*/
private double getMatchValue(CustomizedHashMap<String, Double> map, List<Keyword> list)
{
Set<String> keywordsSet = map.keySet();
double matchValue = 0;
for (Keyword keyword : list)
{
if (keywordsSet.contains(keyword.getName()))
{
matchValue += keyword.getScore() * map.get(keyword.getName());
}
}
return matchValue;
}
private void removeZeroItem(Map<String, Double> map)
{
HashSet<String> toBeDeleteItemSet = new HashSet<String>();
Iterator<String> ite = map.keySet().iterator();
while (ite.hasNext())
{
String newsId = ite.next();
if (map.get(newsId) <= 0)
{
toBeDeleteItemSet.add(newsId);
}
}
for (String item : toBeDeleteItemSet)
{
map.remove(item);
}
}
/**
* 使用 Map按value进行排序
* @param map
* @return
*/
public static Map<String, Double> sortMapByValue(Map<String, Double> oriMap) {
if (oriMap == null || oriMap.isEmpty()) {
return null;
}
Map<String, Double> sortedMap = new LinkedHashMap<String, Double>();
List<Map.Entry<String, Double>> entryList = new ArrayList<Map.Entry<String, Double>>(
oriMap.entrySet());
Collections.sort(entryList, new MapValueComparator());
Iterator<Map.Entry<String, Double>> iter = entryList.iterator();
Map.Entry<String, Double> tmpEntry = null;
while (iter.hasNext()) {
tmpEntry = iter.next();
sortedMap.put(tmpEntry.getKey(), tmpEntry.getValue());
}
return sortedMap;
}
}
src/top/qianxinyao/contentbasedrecommend/CustomizedComparator.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.util.Comparator;
import java.util.Map;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月30日
*/
public class CustomizedComparator implements Comparator<String>
{
Map<String, Double> base;
public CustomizedComparator(Map<String, Double> base) {
this.base = base;
}
@Override
public int compare(String a, String b)
{
if (base.get(a) >= base.get(b)) {
return 1;
}
else {
return -1;
}
}
}
src/top/qianxinyao/contentbasedrecommend/CustomizedHashMap.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
*/
public class CustomizedHashMap<K, V> extends HashMap<K,V>
{
private static final long serialVersionUID = 1L;
@Override
public String toString(){
String toString="{";
Iterator<K> keyIte=this.keySet().iterator();
while(keyIte.hasNext()){
K key=keyIte.next();
toString+="\""+key+"\":"+this.get(key)+",";
}
if(toString.equals("{")){
toString="{}";
}
else{
toString=toString.substring(0, toString.length()-1)+"}";
}
return toString;
}
public CustomizedHashMap<K,V> copyFromLinkedHashMap(LinkedHashMap<K,V> linkedHashMap){
// Iterator<K> ite = linkedHashMap.keySet().iterator();
// while(ite.hasNext()){
// K key=ite.next();
// this.put(key,linkedHashMap.get(key));
// }
this.putAll(linkedHashMap);
return this;
}
}
src/top/qianxinyao/contentbasedrecommend/MapValueComparator.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.util.Comparator;
import java.util.Map;
import java.util.Map.Entry;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年12月2日
*/
class MapValueComparator implements Comparator<Map.Entry<String, Double>> {
@Override
public int compare(Entry<String, Double> me1, Entry<String, Double> me2) {
return me1.getValue().compareTo(me2.getValue());
}
}
\ No newline at end of file
src/top/qianxinyao/contentbasedrecommend/TFIDF.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.util.List;
import org.ansj.app.keyword.KeyWordComputer;
import org.ansj.app.keyword.Keyword;
import org.ansj.domain.Result;
import org.ansj.splitWord.analysis.ToAnalysis;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月23日
*/
public class TFIDF
{
public static Result split(String text)
{
return ToAnalysis.parse(text);
}
/**
*
* @param title 文本标题
* @param content 文本内容
* @param keyNums 返回的关键词数目
* @return
*/
public static List<Keyword> getTFIDE(String title, String content,int keyNums)
{
// String
// sentence="我今天很开心，所以一口气买了好多东西。然而我一不小心把本月预算透支了，现在有很不开心了，因为后面的日子得吃土了！";
KeyWordComputer kwc = new KeyWordComputer(keyNums);
return kwc.computeArticleTfidf(title, content);
}
/**
*
* @param content 文本内容
* @param keyNums 返回的关键词数目
* @return
*/
public static List<Keyword> getTFIDE(String content,int keyNums)
{
KeyWordComputer kwc = new KeyWordComputer(keyNums);
return kwc.computeArticleTfidf(content);
}
}
src/top/qianxinyao/contentbasedrecommend/UserPrefRefresher.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import org.ansj.app.keyword.Keyword;
import top.qianxinyao.algorithms.JsonKit;
import top.qianxinyao.algorithms.RecommendKit;
import top.qianxinyao.dbconnection.ConnectionFactory;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月3日 每次用户浏览新的新闻时，用以更新用户的喜好关键词列表
*/
public class UserPrefRefresher
{
ResultSet rs=null;
//设置TFIDF提取的关键词数目
private static final int KEY_WORDS_NUM = 10;
//每日衰减系数
private static final double DEC_COEE=0.7;
public void refresh(){
refresh(RecommendKit.getUserList());
}
@SuppressWarnings("unchecked")
public void refresh(Collection<String> userIdsCol){
//首先对用户的喜好关键词列表进行衰减更新
autoDecRefresh(userIdsCol);
//用户浏览新闻纪录：userBrowsexMap:<String(userid),ArrayList<String>(newsid List)>
HashMap<String,ArrayList<String>> userBrowsedMap=getBrowsedHistoryMap();
//用户喜好关键词列表：userPrefListMap:<String(userid),String(json))>
HashMap<String,CustomizedHashMap<Integer,CustomizedHashMap<String,Double>>> userPrefListMap=RecommendKit.getUserPrefListMap(userBrowsedMap.keySet());
//新闻对应关键词列表与模块ID：newsTFIDFMap:<String(newsid),List<Keyword>>,<String(newsModuleId),Integer(moduleid)>
HashMap<String,Object> newsTFIDFMap=getNewsTFIDFMap();
//开始遍历用户浏览记录，更新用户喜好关键词列表
//对每个用户（外层循环），循环他所看过的每条新闻（内层循环），对每个新闻，更新它的关键词列表到用户的对应模块中
Iterator<String> ite=userBrowsedMap.keySet().iterator();
while(ite.hasNext()){
String userId=ite.next();
ArrayList<String> newsList=userBrowsedMap.get(userId);
for(String news:newsList){
Integer moduleId=(Integer) newsTFIDFMap.get(news+"moduleid");
//获得对应模块的（关键词：喜好）map
CustomizedHashMap<String,Double> rateMap=userPrefListMap.get(userId).get(moduleId);
//获得新闻的（关键词：TFIDF值）map
List<Keyword> keywordList=(List<Keyword>) newsTFIDFMap.get(news);
Iterator<Keyword> keywordIte=keywordList.iterator();
while(keywordIte.hasNext()){
Keyword keyword=keywordIte.next();
String name=keyword.getName();
if(rateMap.containsKey(name)){
rateMap.put(name, rateMap.get(name)+keyword.getScore());
}
else{
rateMap.put(name,keyword.getScore());
}
}
userPrefListMap.get(userId);
}
}
Iterator<String> iterator=userBrowsedMap.keySet().iterator();
while(iterator.hasNext()){
String userId=iterator.next();
try
{
ConnectionFactory.getStatement().executeUpdate("update users set upreflist='"+userPrefListMap.get(userId)+"' where userid='"+userId+"'");
}
catch (SQLException e)
{
e.printStackTrace();
}
}
}
/**
* 所有用户的喜好关键词列表TFIDF值随时间进行自动衰减更新
*/
public void autoDecRefresh(){
autoDecRefresh(RecommendKit.getUserList());
}
/**
* 所有用户的喜好关键词列表TFIDF值随时间进行自动衰减更新
*/
public void autoDecRefresh(Collection<String> userIdsCol){
try
{
String inQuery=RecommendKit.getInQueryStringWithSingleQuote(userIdsCol.iterator());
if(inQuery.equals("()")){
return;
}
ResultSet rs=ConnectionFactory.getStatement().executeQuery("select userid,upreflist from users where userid in "+inQuery);
//用以更新的用户喜好关键词map的json串
//用于删除喜好值过低的关键词
ArrayList<String> keywordToDelete=new ArrayList<String>();
while(rs.next()){
String newPrefList="{";
HashMap<Integer,CustomizedHashMap<String,Double>> map=JsonKit.jsonPrefListtoMap(rs.getString(2));
Iterator<Integer> ite=map.keySet().iterator();
while(ite.hasNext()){
//用户对应模块的喜好不为空
Integer moduleId=ite.next();
CustomizedHashMap<String,Double> moduleMap=map.get(moduleId);
newPrefList+="\""+moduleId+"\":";
//N:{"X1":n1,"X2":n2,.....}
if(!(moduleMap.toString().equals("{}"))){
Iterator<String> inIte=moduleMap.keySet().iterator();
while(inIte.hasNext()){
String key=inIte.next();
//累计TFIDF值乘以衰减系数
double result=moduleMap.get(key)*DEC_COEE;
if(result<10){
keywordToDelete.add(key);
}
moduleMap.put(key,result);
}
}
for(String deleteKey:keywordToDelete){
moduleMap.remove(deleteKey);
}
keywordToDelete.clear();
newPrefList+=moduleMap.toString()+",";
}
newPrefList="'"+newPrefList.substring(0,newPrefList.length()-1)+"}'";
ConnectionFactory.getNewStatement().executeUpdate("update users set upreflist="+newPrefList+" where userid='"+rs.getString(1)+"'");
}
}
catch (SQLException e)
{
e.printStackTrace();
}
}
/**
* 提取出当天所有用户浏览新闻纪录
* @return
*/
private HashMap<String,ArrayList<String>> getBrowsedHistoryMap(){
HashMap<String, ArrayList<String>> userBrowsedMap=null;
try
{
userBrowsedMap=new HashMap<String,ArrayList<String>>();
ResultSet rs = ConnectionFactory.getStatement().executeQuery("select * from newslogs where nltime>"+RecommendKit.getSpecificDayFormat(0));
while(rs.next()){
if(userBrowsedMap.containsKey(rs.getString(2))){
userBrowsedMap.get(rs.getString(2)).add(rs.getString(3));
}
else{
userBrowsedMap.put(rs.getString(2), new ArrayList<String>());
userBrowsedMap.get(rs.getString(2)).add(rs.getString(3));
}
}
}
catch (SQLException e)
{
e.printStackTrace();
}
return userBrowsedMap;
}
private HashSet<String> getBrowsedNewsSet(){
HashMap<String,ArrayList<String>> browsedMap=getBrowsedHistoryMap();
HashSet<String> newsIdSet=new HashSet<String>();
Iterator<String> ite=getBrowsedHistoryMap().keySet().iterator();
while(ite.hasNext()){
Iterator<String> inIte=browsedMap.get(ite.next()).iterator();
while(inIte.hasNext()){
newsIdSet.add(inIte.next());
}
}
return newsIdSet;
}
/**
* 将所有当天被浏览过的新闻提取出来，以便进行TFIDF求值操作，以及对用户喜好关键词列表的更新。
* @return
*/
private HashMap<String,Object> getNewsTFIDFMap(){
HashMap<String,Object> newsTFIDFMap=null;
try
{
Iterator<String> ite=getBrowsedNewsSet().iterator();
String newsIdListQuery="(";
while(ite.hasNext()){
newsIdListQuery+=ite.next()+",";
}
//用户如果当天没看新闻
if(newsIdListQuery.length()>1){
newsIdListQuery=newsIdListQuery.substring(0, newsIdListQuery.length()-1)+")";
//提取出所有新闻的关键词列表及对应TF-IDf值，并放入一个map中
rs=ConnectionFactory.getStatement().executeQuery("select newsid,ntitle,ncontent,nmoduleid from news where newsid in "+newsIdListQuery);
newsTFIDFMap=new HashMap<String,Object>();
while(rs.next()){
newsTFIDFMap.put(rs.getString(1), TFIDF.getTFIDE(rs.getString(2), rs.getString(3),KEY_WORDS_NUM));
newsTFIDFMap.put(rs.getString(1)+"moduleid", rs.getInt(4));
}
}
else
return null;
}
catch (SQLException e)
{
e.printStackTrace();
}
return newsTFIDFMap;
}
}
src/top/qianxinyao/contentbasedrecommend/quartz/CBCronTriggerRunner.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend.quartz;
import java.util.List;
import org.quartz.CronExpression;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.JobDetailImpl;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.triggers.CronTriggerImpl;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
*/
public class CBCronTriggerRunner
{
public void task(List<String> users,String cronExpression) throws SchedulerException
{
// Initiate a Schedule Factory
SchedulerFactory schedulerFactory = new StdSchedulerFactory();
// Retrieve a scheduler from schedule factory
Scheduler scheduler = schedulerFactory.getScheduler();
// Initiate JobDetail with job name, job group, and executable job class
JobDetailImpl jobDetailImpl =
new JobDetailImpl();
jobDetailImpl.setJobClass(CBJob.class);
jobDetailImpl.setKey(new JobKey("CBJob1"));
jobDetailImpl.getJobDataMap().put("users", users);
// Initiate CronTrigger with its name and group name
CronTriggerImpl cronTriggerImpl = new CronTriggerImpl();
cronTriggerImpl.setName("CBCronTrigger1");
try {
// setup CronExpression
CronExpression cexp = new CronExpression(cronExpression);
// Assign the CronExpression to CronTrigger
cronTriggerImpl.setCronExpression(cexp);
} catch (Exception e) {
e.printStackTrace();
}
// schedule a job with JobDetail and Trigger
scheduler.scheduleJob(jobDetailImpl, cronTriggerImpl);
// start the scheduler
scheduler.start();
}
}
src/top/qianxinyao/contentbasedrecommend/quartz/CBJob.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.contentbasedrecommend.quartz;
import java.util.List;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import top.qianxinyao.contentbasedrecommend.ContentBasedRecommender;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
* 每天定时根据用户当日的新闻浏览记录来更新用户的喜好关键词列表
*/
public class CBJob implements Job
{
@SuppressWarnings("unchecked")
@Override
public void execute(JobExecutionContext arg0) throws JobExecutionException
{
List<String> users=(List<String>) arg0.getJobDetail().getJobDataMap().get("users");
new ContentBasedRecommender().recommend(users);
}
}
src/top/qianxinyao/dbconnection/ConnectionFactory.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.dbconnection;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Properties;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.apache.mahout.cf.taste.impl.model.jdbc.MySQLJDBCDataModel;
import org.apache.mahout.cf.taste.impl.model.jdbc.PostgreSQLBooleanPrefJDBCDataModel;
import org.postgresql.jdbc3.Jdbc3SimpleDataSource;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年10月21日
*/
public class ConnectionFactory
{
public static final Logger logger = Logger.getLogger(ConnectionFactory.class);
//偏好表表名
public static final String PREF_TABLE="newslogs";
//用户id列名
public static final String PREF_TABLE_USERID="nlonguserid";
//新闻id列名
public static final String PREF_TABLE_NEWSID="nlnewsid";
//偏好值列名
public static final String PREF_TABLE_PREFVALUE="nprefer";
//用户浏览时间列名
public static final String PREF_TABLE_TIME="nltime";
public static final String MYSQL="com.mysql.jdbc.Driver";
public static final String POSTGRE="org.postgresql.Driver";
public static Connection conn;
public static Statement stmt;
// private static void initalize(String database)
// {
// try
// {
// Class.forName(database).newInstance();
// HashMap<String, String> info = getDBInfo();
// conn = (Connection) DriverManager.getConnection(info.get("url"), info.get("user"), info.get("password"));
// }
// catch (ClassNotFoundException e)
// {
// logger.error("找不到驱动程序类 ，加载驱动失败！");
// }
// catch (SQLException se)
// {
// logger.error("数据库连接失败！");
// }
// catch (Exception e)
// {
// logger.error("数据库连接初始化错误！");
// }
// return;
// }
public static Connection getConnection()
{
if (null == conn)
{
try
{
conn=getDataSource().getConnection();
}
catch (SQLException e)
{
e.printStackTrace();
}
}
return conn;
}
public static Statement getStatement(){
if(null==stmt){
try
{
stmt=getConnection().createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE,
ResultSet.CONCUR_UPDATABLE);
}
catch (SQLException e)
{
e.printStackTrace();
}
}
return stmt;
}
public static Statement getNewStatement(){
Statement watchedStmt=null;
try
{
watchedStmt=getConnection().createStatement();
new StatementWatcher(watchedStmt).start();
}
catch (SQLException e)
{
e.printStackTrace();
}
return watchedStmt;
}
public static DataSource getDataSource()
{
Jdbc3SimpleDataSource dataSource=new Jdbc3SimpleDataSource();
HashMap<String,String> info=getDBInfo();
dataSource.setUrl(info.get("url"));
dataSource.setUser(info.get("user"));
dataSource.setPassword(info.get("password"));
return dataSource;
}
public static PostgreSQLBooleanPrefJDBCDataModel getPostgreSQLBooleanPrefJDBCDataModel(){
return new PostgreSQLBooleanPrefJDBCDataModel(ConnectionFactory.getDataSource(), PREF_TABLE, PREF_TABLE_USERID,
PREF_TABLE_NEWSID,PREF_TABLE_TIME);
}
public static MySQLJDBCDataModel getMySQLJDBCDataModel(){
return new MySQLJDBCDataModel(ConnectionFactory.getDataSource(), "user_likes", "uid",
"nid", "likes", "recording_time");
}
private static HashMap<String, String> getDBInfo()
{
HashMap<String, String> info = null;
try
{
Properties p = new Properties();
p.load(new FileInputStream(System.getProperty("user.dir") + "/res/dbconfig.properties"));
info = new HashMap<String, String>();
info.put("url", p.getProperty("url"));
info.put("user", p.getProperty("user"));
info.put("password", p.getProperty("password"));
}
catch (FileNotFoundException e)
{
logger.error("读取属性文件--->失败！- 原因：文件路径错误或者文件不存在");
}
catch (IOException e)
{
logger.error("装载文件--->失败!");
}
return info;
}
}
src/top/qianxinyao/dbconnection/StatementWatcher.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.dbconnection;
import java.sql.SQLException;
import java.sql.Statement;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
*/
public class StatementWatcher extends Thread
{
private Statement watchedStmt;
public StatementWatcher(Statement watchedStmt){
this.watchedStmt=watchedStmt;
}
public void run(){
//防止还未开始
try
{
Thread.sleep(10000);
}
catch (InterruptedException e1)
{
// TODO Auto-generated catch block
e1.printStackTrace();
}
while(true){
try
{
if(this.watchedStmt.isClosed()){
break;
}
else{
this.watchedStmt.close();
}
}
catch (SQLException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
src/top/qianxinyao/hotrecommend/HotRecommender.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.hotrecommend;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import top.qianxinyao.algorithms.RecommendAlgorithm;
import top.qianxinyao.algorithms.RecommendKit;
import top.qianxinyao.dbconnection.ConnectionFactory;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月30日 基于“热点新闻”生成的推荐，一般用于在CF和CB算法推荐结果数较少时进行数目的补充
*/
public class HotRecommender implements RecommendAlgorithm
{
// 热点新闻的有效时间
public static int beforeDays = -10;
// 推荐系统每日为每位用户生成的推荐结果的总数，当CF与CB算法生成的推荐结果数不足此数时，由该算法补充
public static int TOTAL_REC_NUM = 20;
// 将每天生成的“热点新闻”ID，按照新闻的热点程度从高到低放入此List
private static ArrayList<String> topHotNewsList = new ArrayList<String>();
@Override
public void recommend(List<String> users)
{
System.out.println("HR start at "+new Date());
int count=0;
Timestamp timestamp = getCertainTimestamp(0, 0, 0);
for (String userId : users)
{
try
{
ResultSet rs = ConnectionFactory.getNewStatement()
.executeQuery("select ruserid,count(*) as recnums from recommend where rrectime>'" + timestamp
+ "' and ruserid='" + userId + "' group by ruserid");
boolean flag=rs.next();
int delta=flag?TOTAL_REC_NUM - rs.getInt("recnums"):TOTAL_REC_NUM;
Set<String> toBeRecommended = new HashSet<String>();
if (delta > 0)
{
int i = topHotNewsList.size() > delta ? delta : topHotNewsList.size();
while (i-- > 0)
toBeRecommended.add(topHotNewsList.get(i));
}
RecommendKit.filterBrowsedNews(toBeRecommended, userId);
RecommendKit.filterReccedNews(toBeRecommended, userId);
RecommendKit.insertRecommend(userId, toBeRecommended.iterator(), RecommendAlgorithm.HR);
count+=toBeRecommended.size();
}
catch (SQLException e)
{
e.printStackTrace();
}
}
System.out.println("HR has contributed " + (count/users.size()) + " recommending news on average");
System.out.println("HR end at "+new Date());
}
public static void formTodayTopHotNewsList()
{
topHotNewsList.clear();
ArrayList<String> hotNewsTobeReccommended = new ArrayList<String>();
try
{
ResultSet rs = ConnectionFactory.getNewStatement()
.executeQuery("select nlnewsid,count(*) as visitNums from newslogs where nltime>"
+ RecommendKit.getInRecDate(beforeDays) + " group by nlnewsid order by visitNums desc");
while (rs.next())
{
hotNewsTobeReccommended.add(rs.getString(1));
}
for (String news : hotNewsTobeReccommended)
{
topHotNewsList.add(news);
}
System.out.println(topHotNewsList);
}
catch (SQLException e)
{
e.printStackTrace();
}
}
public static List<String> getTopHotNewsList()
{
return topHotNewsList;
}
public static int getTopHopNewsListSize()
{
return topHotNewsList.size();
}
private Timestamp getCertainTimestamp(int hour, int minute, int second)
{
Calendar calendar = Calendar.getInstance(); // 得到日历
calendar.set(Calendar.HOUR_OF_DAY, hour); // 设置为前beforeNum天
calendar.set(Calendar.MINUTE, minute);
calendar.set(Calendar.SECOND, second);
return new Timestamp(calendar.getTime().getTime());
}
}
src/top/qianxinyao/hotrecommend/quartz/HRCronTriggerRunner.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.hotrecommend.quartz;
import java.util.List;
import org.quartz.CronExpression;
import org.quartz.JobKey;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.JobDetailImpl;
import org.quartz.impl.StdSchedulerFactory;
import org.quartz.impl.triggers.CronTriggerImpl;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
*/
public class HRCronTriggerRunner
{
public void task(List<String> users,String cronExpression) throws SchedulerException
{
// Initiate a Schedule Factory
SchedulerFactory schedulerFactory = new StdSchedulerFactory();
// Retrieve a scheduler from schedule factory
Scheduler scheduler = schedulerFactory.getScheduler();
// Initiate JobDetail with job name, job group, and executable job class
JobDetailImpl jobDetailImpl =
new JobDetailImpl();
jobDetailImpl.setJobClass(HRJob.class);
jobDetailImpl.setKey(new JobKey("HRJob1"));
jobDetailImpl.getJobDataMap().put("users",users);
// Initiate CronTrigger with its name and group name
CronTriggerImpl cronTriggerImpl = new CronTriggerImpl();
cronTriggerImpl.setName("HRCronTrigger1");
try {
// setup CronExpression
CronExpression cexp = new CronExpression(cronExpression);
// Assign the CronExpression to CronTrigger
cronTriggerImpl.setCronExpression(cexp);
} catch (Exception e) {
e.printStackTrace();
}
// schedule a job with JobDetail and Trigger
scheduler.scheduleJob(jobDetailImpl, cronTriggerImpl);
// start the scheduler
scheduler.start();
}
}
src/top/qianxinyao/hotrecommend/quartz/HRJob.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.hotrecommend.quartz;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import top.qianxinyao.hotrecommend.HotRecommender;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年11月23日
* 每天定时生成热点新闻的列表
*/
public class HRJob implements Job
{
@Override
public void execute(JobExecutionContext arg0) throws JobExecutionException
{
HotRecommender.getTopHotNewsList().clear();
HotRecommender.formTodayTopHotNewsList();
}
}
src/top/qianxinyao/performance/Judge.java 0 → 100644
浏览文件 @ cd5a83f1
/**
*
*/
package top.qianxinyao.performance;
/**
* @author qianxinyao
* @email tomqianmaple@gmail.com
* @github https://github.com/bluemapleman
* @date 2016年12月2日
* 用以评估推荐效果的类
* 1、准确率（precision） 2、召回率（recall）
*/
public class Judge
{
public double getPrecision(){
return 0;
}
}
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或