Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
南宫伊儿
elasticsearch-analysis-ik
提交
f0dc55aa
E
elasticsearch-analysis-ik
项目概览
南宫伊儿
/
elasticsearch-analysis-ik
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
E
elasticsearch-analysis-ik
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f0dc55aa
编写于
9月 24, 2014
作者:
goBD
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Update Dictionary.java
给远程加载建立监控
上级
11a89476
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
119 addition
and
3 deletion
+119
-3
src/main/java/org/wltea/analyzer/dic/Dictionary.java
src/main/java/org/wltea/analyzer/dic/Dictionary.java
+119
-3
未找到文件。
src/main/java/org/wltea/analyzer/dic/Dictionary.java
浏览文件 @
f0dc55aa
...
...
@@ -25,11 +25,18 @@
*/
package
org.wltea.analyzer.dic
;
import
org.apache.http.client.ClientProtocolException
;
import
org.apache.http.client.config.RequestConfig
;
import
org.apache.http.client.methods.CloseableHttpResponse
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.impl.client.CloseableHttpClient
;
import
org.apache.http.impl.client.HttpClients
;
import
org.elasticsearch.common.logging.ESLogger
;
import
org.elasticsearch.common.logging.Loggers
;
import
org.wltea.analyzer.cfg.Configuration
;
import
java.io.*
;
import
java.util.ArrayList
;
import
java.util.Collection
;
import
java.util.List
;
...
...
@@ -92,6 +99,17 @@ public class Dictionary {
singleton
.
loadSuffixDict
();
singleton
.
loadPrepDict
();
singleton
.
loadStopWordDict
();
//建立监控线程
for
(
String
location:
cfg
.
getRemoteExtDictionarys
()){
Thread
monitor
=
new
Thread
(
new
Monitor
(
location
));
monitor
.
start
();
}
for
(
String
location:
cfg
.
getRemoteExtStopWordDictionarys
()){
Thread
monitor
=
new
Thread
(
new
Monitor
(
location
));
monitor
.
start
();
}
return
singleton
;
}
}
...
...
@@ -224,6 +242,8 @@ public class Dictionary {
}
//加载扩展词典
this
.
loadExtDict
();
//加载远程自定义词库
this
.
loadRemoteExtDict
();
}
/**
...
...
@@ -275,6 +295,76 @@ public class Dictionary {
}
}
/**
* 加载远程扩展词典到主词库表
*/
private
void
loadRemoteExtDict
(){
List
<
String
>
remoteExtDictFiles
=
configuration
.
getRemoteExtDictionarys
();
for
(
String
location:
remoteExtDictFiles
){
logger
.
info
(
"[Dict Loading]"
+
location
);
List
<
String
>
lists
=
getRemoteWords
(
location
);
//如果找不到扩展的字典,则忽略
if
(
lists
==
null
){
logger
.
error
(
"[Dict Loading]"
+
location
+
"加载失败"
);
continue
;
}
for
(
String
theWord:
lists
){
if
(
theWord
!=
null
&&
!
""
.
equals
(
theWord
.
trim
()))
{
//加载扩展词典数据到主内存词典中
logger
.
info
(
theWord
);
_MainDict
.
fillSegment
(
theWord
.
trim
().
toLowerCase
().
toCharArray
());
}
}
}
}
/**
* 从远程服务器上下载自定义词条
*/
private
static
List
<
String
>
getRemoteWords
(
String
location
){
List
<
String
>
buffer
=
new
ArrayList
<
String
>();
RequestConfig
rc
=
RequestConfig
.
custom
().
setConnectionRequestTimeout
(
10
*
1000
)
.
setConnectTimeout
(
10
*
1000
).
setSocketTimeout
(
60
*
1000
).
build
();
CloseableHttpClient
httpclient
=
HttpClients
.
createDefault
();
CloseableHttpResponse
response
;
BufferedReader
in
;
HttpGet
get
=
new
HttpGet
(
location
);
get
.
setConfig
(
rc
);
try
{
response
=
httpclient
.
execute
(
get
);
if
(
response
.
getStatusLine
().
getStatusCode
()==
200
){
String
charset
=
"UTF-8"
;
//获取编码,默认为utf-8
if
(
response
.
getEntity
().
getContentType
().
getValue
().
contains
(
"charset="
)){
String
contentType
=
response
.
getEntity
().
getContentType
().
getValue
();
charset
=
contentType
.
substring
(
contentType
.
lastIndexOf
(
"="
)+
1
);
}
in
=
new
BufferedReader
(
new
InputStreamReader
(
response
.
getEntity
().
getContent
(),
charset
));
String
line
;
while
((
line
=
in
.
readLine
())!=
null
){
buffer
.
add
(
line
);
}
in
.
close
();
response
.
close
();
return
buffer
;
}
response
.
close
();
}
catch
(
ClientProtocolException
e
)
{
e
.
printStackTrace
();
}
catch
(
IllegalStateException
e
)
{
e
.
printStackTrace
();
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
return
buffer
;
}
/**
* 加载用户扩展的停止词词典
*/
...
...
@@ -360,7 +450,28 @@ public class Dictionary {
}
}
}
}
}
//加载远程停用词典
List
<
String
>
remoteExtStopWordDictFiles
=
configuration
.
getRemoteExtStopWordDictionarys
();
for
(
String
location:
remoteExtStopWordDictFiles
){
logger
.
info
(
"[Dict Loading]"
+
location
);
List
<
String
>
lists
=
getRemoteWords
(
location
);
//如果找不到扩展的字典,则忽略
if
(
lists
==
null
){
logger
.
error
(
"[Dict Loading]"
+
location
+
"加载失败"
);
continue
;
}
for
(
String
theWord:
lists
){
if
(
theWord
!=
null
&&
!
""
.
equals
(
theWord
.
trim
()))
{
//加载远程词典数据到主内存中
logger
.
info
(
theWord
);
_StopWords
.
fillSegment
(
theWord
.
trim
().
toLowerCase
().
toCharArray
());
}
}
}
}
/**
...
...
@@ -511,6 +622,11 @@ public class Dictionary {
}
}
}
public
void
reLoadMainDict
(){
logger
.
info
(
"重新加载词典..."
);
loadMainDict
();
loadStopWordDict
();
}
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录