Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
ssssssss-team
spider-flow
提交
7761eee3
S
spider-flow
项目概览
ssssssss-team
/
spider-flow
10 个月 前同步成功
通知
0
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
spider-flow
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
7761eee3
编写于
4月 09, 2020
作者:
M
mxd
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
rest服务增加异步接口
上级
2a4c1ce2
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
88 addition
and
12 deletion
+88
-12
spider-flow-api/src/main/java/org/spiderflow/concurrent/SpiderFlowThreadPoolExecutor.java
...g/spiderflow/concurrent/SpiderFlowThreadPoolExecutor.java
+4
-0
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java
...r-flow-core/src/main/java/org/spiderflow/core/Spider.java
+3
-3
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Tokenizer.java
...ava/org/spiderflow/core/expression/parsing/Tokenizer.java
+0
-1
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java
...core/src/main/java/org/spiderflow/core/job/SpiderJob.java
+10
-4
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJobManager.java
...c/main/java/org/spiderflow/core/job/SpiderJobManager.java
+3
-2
spider-flow-web/src/main/java/org/spiderflow/controller/SpiderRestController.java
.../java/org/spiderflow/controller/SpiderRestController.java
+68
-2
未找到文件。
spider-flow-api/src/main/java/org/spiderflow/concurrent/SpiderFlowThreadPoolExecutor.java
浏览文件 @
7761eee3
...
...
@@ -42,6 +42,10 @@ public class SpiderFlowThreadPoolExecutor {
});
}
public
Future
<?>
submit
(
Runnable
runnable
){
return
this
.
executor
.
submit
(
runnable
);
}
/**
* 创建子线程池
...
...
spider-flow-core/src/main/java/org/spiderflow/core/Spider.java
浏览文件 @
7761eee3
...
...
@@ -54,7 +54,7 @@ public class Spider {
@Autowired
private
FlowNoticeService
flowNoticeService
;
p
rivate
static
SpiderFlowThreadPoolExecutor
threadPoolExecutor
;
p
ublic
static
SpiderFlowThreadPoolExecutor
executorInstance
;
private
static
final
String
ATOMIC_DEAD_CYCLE
=
"__atomic_dead_cycle"
;
...
...
@@ -62,7 +62,7 @@ public class Spider {
@PostConstruct
private
void
init
()
{
threadPoolExecutor
=
new
SpiderFlowThreadPoolExecutor
(
totalThreads
);
executorInstance
=
new
SpiderFlowThreadPoolExecutor
(
totalThreads
);
}
public
List
<
SpiderOutput
>
run
(
SpiderFlow
spiderFlow
,
SpiderContext
context
,
Map
<
String
,
Object
>
variables
)
{
...
...
@@ -120,7 +120,7 @@ public class Spider {
submitStrategy
=
new
RandomThreadSubmitStrategy
();
}
//创建子线程池,采用一父多子的线程池,子线程数不能超过总线程数(超过时进入队列等待),+1是因为会占用一个线程用来调度执行下一级
SubThreadPoolExecutor
pool
=
threadPoolExecutor
.
createSubThreadPoolExecutor
(
Math
.
max
(
nThreads
,
1
)
+
1
,
submitStrategy
);
SubThreadPoolExecutor
pool
=
executorInstance
.
createSubThreadPoolExecutor
(
Math
.
max
(
nThreads
,
1
)
+
1
,
submitStrategy
);
context
.
setRootNode
(
root
);
context
.
setThreadPool
(
pool
);
//触发监听器
...
...
spider-flow-core/src/main/java/org/spiderflow/core/expression/parsing/Tokenizer.java
浏览文件 @
7761eee3
...
...
@@ -21,7 +21,6 @@ public class Tokenizer {
stream
.
startSpan
();
RuntimeException
re
=
null
;
// TODO: this will fall on its face if we have something like {{ "}}" }}.
while
(
stream
.
hasMore
())
{
if
(
stream
.
match
(
"${"
,
false
))
{
if
(!
stream
.
isSpanEmpty
())
tokens
.
add
(
new
Token
(
TokenType
.
TextBlock
,
stream
.
endSpan
()));
...
...
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJob.java
浏览文件 @
7761eee3
...
...
@@ -55,7 +55,9 @@ public class SpiderJob extends QuartzJobBean {
}
JobDataMap
dataMap
=
context
.
getMergedJobDataMap
();
SpiderFlow
spiderFlow
=
(
SpiderFlow
)
dataMap
.
get
(
SpiderJobManager
.
JOB_PARAM_NAME
);
run
(
spiderFlow
,
context
.
getNextFireTime
());
if
(
"1"
.
equalsIgnoreCase
(
spiderFlow
.
getEnabled
())){
run
(
spiderFlow
,
context
.
getNextFireTime
());
}
}
public
void
run
(
String
id
)
{
...
...
@@ -63,13 +65,17 @@ public class SpiderJob extends QuartzJobBean {
}
public
void
run
(
SpiderFlow
spiderFlow
,
Date
nextExecuteTime
)
{
SpiderJobContext
context
=
null
;
Date
now
=
new
Date
();
Task
task
=
new
Task
();
task
.
setFlowId
(
spiderFlow
.
getId
());
task
.
setBeginTime
(
new
Date
());
taskService
.
save
(
task
);
run
(
spiderFlow
,
task
,
nextExecuteTime
);
}
public
void
run
(
SpiderFlow
spiderFlow
,
Task
task
,
Date
nextExecuteTime
)
{
SpiderJobContext
context
=
null
;
Date
now
=
new
Date
();
try
{
taskService
.
save
(
task
);
context
=
SpiderJobContext
.
create
(
this
.
workspace
,
spiderFlow
.
getId
(),
task
.
getId
(),
false
);
SpiderContextHolder
.
set
(
context
);
contextMap
.
put
(
task
.
getId
(),
context
);
...
...
spider-flow-core/src/main/java/org/spiderflow/core/job/SpiderJobManager.java
浏览文件 @
7761eee3
...
...
@@ -11,6 +11,7 @@ import org.quartz.TriggerBuilder;
import
org.quartz.TriggerKey
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.spiderflow.core.Spider
;
import
org.spiderflow.core.model.SpiderFlow
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Component
;
...
...
@@ -70,9 +71,9 @@ public class SpiderJobManager {
}
public
void
run
(
String
id
){
new
Thread
(()->{
Spider
.
executorInstance
.
submit
(()->{
spiderJob
.
run
(
id
);
})
.
start
()
;
});
}
public
boolean
remove
(
String
id
){
...
...
spider-flow-web/src/main/java/org/spiderflow/controller/SpiderRestController.java
浏览文件 @
7761eee3
...
...
@@ -2,10 +2,14 @@ package org.spiderflow.controller;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.spiderflow.context.SpiderContext
;
import
org.spiderflow.core.Spider
;
import
org.spiderflow.core.job.SpiderJob
;
import
org.spiderflow.core.job.SpiderJobContext
;
import
org.spiderflow.core.model.SpiderFlow
;
import
org.spiderflow.core.model.Task
;
import
org.spiderflow.core.service.SpiderFlowService
;
import
org.spiderflow.core.service.TaskService
;
import
org.spiderflow.model.JsonBean
;
import
org.spiderflow.model.SpiderOutput
;
import
org.springframework.beans.factory.annotation.Autowired
;
...
...
@@ -15,6 +19,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import
org.springframework.web.bind.annotation.RequestMapping
;
import
org.springframework.web.bind.annotation.RestController
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.Map
;
...
...
@@ -32,14 +37,75 @@ public class SpiderRestController {
@Value
(
"${spider.workspace}"
)
private
String
workspace
;
@Autowired
private
SpiderJob
spiderJob
;
@Autowired
private
TaskService
taskService
;
/**
* 异步运行
* @param id
* @return
*/
@RequestMapping
(
"/runAsync/{id}"
)
public
JsonBean
<
Integer
>
runAsync
(
@PathVariable
(
"id"
)
String
id
){
SpiderFlow
flow
=
spiderFlowService
.
getById
(
id
);
if
(
flow
==
null
){
return
new
JsonBean
<>(
0
,
"找不到此爬虫信息"
);
}
Task
task
=
new
Task
();
task
.
setFlowId
(
flow
.
getId
());
task
.
setBeginTime
(
new
Date
());
taskService
.
save
(
task
);
Spider
.
executorInstance
.
submit
(()->{
spiderJob
.
run
(
flow
,
task
,
null
);
});
return
new
JsonBean
<>(
task
.
getId
());
}
/**
* 停止运行任务
* @param taskId
*/
@RequestMapping
(
"/stop/{taskId}"
)
public
JsonBean
<
Void
>
stop
(
@PathVariable
(
"taskId"
)
Integer
taskId
){
SpiderContext
context
=
SpiderJob
.
getSpiderContext
(
taskId
);
if
(
context
==
null
){
return
new
JsonBean
<>(
0
,
"任务不存在!"
);
}
context
.
setRunning
(
false
);
return
new
JsonBean
<>(
1
,
"停止成功!"
);
}
/**
* 查询任务状态
* @param taskId
*/
@RequestMapping
(
"/status/{taskId}"
)
public
JsonBean
<
Integer
>
status
(
@PathVariable
(
"taskId"
)
Integer
taskId
){
SpiderContext
context
=
SpiderJob
.
getSpiderContext
(
taskId
);
if
(
context
==
null
){
return
new
JsonBean
<>(
0
);
//
}
return
new
JsonBean
<>(
1
);
//正在运行中
}
/**
* 同步运行
* @param id
* @param params
* @return
*/
@RequestMapping
(
"/run/{id}"
)
public
JsonBean
<
List
<
SpiderOutput
>>
run
(
@PathVariable
(
"id"
)
String
id
,
@RequestBody
(
required
=
false
)
Map
<
String
,
Object
>
params
){
SpiderFlow
flow
=
spiderFlowService
.
getById
(
id
);
if
(
flow
==
null
){
return
new
JsonBean
<>(
0
,
"找不到此爬虫信息"
);
}
List
<
SpiderOutput
>
outputs
=
null
;
List
<
SpiderOutput
>
outputs
;
Integer
maxId
=
spiderFlowService
.
getFlowMaxTaskId
(
id
);
SpiderJobContext
context
=
SpiderJobContext
.
create
(
workspace
,
id
,
maxId
,
true
);
try
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录