diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml deleted file mode 100644 index 276d75010..000000000 --- a/.github/workflows/nodejs.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Node CI - -on: [push] - -jobs: - build: - - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [8.x, 10.x, 12.x] - - steps: - - uses: actions/checkout@v1 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: build docs - run: | - npm ci - npm install -g gitbook-cli - gitbook build documentation docs - env: - CI: true diff --git a/backend/main.go b/backend/main.go index 92863a203..3f87125de 100644 --- a/backend/main.go +++ b/backend/main.go @@ -47,7 +47,7 @@ func main() { panic(err) } log.Info("初始化定期清理日志配置成功") - }else { + } else { log.Info("默认未开启定期清理日志配置") } @@ -154,6 +154,7 @@ func main() { authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 authGroup.PUT("/tasks", routes.PutTask) // 派发任务 authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 + authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) //删除指定状态的任务 authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 diff --git a/backend/model/task.go b/backend/model/task.go index 64f06cd71..588db6b38 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -158,6 +158,8 @@ func GetTask(id string) (Task, error) { return task, nil } + + func AddTask(item Task) error { s, c := database.GetCol("tasks") defer s.Close() @@ -187,6 +189,20 @@ func RemoveTask(id string) error { return nil } +func RemoveTaskByStatus(status string) error { + tasks, err := GetTaskList(bson.M{"status": status}, 0, constants.Infinite, "-create_ts") + if err != nil { + log.Error("get tasks error:" + err.Error()) + } + for _, task := range tasks { + if err := RemoveTask(task.Id); err != nil { + log.Error("remove task error:" + err.Error()) + continue + } + } + return nil +} + // 删除task by spider_id func RemoveTaskBySpiderId(id bson.ObjectId) error { tasks, err := GetTaskList(bson.M{"spider_id": id}, 0, constants.Infinite, "-create_ts") diff --git a/backend/routes/task.go b/backend/routes/task.go index 45a16675d..3d6edece1 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -17,6 +17,7 @@ type TaskListRequestData struct { PageSize int `form:"page_size"` NodeId string `form:"node_id"` SpiderId string `form:"spider_id"` + Status string `form:"status"` } type TaskResultsRequestData struct { @@ -46,6 +47,10 @@ func GetTaskList(c *gin.Context) { if data.SpiderId != "" { query["spider_id"] = bson.ObjectIdHex(data.SpiderId) } + //新增根据任务状态获取task列表 + if data.Status != "" { + query["status"] = data.Status + } // 获取任务列表 tasks, err := model.GetTaskList(query, (data.PageNum-1)*data.PageSize, data.PageSize, "-create_ts") @@ -156,6 +161,27 @@ func PutTask(c *gin.Context) { }) } +func DeleteTaskByStatus(c *gin.Context) { + status := c.Query("status") + + //删除相应的日志文件 + if err := services.RemoveLogByTaskStatus(status); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + //删除该状态下的task + if err := model.RemoveTaskByStatus(status); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + func DeleteTask(c *gin.Context) { id := c.Param("id") diff --git a/backend/services/log.go b/backend/services/log.go index 5b5cd7aed..60909c612 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -119,6 +119,18 @@ func RemoveLogByTaskId(id string) error { return nil } +func RemoveLogByTaskStatus(status string) error { + tasks, err := model.GetTaskList(bson.M{"status": status}, 0, constants.Infinite, "-create_ts") + if err != nil { + log.Error("get tasks error:" + err.Error()) + return err + } + for _, task := range tasks { + RemoveLogByTaskId(task.Id) + } + return nil +} + func removeLog(t model.Task) { if err := RemoveLocalLog(t.LogPath); err != nil { log.Errorf("remove local log error: %s", err.Error()) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index f419a36c0..5c7a9dc21 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -549,7 +549,8 @@ export default { } }) }, - onCrawl () { + async onCrawl () { + await this.onSave() this.crawlConfirmDialogVisible = true this.$st.sendEv('爬虫详情-配置', '点击运行') },