Merge pull request #352 from crawlab-team/develop

Develop

backend/main.go

@@ -47,7 +47,7 @@ func main() {
 			panic(err)
 		}
 		log.Info("初始化定期清理日志配置成功")
-	}else {
+	} else {
 		log.Info("默认未开启定期清理日志配置")
 	}
 
@@ -154,6 +154,7 @@ func main() {
 			authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
 			authGroup.PUT("/tasks", routes.PutTask)                                     // 派发任务
 			authGroup.DELETE("/tasks/:id", routes.DeleteTask)                           // 删除任务
+			authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus)             //删除指定状态的任务
 			authGroup.POST("/tasks/:id/cancel", routes.CancelTask)                      // 取消任务
 			authGroup.GET("/tasks/:id/log", routes.GetTaskLog)                          // 任务日志
 			authGroup.GET("/tasks/:id/results", routes.GetTaskResults)                  // 任务结果

backend/model/task.go

@@ -158,6 +158,8 @@ func GetTask(id string) (Task, error) {
 	return task, nil
 }
 
+
+
 func AddTask(item Task) error {
 	s, c := database.GetCol("tasks")
 	defer s.Close()
@@ -187,6 +189,20 @@ func RemoveTask(id string) error {
 	return nil
 }
 
+func RemoveTaskByStatus(status string) error {
+	tasks, err := GetTaskList(bson.M{"status": status}, 0, constants.Infinite, "-create_ts")
+	if err != nil {
+		log.Error("get tasks error:" + err.Error())
+	}
+	for _, task := range tasks {
+		if err := RemoveTask(task.Id); err != nil {
+			log.Error("remove task error:" + err.Error())
+			continue
+		}
+	}
+	return nil
+}
+
 // 删除task by spider_id
 func RemoveTaskBySpiderId(id bson.ObjectId) error {
 	tasks, err := GetTaskList(bson.M{"spider_id": id}, 0, constants.Infinite, "-create_ts")

backend/routes/task.go

@@ -17,6 +17,7 @@ type TaskListRequestData struct {
 	PageSize int    `form:"page_size"`
 	NodeId   string `form:"node_id"`
 	SpiderId string `form:"spider_id"`
+	Status   string `form:"status"`
 }
 
 type TaskResultsRequestData struct {
@@ -46,6 +47,10 @@ func GetTaskList(c *gin.Context) {
 	if data.SpiderId != "" {
 		query["spider_id"] = bson.ObjectIdHex(data.SpiderId)
 	}
+	//新增根据任务状态获取task列表
+	if data.Status != "" {
+		query["status"] = data.Status
+	}
 
 	// 获取任务列表
 	tasks, err := model.GetTaskList(query, (data.PageNum-1)*data.PageSize, data.PageSize, "-create_ts")
@@ -156,6 +161,27 @@ func PutTask(c *gin.Context) {
 	})
 }
 
+func DeleteTaskByStatus(c *gin.Context) {
+	status := c.Query("status")
+
+	//删除相应的日志文件
+	if err := services.RemoveLogByTaskStatus(status); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	//删除该状态下的task
+	if err := model.RemoveTaskByStatus(status); err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+	})
+}
+
 func DeleteTask(c *gin.Context) {
 	id := c.Param("id")
 

backend/services/log.go

@@ -119,6 +119,18 @@ func RemoveLogByTaskId(id string) error {
 	return nil
 }
 
+func RemoveLogByTaskStatus(status string) error {
+	tasks, err := model.GetTaskList(bson.M{"status": status}, 0, constants.Infinite, "-create_ts")
+	if err != nil {
+		log.Error("get tasks error:" + err.Error())
+		return err
+	}
+	for _, task := range tasks {
+		RemoveLogByTaskId(task.Id)
+	}
+	return nil
+}
+
 func removeLog(t model.Task) {
 	if err := RemoveLocalLog(t.LogPath); err != nil {
 		log.Errorf("remove local log error: %s", err.Error())

frontend/src/components/Config/ConfigList.vue

@@ -549,7 +549,8 @@ export default {
         }
       })
     },
-    onCrawl () {
+    async onCrawl () {
+      await this.onSave()
       this.crawlConfirmDialogVisible = true
       this.$st.sendEv('爬虫详情-配置', '点击运行')
     },