martin 3 سال پیش
والد
کامیت
8669e93ecf
14فایلهای تغییر یافته به همراه532 افزوده شده و 602 حذف شده
  1. 2 1
      api-common/src/main/java/api/common/pojo/constants/DictConstants.java
  2. 2 2
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/AutoProjectConsumer.java
  3. 33 295
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumer.java
  4. 2 2
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumerTest.java
  5. 6 6
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/manager/TaskIndexManager.java
  6. 4 4
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/IndexMapper.java
  7. 18 1
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/IndexTemplateMapper.java
  8. 2 2
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/ManualProjectMapper.java
  9. 1 0
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/IndexTemplatePO.java
  10. 1 1
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/LeafIndexPO.java
  11. 19 97
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/ProjectScheduler.java
  12. 238 9
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ManualProjectService.java
  13. 179 157
      simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java
  14. 25 25
      simulation-resource-scheduler/src/test/java/com/css/simulation/resource/scheduler/SchedulerTest.java

+ 2 - 1
api-common/src/main/java/api/common/pojo/constants/DictConstants.java

@@ -66,6 +66,7 @@ public class DictConstants {
     public static final String TASK_COMPLETED = "Completed"; // 任务执行状态,已完成
     public static final String TASK_TERMINATING = "Terminating"; // 任务执行状态,终止中
     public static final String TASK_TERMINATED = "Terminated"; // 任务执行状态,已终止
+    public static final String TASK_TIMEOUT = "Timeout"; // 非常用状态,任务超时
 
 
 
@@ -93,7 +94,7 @@ public class DictConstants {
 
     //任务失败原因
     public static final String TASK_ERROR_REASON_1 = "任务执行超时!";
-    public static final String TASK_ERROR_REASON_2 = "未知错误!";
+    public static final String TASK_ERROR_REASON_2 = "任务执行超时或未知错误!";
     public static final String TASK_ERROR_REASON_3 = "容器终止!";
     public static final String TASK_ERROR_REASON_4 = "未知状态!";
     public static final String TASK_ERROR_REASON_5 = "打分出错!";

+ 2 - 2
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/AutoProjectConsumer.java

@@ -55,7 +55,7 @@ public class AutoProjectConsumer {
     @Autowired
     TaskMapper taskMapper;
     @Autowired
-    TaskIndexMapper taskIndexMapper;
+    IndexMapper indexMapper;
     @Autowired
     IndexTemplateMapper indexTemplateMapper;
     @Autowired
@@ -162,7 +162,7 @@ public class AutoProjectConsumer {
             // 根据 projectId 和 sceneId 查询任务信息
             List<String> taskIdList = taskMapper.selectIdByProjectIdAndSceneId(subProjectId, sceneId);
             // 可能会存在多个指标下有同样的场景,所以会查出多个指标
-            List<String> lastTargetIdList = taskIndexMapper.selectLeafIndexIdByProjectAndSceneId(subProjectId, "%" + sceneId + "%");
+            List<String> lastTargetIdList = indexMapper.selectLeafIndexIdByProjectAndSceneId(subProjectId, "%" + sceneId + "%");
 
             if (CollectionUtil.isEmpty(taskIdList)) {    // 如果没有旧任务说明是创建任务
                 lastTargetIdList.forEach(lastTargetId -> {

تفاوت فایلی نمایش داده نمی شود زیرا این فایل بسیار بزرگ است
+ 33 - 295
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumer.java


+ 2 - 2
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumerTest.java

@@ -53,7 +53,7 @@ public class ManualProjectConsumerTest {
     @Autowired
     TaskMapper taskMapper;
     @Autowired
-    TaskIndexMapper taskIndexMapper;
+    IndexMapper indexMapper;
     @Autowired
     IndexTemplateMapper indexTemplateMapper;
     @Autowired
@@ -160,7 +160,7 @@ public class ManualProjectConsumerTest {
             // 根据 projectId 和 sceneId 查询任务信息
             List<String> taskIdList = taskMapper.selectIdByProjectIdAndSceneId(projectId, sceneId);
             // 可能会存在多个指标下有同样的场景,所以会查出多个指标
-            List<String> lastTargetIdList = taskIndexMapper.selectLeafIndexIdByProjectAndSceneId(projectId, "%" + sceneId + "%");
+            List<String> lastTargetIdList = indexMapper.selectLeafIndexIdByProjectAndSceneId(projectId, "%" + sceneId + "%");
 
             if (CollectionUtil.isEmpty(taskIdList)) {    // 如果没有旧任务说明是创建任务
                 lastTargetIdList.forEach(lastTargetId -> {

+ 6 - 6
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/manager/TaskIndexManager.java

@@ -1,7 +1,7 @@
 package com.css.simulation.resource.scheduler.manager;
 
-import com.css.simulation.resource.scheduler.mapper.TaskIndexMapper;
-import com.css.simulation.resource.scheduler.pojo.po.TaskIndexPO;
+import com.css.simulation.resource.scheduler.mapper.IndexMapper;
+import com.css.simulation.resource.scheduler.pojo.po.LeafIndexPO;
 import org.apache.ibatis.session.ExecutorType;
 import org.apache.ibatis.session.SqlSession;
 import org.apache.ibatis.session.SqlSessionFactory;
@@ -16,12 +16,12 @@ public class TaskIndexManager {
     @Autowired
     private SqlSessionFactory sqlSessionFactory;
 
-    public void batchInsertLeafIndex(List<TaskIndexPO> leafTaskIndexList) {
+    public void batchInsertLeafIndex(List<LeafIndexPO> leafTaskIndexList) {
 
         try(SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH,false)){
-            TaskIndexMapper taskIndexMapper = sqlSession.getMapper(TaskIndexMapper.class);
-            for (TaskIndexPO taskIndexPO : leafTaskIndexList) {
-                taskIndexMapper.insertLeafIndex(taskIndexPO);
+            IndexMapper indexMapper = sqlSession.getMapper(IndexMapper.class);
+            for (LeafIndexPO leafIndexPO : leafTaskIndexList) {
+                indexMapper.insertLeafIndex(leafIndexPO);
             }
             sqlSession.commit();
         }

+ 4 - 4
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/TaskIndexMapper.java → simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/IndexMapper.java

@@ -1,7 +1,7 @@
 package com.css.simulation.resource.scheduler.mapper;
 
 
-import com.css.simulation.resource.scheduler.pojo.po.TaskIndexPO;
+import com.css.simulation.resource.scheduler.pojo.po.LeafIndexPO;
 import org.apache.ibatis.annotations.*;
 
 import java.util.List;
@@ -12,7 +12,7 @@ import java.util.List;
  * simulation_mpt_last_target_score
  */
 @Mapper
-public interface TaskIndexMapper {
+public interface IndexMapper {
 
 
     @Insert("insert into simulation_mpt_last_target_score(id,\n" +
@@ -37,7 +37,7 @@ public interface TaskIndexMapper {
             "        #{leaf.modifyTime},\n" +
             "        #{leaf.modifyUserId},\n" +
             "        #{leaf.isDeleted})")
-    void insertLeafIndex(@Param("leaf") TaskIndexPO taskIndexPO);
+    void insertLeafIndex(@Param("leaf") LeafIndexPO leafIndexPO);
 
     @Insert("insert into simulation_mpt_first_target_score(id,\n" +
             "                                             p_id,\n" +
@@ -57,7 +57,7 @@ public interface TaskIndexMapper {
             "        #{first.modifyTime},\n" +
             "        #{first.modifyUserId},\n" +
             "        #{first.isDeleted})")
-    void insertFirstIndex(@Param("first") TaskIndexPO firstTaskIndex);
+    void insertFirstIndex(@Param("first") LeafIndexPO firstTaskIndex);
 
     @Select("select sublist_id\n" +
             "from scene_package_sublist\n" +

+ 18 - 1
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/IndexTemplateMapper.java

@@ -25,8 +25,25 @@ public interface IndexTemplateMapper {
             @Result(column = "root_id", property = "rootId", jdbcType = JdbcType.VARCHAR),
             @Result(column = "package_level", property = "packageLevel", jdbcType = JdbcType.VARCHAR),
             @Result(column = "rule_name", property = "ruleName", jdbcType = JdbcType.VARCHAR),
-            @Result(column = "rule_details", property = "ruleDetails", jdbcType = JdbcType.VARCHAR)
+            @Result(column = "rule_details", property = "ruleDetails", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "package_and_rules", property = "ruleId", jdbcType = JdbcType.VARCHAR),
     })
+    @Select("select sublist_id,\n" +
+            "       scene_natural_ids,\n" +
+            "       scene_traffic_ids,\n" +
+            "       scene_statue_ids,\n" +
+            "       weight,\n" +
+            "       parent_id,\n" +
+            "       root_id,\n" +
+            "       package_level,\n" +
+            "       rule_name,\n" +
+            "       package_and_rules\n" +
+            "from scene_package_sublist\n" +
+            "where root_id = #{packageId}")
+    List<IndexTemplatePO> selectByPackageIdIncludeDeleted(@Param("packageId") String packageId);
+
+
+
     @Select("select scene_natural_ids,\n" +
             "       scene_traffic_ids,\n" +
             "       scene_statue_ids\n" +

+ 2 - 2
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/ManualProjectMapper.java

@@ -19,8 +19,8 @@ public interface ManualProjectMapper {
 
     @Select("select id, scene\n" +
             "from simulation_manual_project\n" +
-            "where id = (select p_id from simulation_manual_project_task where id = #{taskId})\n")
-    ProjectPO selectById(@Param("taskId")String taskId);
+            "where id = #{projectId}")
+    ProjectPO selectById(@Param("projectId")String projectId);
 
     @Update("update simulation_manual_project\n" +
             "set now_run_state  = #{state},\n" +

+ 1 - 0
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/IndexTemplatePO.java

@@ -22,6 +22,7 @@ public class IndexTemplatePO {
     private String ruleDetails; // 打分规则代码
     private Double tempScore;
     private Integer packageLevel; // 指标等级
+    private Integer ruleId; // 指标等级
 
 
 }

+ 1 - 1
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/TaskIndexPO.java → simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/LeafIndexPO.java

@@ -8,7 +8,7 @@ import lombok.*;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-public class TaskIndexPO extends CommonPO {
+public class LeafIndexPO extends CommonPO {
 
     private String id;
     private String pId; // 项目 id

+ 19 - 97
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/ProjectScheduler.java

@@ -9,9 +9,7 @@ import com.css.simulation.resource.scheduler.mapper.ManualProjectMapper;
 import com.css.simulation.resource.scheduler.mapper.TaskMapper;
 import com.css.simulation.resource.scheduler.pojo.po.TaskPO;
 import com.css.simulation.resource.scheduler.service.TaskService;
-import com.css.simulation.resource.scheduler.util.KubernetesUtil;
 import io.kubernetes.client.openapi.ApiClient;
-import io.kubernetes.client.openapi.ApiException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.sshd.client.SshClient;
 import org.apache.sshd.client.session.ClientSession;
@@ -23,8 +21,9 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
 
 @Component
 @Slf4j
@@ -34,133 +33,56 @@ public class ProjectScheduler {
     String manualProjectTopic;
     @Autowired
     StringRedisTemplate redisTemplate;
-
     @Autowired
     TaskService taskService;
-
     @Autowired
     TaskMapper taskMapper;
     @Autowired
     ManualProjectMapper projectMapper;
     @Value("${scheduler.manual-project.job-yaml}")
     String jobYaml;
-
     @Value("${scheduler.score.hostname}")
-    String hostnameScore;
+    String hostname;
     @Value("${scheduler.score.username}")
-    String usernameScore;
+    String username;
     @Value("${scheduler.score.password}")
-    String passwordScore;
-
+    String password;
     @Autowired
     ApiClient apiClient;
-
     @Autowired
     KafkaTemplate<String, String> kafkaTemplate;
 
 
     /**
-     * 解决 pod 莫名奇妙关闭的问题
+     * 处理 pod 超时
+     * 同时也可处理 pod 莫名关闭
      *
-     * @throws ApiException 异常
+     * @throws IOException 超时时间
      */
     @Scheduled(fixedDelay = 60 * 1000)
-    public void retry() throws ApiException {
-
-
-        log.info("------- ProjectScheduler--retry 检查 pod 是否需要重试!");
-        //1 从 redis 获取 手动运行项目的 key 列表
-        Set<String> keys = redisTemplate.keys("manualProject:*");
-
-        //2 根据 key 列表从 redis 获取 pod 列表
-        assert keys != null;
-        List<String> podKeyList = keys.stream().filter(key -> key.contains("pod")).collect(Collectors.toList());
-        Map<String, String> podNameMapShouldBe = new HashMap<>();
-        podKeyList.forEach(podKey -> {
-            String podName = redisTemplate.opsForValue().get(podKey);
-            podNameMapShouldBe.put(podKey, podName);
-        });
-        //3 通过 kubernetes 获取 pod 列表
-        List<String> podNameListReally = KubernetesUtil.getPod(apiClient, "all");
-        //4 比对 redis 中的 pod 列表 和 kubernetes 中的 pod 列表,如果有 redis 中存在但 kubernetes 中不存在则准备重试
-        podNameMapShouldBe.forEach((podKeyShouldBe, podNameShouldBe) -> {
-            if (!podNameListReally.contains(podNameShouldBe)) {
-
-                //4-1 根据 podKey 获取 projectId 和 taskId
-                String[] split = podKeyShouldBe.split(":");
-                String projectId = split[1];
-                String taskId = split[2];
-                //4-2 根据 projectId 和 taskId 列表从 redis 获取重试次数 retry
-                String retry = redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":retry");
-                assert retry != null;
-                int retryNumber = Integer.parseInt(retry);
-                if (retryNumber < 3) {
-                    log.info("------- ProjectScheduler--retry 准备第" + retryNumber + "次重试!");
-                    String taskJson = redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":message");
-                    int finalRetryNumber = retryNumber + 1;
-                    redisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId + ":retry", finalRetryNumber + "");
-                    kafkaTemplate.send(projectId, taskJson).addCallback(success -> {
-                        // 消息发送到的topic
-                        assert success != null;
-                        String topic = success.getRecordMetadata().topic();
-                        // 消息发送到的分区
-                        int partition = success.getRecordMetadata().partition();
-                        // 消息在分区内的offset
-                        long offset = success.getRecordMetadata().offset();
-                        log.info("------- ProjectScheduler--retry 项目 " + projectId + "的任务准备第" + finalRetryNumber + "次重试,"
-                                + "发送消息成功:\n"
-                                + "主题 topic 为:" + topic + "\n"
-                                + "分区 partition 为:" + partition + "\n"
-                                + "偏移量为:" + offset + "\n"
-                                + "消息体为:" + taskJson);
-                    }, failure -> {
-                        log.error("------- ProjectScheduler--retry 发送消息失败:" + failure.getMessage());
-                    });
-                }
-            }
-        });
-
-    }
+    public void timeout() throws IOException {
 
-    @Scheduled(fixedDelay = 60 * 1000)
-    public void tick() throws IOException {
+        long timeout = 2 * 60 * 1000L;
 
         SshClient client = SshUtil.getClient();
-        ClientSession session = SshUtil.getSession(client, hostnameScore, usernameScore, passwordScore);
-
+        ClientSession session = SshUtil.getSession(client, hostname, username, password);
         ArrayList<TaskPO> executingTaskList = taskMapper.selectExecuting();
         if (CollectionUtil.isEmpty(executingTaskList)) {
-            //        log.info("------- ProjectScheduler 查询出所有执行中的任务('Running'):" + executingTaskList);
             //2 根据 key 查出任务的心跳时间
-            for (TaskPO task : executingTaskList) {
+            executingTaskList.forEach(task -> {
                 String taskId = task.getId();
                 String projectId = task.getPId();
-                String s = redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":tick");
-//                Optional.ofNullable(s).orElseThrow(() -> new RuntimeException("项目 " + projectId + " 下的任务 " + taskId + " 的心跳查询失败"));
-                assert s != null;
-                long tickTime = Long.parseLong(s);
-                long timeout = 2 * 60 * 1000L;
-                long now = TimeUtil.getNow();
-                long difference = now - tickTime;
-//                log.info("------- ProjectScheduler 任务" + taskId + "心跳时间为:" + tickTime + "最大仿真时间为:" + tickTime + "时间差为:" + difference);
-                if (difference > timeout) {
+                long lastTickTime = Long.parseLong(Objects.requireNonNull(redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":tick")));
+                if (TimeUtil.getNow() - lastTickTime > timeout) {
                     String podName = redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":pod");
-                    String podDeleteCommand = "kubectl delete pod " + podName;
-                    if (podName != null) {
-                        log.info("ProjectScheduler--tick 修改任务 " + taskId + "已超时,状态修改为 Aborted,pod 名称为:" + podName
-                                + ",并执行删除 pod 命令:" + podDeleteCommand);
-                        SshUtil.execute(session, podDeleteCommand);
-//            taskManager.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql()); // 如果任务 abort 代表项目失败
-                        taskMapper.updateFailStateWithStopTime(taskId, DictConstants.TASK_ABORTED, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_1);
-                    }
+                    taskService.taskState(taskId, DictConstants.TASK_ABORTED, podName);
                 }
-            }
+            });
+
         }
         session.close();
         client.stop();
     }
-
-
     /**
      * 解决 pod 莫名全部关闭但是 job 还在的问题
      * 检查如果有 job 在运行但是 pod 全部关闭的情况,此时需要重启一下 job
@@ -168,7 +90,7 @@ public class ProjectScheduler {
     @Scheduled(fixedDelay = 30 * 1000)
     public void checkProject() throws IOException {
         SshClient client = SshUtil.getClient();
-        ClientSession session = SshUtil.getSession(client, hostnameScore, usernameScore, passwordScore);
+        ClientSession session = SshUtil.getSession(client, hostname, username, password);
 
         //1 查询出正在运行中的 project
         List<String> projectIdList = projectMapper.selectIdByState("20");

تفاوت فایلی نمایش داده نمی شود زیرا این فایل بسیار بزرگ است
+ 238 - 9
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ManualProjectService.java


+ 179 - 157
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java

@@ -4,13 +4,13 @@ import api.common.pojo.constants.DictConstants;
 import api.common.util.*;
 import com.css.simulation.resource.scheduler.manager.TaskIndexManager;
 import com.css.simulation.resource.scheduler.manager.TaskManager;
+import com.css.simulation.resource.scheduler.mapper.IndexMapper;
 import com.css.simulation.resource.scheduler.mapper.IndexTemplateMapper;
 import com.css.simulation.resource.scheduler.mapper.ManualProjectMapper;
-import com.css.simulation.resource.scheduler.mapper.TaskIndexMapper;
 import com.css.simulation.resource.scheduler.mapper.TaskMapper;
 import com.css.simulation.resource.scheduler.pojo.po.IndexTemplatePO;
+import com.css.simulation.resource.scheduler.pojo.po.LeafIndexPO;
 import com.css.simulation.resource.scheduler.pojo.po.ProjectPO;
-import com.css.simulation.resource.scheduler.pojo.po.TaskIndexPO;
 import com.css.simulation.resource.scheduler.pojo.po.TaskPO;
 import com.css.simulation.resource.scheduler.pojo.to.ScoreTO;
 import com.css.simulation.resource.scheduler.util.MinioUtil;
@@ -26,6 +26,7 @@ import org.apache.sshd.client.session.ClientSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
@@ -49,7 +50,7 @@ public class TaskService {
     @Value("${minio.bucket-name}")
     String bucketName;
     @Autowired
-    StringRedisTemplate redisTemplate;
+    StringRedisTemplate stringRedisTemplate;
     @Autowired
     ManualProjectMapper manualProjectMapper;
     //    @Autowired
@@ -61,7 +62,7 @@ public class TaskService {
     @Autowired
     TaskManager taskManager;
     @Autowired
-    TaskIndexMapper taskIndexMapper;
+    IndexMapper indexMapper;
     @Autowired
     IndexTemplateMapper indexTemplateMapper;
     @Value("${scheduler.manual-project.topic}")
@@ -69,18 +70,11 @@ public class TaskService {
     @Value("${scheduler.manual-project.result-path-minio}")
     String resultPathMinio;
     @Value("${scheduler.score.hostname}")
-    String hostnameScore;
+    String hostname;
     @Value("${scheduler.score.username}")
-    String usernameScore;
+    String username;
     @Value("${scheduler.score.password}")
-    String passwordScore;
-
-    @Value("${spring.kafka.hostname}")
-    String hostnameKafka;
-    @Value("${spring.kafka.username}")
-    String usernameKafka;
-    @Value("${spring.kafka.password}")
-    String passwordKafka;
+    String password;
     @Value("${spring.kafka.delete-command}")
     String kafkaDeleteCommand;
     @Value("${scheduler.score.py-path}")
@@ -95,74 +89,60 @@ public class TaskService {
     String tokenUri;
     @Value("${simulation-cloud.evaluation-level-uri}")
     String evaluationLevelUri;
+    @Autowired
+    KafkaTemplate<String, String> kafkaTemplate;
 
 
     @SneakyThrows
     public void taskState(String taskId, String state, String podName) {
-
         String projectId = taskMapper.selectProjectIdById(taskId);
-        redisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId + ":pod", podName);
-        String podDeleteCommand = "kubectl delete pod " + podName;
         SshClient client = SshUtil.getClient();
-        ClientSession session = SshUtil.getSession(client, hostnameScore, usernameScore, passwordScore);
+        ClientSession session = SshUtil.getSession(client, hostname, username, password);
         if ("Running".equals(state)) {
+            // 将运行中的任务的 pod 名称放入 redis
+            stringRedisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId + ":pod", podName);
             log.info("TaskService--state 修改任务 " + taskId + "的状态为 Running,pod 名称为:" + podName);
             taskMapper.updateStateWithStartTime(taskId, state, TimeUtil.getNowForMysql());
-        } else if ("Aborted".equals(state)) {
-            log.info("TaskService--state 修改任务 " + taskId + "的状态为 Aborted,pod 名称为:" + podName
-                    + ",并执行删除 pod 命令:" + podDeleteCommand);
-            SshUtil.execute(session, podDeleteCommand);
-//            taskManager.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql()); // 如果任务 abort 代表项目失败
-            //result-path-minio: /project/manual-project/
-            String minioPathOfErrorLog = resultPathMinio + projectId + "/" + taskId + "error.log";
-            boolean objectExist = MinioUtil.isObjectExist(minioClient, bucketName, minioPathOfErrorLog);
-            String targetEvaluate;
-            if (objectExist) {
-                String errorString = MinioUtil.downloadToString(minioClient, bucketName, minioPathOfErrorLog);
-                String[] lines = errorString.split("\n");
-                StringBuilder errorMessage = new StringBuilder();
-                for (String line : lines) {
-                    if (line.startsWith("Original Error")) {
-                        errorMessage.append(line).append("\n");
-                    }
-
-                    if (line.startsWith("Possible Cause")) {
-                        errorMessage.append(line);
-                        break;
+            return;
+        } else {
+            String podDeleteCommand = "kubectl delete pod " + podName;
+            log.info("TaskService--state 修改任务 " + taskId + "的状态为:" + state + ",pod 名称为:" + podName + ",并执行删除 pod 命令:" + podDeleteCommand);
+            if ("Aborted".equals(state)) {
+                if (retry(projectId, taskId)) {
+                    taskMapper.updateStateById(DictConstants.TASK_RUNNING, taskId);
+                    return;
+                }
+                //result-path-minio: /project/manual-project/
+                String minioPathOfErrorLog = resultPathMinio + projectId + "/" + taskId + "error.log";
+                boolean objectExist = MinioUtil.isObjectExist(minioClient, bucketName, minioPathOfErrorLog);
+                String targetEvaluate;
+                if (objectExist) {
+                    String errorString = MinioUtil.downloadToString(minioClient, bucketName, minioPathOfErrorLog);
+                    String[] lines = errorString.split("\n");
+                    StringBuilder errorMessage = new StringBuilder();
+                    for (String line : lines) {
+                        if (line.startsWith("Original Error")) {
+                            errorMessage.append(line).append("\n");
+                        }
+                        if (line.startsWith("Possible Cause")) {
+                            errorMessage.append(line);
+                            break;
+                        }
                     }
+                    targetEvaluate = errorMessage.toString();
+                } else {
+                    targetEvaluate = DictConstants.TASK_ERROR_REASON_2;
                 }
-                targetEvaluate = errorMessage.toString();
+                taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), targetEvaluate);
+            } else if ("Terminated".equals(state)) {
+                taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_3);
+            } else if ("PendingAnalysis".equals(state)) {
+                taskMapper.updateSuccessStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
             } else {
-                targetEvaluate = DictConstants.TASK_ERROR_REASON_2;
+                taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_4);
             }
-
-
-            taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), targetEvaluate);
-        } else if ("Terminated".equals(state)) {
-            log.info("TaskService--state 修改任务 " + taskId + "的状态为 Terminated,pod 名称为:" + podName
-                    + ",并执行删除 pod 命令:" + podDeleteCommand);
             SshUtil.execute(session, podDeleteCommand);
-            taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_3);
-        } else if ("PendingAnalysis".equals(state)) {
-            log.info("TaskService--state 修改任务 " + taskId + "的状态为 PendingAnalysis,pod 名称为:" + podName
-                    + ",并执行删除 pod 命令:" + podDeleteCommand);
-            SshUtil.execute(session, podDeleteCommand);
-            taskMapper.updateSuccessStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
-        } else {
-            log.error("TaskService--state 出现了未知状态:" + state);
-            taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_4);
         }
-        ProjectPO projectPO = manualProjectMapper.selectById(taskId);
-        if (projectPO == null) {
-            session.close();
-            client.stop();
-            return;
-        }
-        Set<String> keys = redisTemplate.keys("manualProject:" + projectId + "*");
-        assert keys != null;
-        redisTemplate.delete(keys);
-        String scenePackageId = projectPO.getScenePackageId();  // 场景测试包 id,指标根 id
-//        log.info("------- /state 任务 " + taskId + " 的父项目为:" + projectId);
         int taskNum = manualProjectMapper.selectTaskNumById(projectId);
         int endTaskNum = manualProjectMapper.selectEndTaskNum(projectId);    // 查询已结束的任务 'Aborted', 'PendingAnalysis', 'Terminated'
         manualProjectMapper.updateTaskCompleted(projectId, endTaskNum);
@@ -173,65 +153,52 @@ public class TaskService {
             return;
         }
 
-        log.info("结束项目的 job");
-        SshUtil.execute(session, "kubectl delete job project-" + projectId);
-
-        // 删除 kafka topic
-//        SshClient clientKafka = SshUtil.getClient();
-//        ClientSession sessionKafka = SshUtil.getSession(clientKafka, hostnameKafka, usernameKafka, passwordKafka);
-//        String topicDeleteCommand = StringUtil.replace(kafkaDeleteCommand, "topicName", projectId);
-//        SshUtil.execute(sessionKafka, topicDeleteCommand);
-//        SshUtil.stop(clientKafka, sessionKafka);
-
+        // -------------------------------- 打分 --------------------------------
+        log.info("TaskService--state 项目 " + projectId + "准备打分!");
+        ProjectPO projectPO = manualProjectMapper.selectById(projectId);
+        String packageId = projectPO.getScenePackageId();  // 场景测试包 id,指标根 id
         List<TaskPO> taskList = taskMapper.selectTaskListByProjectId(projectId);  // 所有任务信息
-        int taskNumber = taskList.size();
-        log.info("TaskService--state 共有 " + taskNumber + "个任务!");
-        taskIndexMapper.deleteFirstByProjectId(projectId);
-        taskIndexMapper.deleteLastByProjectId(projectId);
-        // -------------------------------- 查询叶子指标 --------------------------------
-        List<IndexTemplatePO> leafIndexTemplateList = indexTemplateMapper.selectLeafIndexWithRuleDetailsByPackageId(scenePackageId);
+        indexMapper.deleteFirstByProjectId(projectId);
+        indexMapper.deleteLastByProjectId(projectId);
+        //1 查询场景包对应指标
+        List<IndexTemplatePO> allIndexList = JsonUtil.jsonToList(stringRedisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + packageId + ":all"), IndexTemplatePO.class);
+        List<IndexTemplatePO> leafIndexList = JsonUtil.jsonToList(stringRedisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + packageId + ":leaf"), IndexTemplatePO.class);
         int maxLevel = 1; // 用于计算指标得分
-        List<TaskIndexPO> leafTaskIndexList = new ArrayList<>();
-        log.info("TaskService--state 共有 " + leafIndexTemplateList.size() + "个叶子节点!");
+        List<LeafIndexPO> leafTaskIndexList = new ArrayList<>();
+        log.info("TaskService--state 共有 " + leafIndexList.size() + "个叶子节点!");
 
-//        SshClient clientScore = SshUtil.getClient();
-//        ClientSession sessionScore = SshUtil.getSession(clientScore, hostnameScore, usernameScore, passwordScore);
-        for (int i = 0; i < leafIndexTemplateList.size(); i++) {
+        for (int i = 0; i < leafIndexList.size(); i++) {
             AtomicReference<String> scoreExplain = new AtomicReference<>(); // 每个叶子指标下的任务的得分说明一样和叶子指标一致
-            IndexTemplatePO indexTemplatePO = leafIndexTemplateList.get(i);
-            String indexId = indexTemplatePO.getIndexId();
-            String parentId = indexTemplatePO.getParentId();
-            String rootId = indexTemplatePO.getRootId();
-            String weight = indexTemplatePO.getWeight();
-            Integer packageLevel = indexTemplatePO.getPackageLevel();
+            IndexTemplatePO leafIndex = leafIndexList.get(i);
+            String indexId = leafIndex.getIndexId();
+            String parentId = leafIndex.getParentId(); // 父 id
+            String rootId = leafIndex.getRootId(); // 包 id
+            String weight = leafIndex.getWeight(); // 权重
+            Integer packageLevel = leafIndex.getPackageLevel(); // 几级指标
+            String ruleName = leafIndex.getRuleName();    // 打分脚本名称,例如 AEB_1-1
+            String ruleDetails = leafIndex.getRuleDetails();    // 打分脚本内容
             if (packageLevel > maxLevel) {
                 maxLevel = packageLevel;
             }
             log.info("TaskService--state 开始执行对第 " + (i + 1) + " 个叶子节点 " + indexId + " 进行打分!");
-            String ruleName = indexTemplatePO.getRuleName();    // 打分脚本名称,例如 AEB_1-1
-            String ruleDetails = indexTemplatePO.getRuleDetails();    // 打分脚本内容
+
             String ruleFilePath = pyPath + "scripts/" + ruleName.split("_")[0] + "/" + ruleName + ".py";
             log.info("TaskService--state 将叶子节点 " + indexId + " 对应的打分规则保存到临时目录:" + ruleFilePath);
             FileUtil.writeInputStreamToLocalFile(IoUtil.stringToInputStream(ruleDetails), ruleFilePath);
-
             List<TaskPO> taskListOfLeafIndex = taskList.stream()
                     .filter(task -> indexId.equals(task.getLastTargetId()))
                     .collect(Collectors.toList());
-
             log.info("TaskService--state 叶子节点 " + indexId + " 包括 " + taskListOfLeafIndex.size() + " 个成功运行结束任务!");
             log.info("TaskService--state 计算叶子节点 " + indexId + " 的得分:" + taskListOfLeafIndex);
-            taskListOfLeafIndex.forEach(task2 -> {
-                String runState = task2.getRunState();
+            taskListOfLeafIndex.forEach(taskOfLeaf -> {
+                String runState = taskOfLeaf.getRunState();
                 if (DictConstants.TASK_ANALYSIS.equals(runState)) {
-                    String task2Id = task2.getId();
+                    String task2Id = taskOfLeaf.getId();
                     taskMapper.updateSuccessStateWithStopTime(task2Id, DictConstants.TASK_ANALYSING, TimeUtil.getNowForMysql());
                     // 计算每个任务的得分
-                    String runResultMinio = task2.getRunResultFilePath() + "/Ego.csv";
+                    String runResultMinio = taskOfLeaf.getRunResultFilePath() + "/Ego.csv";
                     String runResultLinux = linuxTempPath + runResultMinio;
-
-//                        python3 /home/ubuntu/test/Evaluate/main.py /home/ubuntu/test/test_data.csv 4 AEB_1-2
-//                        String command = "python3 " + pyPath + " " + runResultLinux + " " + task2.getSceneType();  // 默认使用场景名称找打分脚本
-                    String scoreCommand = "python3 " + pyPath + "main.py " + runResultLinux + " " + task2.getSceneType() + " " + ruleName; // 指定打分脚本
+                    String scoreCommand = "python3 " + pyPath + "main.py " + runResultLinux + " " + taskOfLeaf.getSceneType() + " " + ruleName; // 指定打分脚本
                     String scoreResult;
                     ScoreTO score;
                     try {
@@ -252,24 +219,23 @@ public class TaskService {
                             throw new RuntimeException("------- TaskService--state 项目" + projectId + "的任务" + task2Id + " 打分出错,命令为:" + scoreCommand + " 修改状态为:" + DictConstants.TASK_ABORTED + "\n" + e.getMessage());
                         }
                     } catch (Exception e) {
-                        task2.setRunState(DictConstants.TASK_ABORTED);
+                        taskOfLeaf.setRunState(DictConstants.TASK_ABORTED);
                         taskMapper.updateFailStateWithStopTime(task2Id, DictConstants.TASK_ABORTED, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_5);
                         log.error(e.getMessage());
                         return; // 如果打分失败则开始下一个打分
                     }
                     assert score != null;
-                    task2.setReturnSceneId(score.getUnit_scene_ID());
-//                    task2.setScore(new Random().nextInt(10) * 10.0);
-                    task2.setScore(score.getUnit_scene_score());
-                    task2.setTargetEvaluate(score.getEvaluate_item());
-                    task2.setScoreExplain(score.getScore_description());
-                    task2.setModifyUserId(USER_ID);
-                    task2.setModifyTime(TimeUtil.getNowForMysql());
+                    taskOfLeaf.setReturnSceneId(score.getUnit_scene_ID());
+                    taskOfLeaf.setScore(score.getUnit_scene_score());
+                    taskOfLeaf.setTargetEvaluate(score.getEvaluate_item());
+                    taskOfLeaf.setScoreExplain(score.getScore_description());
+                    taskOfLeaf.setModifyUserId(USER_ID);
+                    taskOfLeaf.setModifyTime(TimeUtil.getNowForMysql());
                     scoreExplain.set(score.getScore_description());
 
-                    task2.setRunState(DictConstants.TASK_COMPLETED);
+                    taskOfLeaf.setRunState(DictConstants.TASK_COMPLETED);
                     taskMapper.updateSuccessStateAndScoreResultWithStopTime(
-                            task2,
+                            taskOfLeaf,
                             DictConstants.TASK_COMPLETED,
                             TimeUtil.getNowForMysql()
                     );
@@ -282,21 +248,22 @@ public class TaskService {
                     .filter(task -> task.getScore() < 100)
                     .count();
 
-            // 计算总分
+            // 计算叶子指标下任务得分总和
             double leafSum = taskListOfLeafIndex.stream()
                     .mapToDouble(TaskPO::getScore)
                     .sum();
             // 计算任务的个数
             long resultNumberOfCurrentIndex = taskListOfLeafIndex.size();
             log.info("TaskService--state 项目 " + projectId + " 的叶子指标" + indexId + "下成功执行的场景数量为:" + resultNumberOfCurrentIndex);
+            // 计算叶子指标得分(任务得分总和 / 任务数量)
             double leafIndexScore = resultNumberOfCurrentIndex == 0 ? 0 : NumberUtil.cut(leafSum / resultNumberOfCurrentIndex, 2);
-            // -------------------------------- 保存叶子指标得分 --------------------------------
-            indexTemplatePO.setTempScore(leafIndexScore);
+            // 创建叶子指标对象
+            leafIndex.setTempScore(leafIndexScore);
 
-            TaskIndexPO leafTaskIndex = TaskIndexPO.builder()
+            LeafIndexPO leafTaskIndex = LeafIndexPO.builder()
                     .id(StringUtil.getRandomUUID())
                     .pId(projectId)
-                    .target(indexTemplatePO.getIndexId())
+                    .target(leafIndex.getIndexId())
                     .notStandardSceneNum((int) notStandardSceneNumber)
                     .score(leafIndexScore)
                     .indexId(indexId)
@@ -311,80 +278,116 @@ public class TaskService {
             leafTaskIndex.setModifyUserId(USER_ID);
             leafTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
             leafTaskIndex.setIsDeleted("0");
+
             leafTaskIndexList.add(leafTaskIndex);
         }
-//        SshUtil.stop(clientScore, sessionScore);
-//        // 保存任务分数
-//        taskManager.batchUpdateByScoreResult(taskList);
-        // 保存末级指标分数
+
+        // 保存叶子指标得分
         taskIndexManager.batchInsertLeafIndex(leafTaskIndexList);
         // 保存一级指标分数
         log.info("TaskService--state 项目 " + projectId + " 的所有任务分数为:" + taskList);
         log.info("TaskService--state 根据每个指标的得分和权重算出各个一级指标的得分(即 project 对应的场景测试包下的一级指标)!");
-        computeFirst(leafTaskIndexList, projectId, maxLevel);
+        computeFirst(leafTaskIndexList, allIndexList, projectId, maxLevel);
 
         // 调用 server 的接口,计算评价等级
-        String tokenUrl = tokenUri + "?grant_type=client_credentials"
-                + "&client_id=" + clientId
-                + "&client_secret=" + clientSecret;
-        log.info("TaskService--state 获取仿真云平台 token:" + tokenUrl);
-        String response = HttpUtil.get(closeableHttpClient, requestConfig, tokenUrl);
-        ObjectMapper objectMapper = new ObjectMapper();
-        JsonNode jsonNode = objectMapper.readTree(response);
-        String accessToken = jsonNode.path("access_token").asText();
-        log.info("TaskService--state 仿真云平台 token 为:" + accessToken);
-        Map<String, String> headers = new HashMap<>();
-        headers.put("Authorization", "Bearer " + accessToken);
-        Map<String, String> params = new HashMap<>();
-        params.put("id", projectId);
-        String post = HttpUtil.post(closeableHttpClient, requestConfig, evaluationLevelUri, headers, params);
-        log.info("TaskService--state 访问仿真云平台评价等级接口:" + evaluationLevelUri + ",请求头为:" + headers + ",请求体为:" + params + "结果为:" + post);
+        evaluationLevel(projectId);
         log.info("TaskService--state 项目 " + projectId + " 打分完成!");
         manualProjectMapper.updateProjectState(projectId, DictConstants.PROJECT_COMPLETED, TimeUtil.getNowForMysql());   // 修改该 project 的状态为已完成
         log.info("TaskService--state 项目 " + projectId + " 执行完成!");
+
+
+        // -------------------------------- 收尾 --------------------------------
+
+        // 删除所有 key
+//        Set<String> keys = redisTemplate.keys("manualProject:" + projectId + "*");
+//        assert keys != null;
+//        redisTemplate.delete(keys);
+//        log.info("------- /state 任务 " + taskId + " 的父项目为:" + projectId);
+
+
+        // 删除 kafka topic
+//        SshClient clientKafka = SshUtil.getClient();
+//        ClientSession sessionKafka = SshUtil.getSession(clientKafka, hostnameKafka, usernameKafka, passwordKafka);
+//        String topicDeleteCommand = StringUtil.replace(kafkaDeleteCommand, "topicName", projectId);
+//        SshUtil.execute(sessionKafka, topicDeleteCommand);
+//        SshUtil.stop(clientKafka, sessionKafka);
+
+
+        // 删除 job
+        SshUtil.execute(session, "kubectl delete job project-" + projectId);
         session.close();
         client.stop();
 
     }
 
 
-    public void computeFirst(List<TaskIndexPO> leafTaskIndexList, String projectId, int maxLevel) {
+    public boolean retry(String projectId, String taskId) {
+        //1 首先查看任务是否重试过 3 次
+        int retry = Integer.parseInt(Objects.requireNonNull(stringRedisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":retry")));
+        //2 如果重试次数没有超过 3 次,则重试
+        if (retry > 3) {
+            return false;
+        }
+        String taskJson = stringRedisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId + ":message");
+        kafkaTemplate.send(projectId, taskJson).addCallback(success -> {
+            // 消息发送到的topic
+            assert success != null;
+            String topic = success.getRecordMetadata().topic();
+            // 消息发送到的分区
+            int partition = success.getRecordMetadata().partition();
+            // 消息在分区内的offset
+            long offset = success.getRecordMetadata().offset();
+            log.info("------- ManualProjectConsumer 发送消息成功:\n"
+                    + "主题 topic 为:" + topic + "\n"
+                    + "分区 partition 为:" + partition + "\n"
+                    + "偏移量为:" + offset + "\n"
+                    + "消息体为:" + taskJson);
+        }, failure -> {
+            log.error("------- 发送消息失败:" + failure.getMessage());
+        });
+        return true;
+    }
+
 
-        log.info("------- /state computeFirst 计算父指标得分:" + leafTaskIndexList);
-        Iterator<TaskIndexPO> leafTaskIndexIterator = leafTaskIndexList.iterator();
+    public void computeFirst(List<LeafIndexPO> leafIndexList, List<IndexTemplatePO> allIndexTemplateList, String projectId, int maxLevel) {
+
+        log.info("------- /state computeFirst 计算父指标得分:" + leafIndexList);
+        Iterator<LeafIndexPO> leafTaskIndexIterator = leafIndexList.iterator();
         // 把 1 级的指标得分直接保存
         while (leafTaskIndexIterator.hasNext()) {
-            TaskIndexPO leafTaskIndex = leafTaskIndexIterator.next();
+            LeafIndexPO leafTaskIndex = leafTaskIndexIterator.next();
             if (leafTaskIndex.getPackageLevel() == 1) {
                 leafTaskIndex.setCreateUserId(USER_ID);
                 leafTaskIndex.setCreateTime(TimeUtil.getNowForMysql());
                 leafTaskIndex.setModifyUserId(USER_ID);
                 leafTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
                 leafTaskIndex.setIsDeleted("0");
-                taskIndexMapper.insertFirstIndex(leafTaskIndex);
+                indexMapper.insertFirstIndex(leafTaskIndex);
                 leafTaskIndexIterator.remove();
             }
         }
-        if (leafTaskIndexList.size() > 0) {
-
-            List<TaskIndexPO> nextLevelIndexList = new ArrayList<>();
+        if (leafIndexList.size() > 0) {
+            List<LeafIndexPO> nextLevelIndexList = new ArrayList<>();
             // 找出等级和 maxLevel 不相同的指标暂时不计算
-            leafTaskIndexList.stream()
+            leafIndexList.stream()
                     .filter(po -> maxLevel != po.getPackageLevel())
                     .forEach(nextLevelIndexList::add);
             // 找出等级和 maxLevel 相同的指标并根据父指标分组
-            Map<String, List<TaskIndexPO>> sonTaskIndexMap = leafTaskIndexList.stream()
+            Map<String, List<LeafIndexPO>> sonTaskIndexMap = leafIndexList.stream()
                     .filter(po -> maxLevel == po.getPackageLevel())
-                    .collect(Collectors.groupingBy(TaskIndexPO::getParentId));
+                    .collect(Collectors.groupingBy(LeafIndexPO::getParentId));
             Set<String> parentIdSet = sonTaskIndexMap.keySet();
             List<String> parentIdList = CollectionUtil.setToList(parentIdSet);
-            List<IndexTemplatePO> parentIndexTemplateList = indexTemplateMapper.selectByIdList(parentIdList);
+
+            List<IndexTemplatePO> parentIndexTemplateList = allIndexTemplateList.stream()
+                    .filter(indexTemplate -> parentIdList.contains(indexTemplate.getIndexId()))
+                    .collect(Collectors.toList());
             // 计算父指标得分
             parentIndexTemplateList.forEach(indexTemplate -> {
                 String weight = indexTemplate.getWeight();
-                List<TaskIndexPO> sonTaskIndexList = sonTaskIndexMap.get(indexTemplate.getIndexId());
+                List<LeafIndexPO> sonTaskIndexList = sonTaskIndexMap.get(indexTemplate.getIndexId());
                 double parentScore = sonTaskIndexList.stream().mapToDouble(taskIndex -> taskIndex.getScore() * Double.parseDouble(taskIndex.getWeight()) / 100).sum();
-                TaskIndexPO parentTaskIndex = TaskIndexPO.builder()
+                LeafIndexPO parentTaskIndex = LeafIndexPO.builder()
                         .id(StringUtil.getRandomUUID())
                         .pId(projectId)
                         .target(indexTemplate.getIndexId())
@@ -398,10 +401,29 @@ public class TaskService {
                 nextLevelIndexList.add(parentTaskIndex);
             });
             // 将父指标作为叶子指标递归
-            computeFirst(nextLevelIndexList, projectId, maxLevel - 1);
+            computeFirst(nextLevelIndexList, allIndexTemplateList, projectId, maxLevel - 1);
         }
     }
 
+    @SneakyThrows
+    public void evaluationLevel(String projectId) {
+        String tokenUrl = tokenUri + "?grant_type=client_credentials"
+                + "&client_id=" + clientId
+                + "&client_secret=" + clientSecret;
+        log.info("TaskService--state 获取仿真云平台 token:" + tokenUrl);
+        String response = HttpUtil.get(closeableHttpClient, requestConfig, tokenUrl);
+        ObjectMapper objectMapper = new ObjectMapper();
+        JsonNode jsonNode = objectMapper.readTree(response);
+        String accessToken = jsonNode.path("access_token").asText();
+        log.info("TaskService--state 仿真云平台 token 为:" + accessToken);
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Authorization", "Bearer " + accessToken);
+        Map<String, String> params = new HashMap<>();
+        params.put("id", projectId);
+        String post = HttpUtil.post(closeableHttpClient, requestConfig, evaluationLevelUri, headers, params);
+        log.info("TaskService--state 访问仿真云平台评价等级接口:" + evaluationLevelUri + ",请求头为:" + headers + ",请求体为:" + params + "结果为:" + post);
+    }
+
 
     public Boolean taskConfirm(String taskId) {
         // 查询 task 如果不是 pending 则不执行
@@ -414,7 +436,7 @@ public class TaskService {
         // 刷新 redis 心跳时间
         ProjectPO projectPO = manualProjectMapper.selectById(taskId);
         String projectId = projectPO.getId();
-        redisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId, TimeUtil.getNowString());
+        stringRedisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId + ":tick", TimeUtil.getNowString());
     }
 
 

+ 25 - 25
simulation-resource-scheduler/src/test/java/com/css/simulation/resource/scheduler/SchedulerTest.java

@@ -5,10 +5,10 @@ import api.common.util.CollectionUtil;
 import api.common.util.StringUtil;
 import api.common.util.TimeUtil;
 import com.css.simulation.resource.scheduler.mapper.IndexTemplateMapper;
-import com.css.simulation.resource.scheduler.mapper.TaskIndexMapper;
+import com.css.simulation.resource.scheduler.mapper.IndexMapper;
 import com.css.simulation.resource.scheduler.mapper.TaskMapper;
 import com.css.simulation.resource.scheduler.pojo.po.IndexTemplatePO;
-import com.css.simulation.resource.scheduler.pojo.po.TaskIndexPO;
+import com.css.simulation.resource.scheduler.pojo.po.LeafIndexPO;
 import io.kubernetes.client.openapi.ApiClient;
 import io.kubernetes.client.openapi.ApiException;
 import lombok.extern.slf4j.Slf4j;
@@ -29,7 +29,7 @@ import java.util.stream.Collectors;
 public class SchedulerTest {
 
     @Autowired
-    TaskIndexMapper taskIndexMapper;
+    IndexMapper indexMapper;
     @Autowired
     TaskMapper taskMapper;
     @Autowired
@@ -52,7 +52,7 @@ public class SchedulerTest {
     public void compute() {
 
         /*
-TaskIndexPO(id=0a8cd6c061a24ed2bbb36a45b3a003d4,
+LeafIndexPO(id=0a8cd6c061a24ed2bbb36a45b3a003d4,
          pId=e8337795555541639659d773e28eafd5,
           indexId=433838e404a74f87ba8f78c617134eec,
            parentId=018ae110ca51413ba22268f36c125a03,
@@ -61,7 +61,7 @@ TaskIndexPO(id=0a8cd6c061a24ed2bbb36a45b3a003d4,
               notStandardSceneNum=0, score=100.0, weight=40,
                scoreExplain=1) 未发生碰撞>,得分 100;2) 发生碰撞,得分 0。),
 
-TaskIndexPO(id=24a53877c1b34b3bb61dc9d7e28035d2,
+LeafIndexPO(id=24a53877c1b34b3bb61dc9d7e28035d2,
                  pId=e8337795555541639659d773e28eafd5,
                  indexId=a2a1866e5c7047c3bddc905971540043,
                  parentId=148c7e1fb6474cdf8c106215f92b0d9d,
@@ -71,7 +71,7 @@ TaskIndexPO(id=24a53877c1b34b3bb61dc9d7e28035d2,
                  score=0.0,
                   weight=70, scoreExplain=1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。),
 
-TaskIndexPO(id=94a9225873534e97be79b25c98861b83,
+LeafIndexPO(id=94a9225873534e97be79b25c98861b83,
       pId=e8337795555541639659d773e28eafd5,
       indexId=df913a81a34742808cefc76829bfa398,
        parentId=e28c9687cfdd4fec956d4c3a7fa061ec,
@@ -79,7 +79,7 @@ TaskIndexPO(id=94a9225873534e97be79b25c98861b83,
          target=df913a81a34742808cefc76829bfa398,
          notStandardSceneNum=0, score=100.0, weight=60, scoreExplain=1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。),
 
-TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
+LeafIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
          pId=e8337795555541639659d773e28eafd5,
           indexId=eefbf9234bad410b801ed860b4add56f,
            parentId=e28c9687cfdd4fec956d4c3a7fa061ec,
@@ -89,9 +89,9 @@ TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
          */
 
 
-        List<TaskIndexPO> leafTaskIndexList = new ArrayList<>();
+        List<LeafIndexPO> leafTaskIndexList = new ArrayList<>();
 
-        TaskIndexPO taskIndexPO1 = TaskIndexPO.builder()
+        LeafIndexPO leafIndexPO1 = LeafIndexPO.builder()
                 .id("0a8cd6c061a24ed2bbb36a45b3a003d4")
                 .pId("e8337795555541639659d773e28eafd5")
                 .indexId("433838e404a74f87ba8f78c617134eec")
@@ -104,9 +104,9 @@ TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
                 .scoreExplain("1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。)")
                 .packageLevel(1)
                 .build();
-        leafTaskIndexList.add(taskIndexPO1);
+        leafTaskIndexList.add(leafIndexPO1);
 
-        TaskIndexPO taskIndexPO2 = TaskIndexPO.builder()
+        LeafIndexPO leafIndexPO2 = LeafIndexPO.builder()
                 .id("24a53877c1b34b3bb61dc9d7e28035d2")
                 .pId("e8337795555541639659d773e28eafd5")
                 .indexId("a2a1866e5c7047c3bddc905971540043")
@@ -119,9 +119,9 @@ TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
                 .scoreExplain("1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。)")
                 .packageLevel(2)
                 .build();
-        leafTaskIndexList.add(taskIndexPO2);
+        leafTaskIndexList.add(leafIndexPO2);
 
-        TaskIndexPO taskIndexPO3 = TaskIndexPO.builder()
+        LeafIndexPO leafIndexPO3 = LeafIndexPO.builder()
                 .id("94a9225873534e97be79b25c98861b83")
                 .pId("e8337795555541639659d773e28eafd5")
                 .indexId("df913a81a34742808cefc76829bfa398")
@@ -134,10 +134,10 @@ TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
                 .scoreExplain("1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。)")
                 .packageLevel(3)
                 .build();
-        leafTaskIndexList.add(taskIndexPO3);
+        leafTaskIndexList.add(leafIndexPO3);
 
 
-        TaskIndexPO taskIndexPO4 = TaskIndexPO.builder()
+        LeafIndexPO leafIndexPO4 = LeafIndexPO.builder()
                 .id("14d0017b489844bcb33118f7cfeee04c")
                 .pId("e8337795555541639659d773e28eafd5")
                 .indexId("eefbf9234bad410b801ed860b4add56f")
@@ -150,49 +150,49 @@ TaskIndexPO(id=14d0017b489844bcb33118f7cfeee04c,
                 .scoreExplain("1) 未发生碰撞,得分 100;2) 发生碰撞,得分 0。)")
                 .packageLevel(3)
                 .build();
-        leafTaskIndexList.add(taskIndexPO4);
+        leafTaskIndexList.add(leafIndexPO4);
 
 
         computeFirst(leafTaskIndexList, "e8337795555541639659d773e28eafd5", 3);
     }
 
-    public void computeFirst(List<TaskIndexPO> leafTaskIndexList, String projectId, int maxLevel) {
+    public void computeFirst(List<LeafIndexPO> leafTaskIndexList, String projectId, int maxLevel) {
 
         log.info("------- /state computeFirst 计算父指标得分:" + leafTaskIndexList);
-        Iterator<TaskIndexPO> leafTaskIndexIterator = leafTaskIndexList.iterator();
+        Iterator<LeafIndexPO> leafTaskIndexIterator = leafTaskIndexList.iterator();
         // 把 1 级的指标得分直接保存
         while (leafTaskIndexIterator.hasNext()) {
-            TaskIndexPO leafTaskIndex = leafTaskIndexIterator.next();
+            LeafIndexPO leafTaskIndex = leafTaskIndexIterator.next();
             if (leafTaskIndex.getPackageLevel() == 1) {
                 leafTaskIndex.setCreateUserId(USER_ID);
                 leafTaskIndex.setCreateTime(TimeUtil.getNowForMysql());
                 leafTaskIndex.setModifyUserId(USER_ID);
                 leafTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
                 leafTaskIndex.setIsDeleted("0");
-                taskIndexMapper.insertFirstIndex(leafTaskIndex);
+                indexMapper.insertFirstIndex(leafTaskIndex);
                 leafTaskIndexIterator.remove();
             }
         }
         if (leafTaskIndexList.size() > 0) {
 
-            List<TaskIndexPO> nextLevelIndexList = new ArrayList<>();
+            List<LeafIndexPO> nextLevelIndexList = new ArrayList<>();
             // 找出等级和 maxLevel 不相同的指标暂时不计算
             leafTaskIndexList.stream()
                     .filter(po -> maxLevel != po.getPackageLevel())
                     .forEach(nextLevelIndexList::add);
             // 找出等级和 maxLevel 相同的指标并根据父指标分组
-            Map<String, List<TaskIndexPO>> sonTaskIndexMap = leafTaskIndexList.stream()
+            Map<String, List<LeafIndexPO>> sonTaskIndexMap = leafTaskIndexList.stream()
                     .filter(po -> maxLevel == po.getPackageLevel())
-                    .collect(Collectors.groupingBy(TaskIndexPO::getParentId));
+                    .collect(Collectors.groupingBy(LeafIndexPO::getParentId));
             Set<String> parentIdSet = sonTaskIndexMap.keySet();
             List<String> parentIdList = CollectionUtil.setToList(parentIdSet);
             List<IndexTemplatePO> parentIndexTemplateList = indexTemplateMapper.selectByIdList(parentIdList);
             // 计算父指标得分
             parentIndexTemplateList.forEach(indexTemplate -> {
                 String weight = indexTemplate.getWeight();
-                List<TaskIndexPO> sonTaskIndexList = sonTaskIndexMap.get(indexTemplate.getIndexId());
+                List<LeafIndexPO> sonTaskIndexList = sonTaskIndexMap.get(indexTemplate.getIndexId());
                 double parentScore = sonTaskIndexList.stream().mapToDouble(taskIndex -> taskIndex.getScore() * Double.parseDouble(taskIndex.getWeight()) / 100).sum();
-                TaskIndexPO parentTaskIndex = TaskIndexPO.builder()
+                LeafIndexPO parentTaskIndex = LeafIndexPO.builder()
                         .id(StringUtil.getRandomUUID())
                         .pId(projectId)
                         .target(indexTemplate.getIndexId())

برخی فایل ها در این مقایسه diff نمایش داده نمی شوند زیرا تعداد فایل ها بسیار زیاد است