mlx

api-common/src/main/java/api/common/pojo/constants/DictConstants.java

@@ -45,10 +45,20 @@ public class DictConstants {
     public static final String SENSOR_RADAR = "radar"; // 毫米波雷达表
     public static final String SENSOR_GPS = "gps"; // GPS 传感器表
 
+
+//    当前运行状态(10：未执行，20：执行中，30：执行完成，40：已终止 50 执行出错)
+
+    public static final String PROJECT_WAITING = "10"; // 项目执行状态，待执行
+    public static final String PROJECT_RUNNING = "20"; // 项目执行状态，执行中
+    public static final String PROJECT_COMPLETED = "30"; // 项目执行状态，已完成
+    public static final String PROJECT_TERMINATED = "40"; // 项目执行状态，已终止
+    public static final String PROJECT_ERROR = "50"; // 项目执行状态，执行出错
+
+
     public static final String TASK_PENDING = "Pending"; // 任务执行状态，待执行
     public static final String TASK_RUNNING = "Running"; // 任务执行状态，运行中
     public static final String TASK_ABORTED = "Aborted"; // 任务执行状态，中断
-    public static final String TASK_ANALYSIS = "Analysis"; // 任务执行状态，准备分析
+    public static final String TASK_ANALYSIS = "PendingAnalysis"; // 任务执行状态，准备分析
     public static final String TASK_ANALYSING = "Analysing"; // 任务执行状态，分析中
     public static final String TASK_COMPLETED = "Completed"; // 任务执行状态，已完成
     public static final String TASK_TERMINATING = "Terminating"; // 任务执行状态，终止中

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/SimulationResourceSchedulerApplication.java

@@ -4,11 +4,13 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 
 @SpringBootApplication
 @EnableFeignClients
 @EnableDiscoveryClient
+@EnableScheduling
 public class SimulationResourceSchedulerApplication {
 
     public static void main(String[] args) {

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/kubernetes/KubernetesConfiguration.java

@@ -5,7 +5,6 @@ import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.util.ResourceUtils;
 
 import java.io.File;
 import java.io.FileReader;
@@ -16,9 +15,9 @@ public class KubernetesConfiguration {
 
     @Bean
     public ApiClient apiClient() throws IOException {
-        File config = ResourceUtils.getFile("classpath:kubernetes/config");  // 开发环境可用，生产环境不行，无法从jar 包读取
+//        File config = ResourceUtils.getFile("classpath:kubernetes/config");  // 开发环境可用，生产环境不行，无法从jar 包读取
 //        File config = new File("D:\\idea-project\\simulation-cloud\\simulation-resource-scheduler\\src\\main\\resources\\kubernetes\\config");  //windows
-//        File config = new File("/root/.kube/config");   //linux
+        File config = new File("/root/.kube/config");   //linux
 //
 //        ClassPathResource classPathResource = new ClassPathResource("kubernetes/config");
 //        InputStream inputStream =classPathResource.getInputStream();

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumer.java

@@ -221,10 +221,10 @@ public class ManualProjectConsumer {
         }
 
         // -------------------------------- 4 算法导入（一期按单机版做） --------------------------------
-//        // 私有仓库导入算法镜像（搭建私有仓库）
-//        String algorithmId = projectMessageDTO.getAlgorithmId();    // 算法 id
-//        //4-1 根据算法 id 获取算法文件地址、是否已导入成镜像。
-//        AlgorithmPO algorithmPO = algorithmMapper.selectById(algorithmId);
+        // 私有仓库导入算法镜像
+        String algorithmId = projectMessageDTO.getAlgorithmId();    // 算法 id
+        //4-1 根据算法 id 获取算法文件地址、是否已导入成镜像。
+        AlgorithmPO algorithmPO = algorithmMapper.selectById(algorithmId);
 //        String minioPath = algorithmPO.getMinioPath();
 //        String dockerImage;
 //        if ("0".equals(algorithmPO.getDockerImport())) {
@@ -294,7 +294,7 @@ public class ManualProjectConsumer {
 //        String projectJson = projectRecord.value();
         ProjectMessageDTO projectMessageDTO = JsonUtil.jsonToBean(projectJson, ProjectMessageDTO.class);
         String projectId = projectMessageDTO.getProjectId();    // 项目 id
-        projectMapper.updateProjectState(projectId, "20");   // 修改该 project 的状态为执行中
+        projectMapper.updateProjectState(projectId, DictConstants.PROJECT_RUNNING);   // 修改该 project 的状态为执行中
 
 
         // -------------------------------- 1 场景 --------------------------------

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/AlgorithmMapper.java

@@ -21,7 +21,8 @@ public interface AlgorithmMapper {
     })
     @Select("select minio_path,\n" +
             "   docker_import,\n" +
-            "   docker_image\n" +
+            "   docker_image,\n" +
+            "   upload_mode\n" +
             "from algorithm\n" +
             "where is_deleted = '0'\n" +
             "  and id = #{id}")

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/IndexTemplateMapper.java

@@ -2,7 +2,6 @@ package com.css.simulation.resource.scheduler.mapper;
 
 
 import com.css.simulation.resource.scheduler.pojo.po.IndexTemplatePO;
-import com.css.simulation.resource.scheduler.pojo.po.TaskIndexPO;
 import org.apache.ibatis.annotations.*;
 import org.apache.ibatis.type.JdbcType;
 
@@ -36,7 +35,8 @@ public interface IndexTemplateMapper {
     List<IndexTemplatePO> selectLeafIndexByPackageId(@Param("packageId") String packageId);
 
     @ResultMap("index")
-    @Select("select sps.scene_natural_ids,\n" +
+    @Select("select sps.sublist_id," +
+            "       sps.scene_natural_ids,\n" +
             "       sps.scene_traffic_ids,\n" +
             "       sps.scene_statue_ids,\n" +
             "       sps.weight,\n" +

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/ProjectMapper.java

@@ -15,8 +15,8 @@ public interface ProjectMapper {
     })
 
     @Select("select id, scene\n" +
-            "from simulation_manual_project \n" +
-            "where id = (select project_id from simulation_manual_project_task where id = #{taskId})")
+            "from simulation_manual_project\n" +
+            "where id = (select p_id from simulation_manual_project_task where id = #{taskId})\n")
     ProjectPO selectById(@Param("taskId")String taskId);
 
     @Update("update simulation_manual_project\n" +
@@ -33,12 +33,12 @@ public interface ProjectMapper {
 
     @Select("select task_number\n" +
             "from simulation_manual_project\n" +
-            "where project_id = #{id}")
+            "where id = #{id}")
     int selectTaskNumById(@Param("id") String id);
 
     @Select("select count(1)\n" +
             "from simulation_manual_project_task\n" +
-            "where run_state = #{state}\n" +
-            "  and p_id = #{projectId}")
-    int selectTaskNumByProjectIdAndState(@Param("projectId") String projectId,@Param("state") String state);
+            "where run_state in ('Aborted', 'PendingAnalysis', 'Terminated')\n" +
+            "  and p_id in #{projectId}")
+    int selectEndTaskNum(@Param("projectId") String projectId);
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/SceneMapper.java

@@ -40,8 +40,8 @@ public interface SceneMapper {
             @Result(column = "name", property = "name", jdbcType = JdbcType.VARCHAR),
             @Result(column = "type", property = "type", jdbcType = JdbcType.VARCHAR),
             @Result(column = "osc", property = "scenarioOsc", jdbcType = JdbcType.VARCHAR),
-            @Result(column = "osgb", property = "scenarioOdr", jdbcType = JdbcType.VARCHAR),
-            @Result(column = "odr", property = "scenarioOsgb", jdbcType = JdbcType.VARCHAR)
+            @Result(column = "osgb", property = "scenarioOsgb", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "odr", property = "scenarioOdr", jdbcType = JdbcType.VARCHAR)
     })
     @Select("<script>" +
             "select natural_id id,\n" +

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/SensorCameraMapper.java

@@ -10,7 +10,7 @@ import java.util.List;
 public interface SensorCameraMapper {
 
     @Results(id = "camera", value = {
-            @Result(column = "name", property = "sensor_name", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "sensor_name", property = "sensor_name", jdbcType = JdbcType.VARCHAR),
             @Result(column = "fov_h", property = "sensor_fovH", jdbcType = JdbcType.DECIMAL),
             @Result(column = "fov_v", property = "sensor_fovV", jdbcType = JdbcType.DECIMAL),
             @Result(column = "near_distance", property = "sensor_near", jdbcType = JdbcType.DECIMAL),

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/SensorOgtMapper.java

@@ -13,9 +13,10 @@ public interface SensorOgtMapper {
             @Result(column = "sensor_name", property = "sensor_name", jdbcType = JdbcType.VARCHAR),
             @Result(column = "fov_h_left", property = "sensor_fovHLeft", jdbcType = JdbcType.DECIMAL),
             @Result(column = "fov_h_right", property = "sensor_fovHRight", jdbcType = JdbcType.DECIMAL),
+            @Result(column = "fov_v_top", property = "sensor_fovVTop", jdbcType = JdbcType.DECIMAL),
             @Result(column = "fov_v_bottom", property = "sensor_fovVBottom", jdbcType = JdbcType.DECIMAL),
-            @Result(column = "sensor_near", property = "sensor_near", jdbcType = JdbcType.DECIMAL),
-            @Result(column = "sensor_far", property = "sensor_far", jdbcType = JdbcType.DECIMAL),
+            @Result(column = "near_distance", property = "sensor_near", jdbcType = JdbcType.DECIMAL),
+            @Result(column = "far_distance", property = "sensor_far", jdbcType = JdbcType.DECIMAL),
             @Result(column = "sensor_x", property = "sensor_x", jdbcType = JdbcType.DECIMAL),
             @Result(column = "sensor_y", property = "sensor_y", jdbcType = JdbcType.DECIMAL),
             @Result(column = "sensor_z", property = "sensor_z", jdbcType = JdbcType.DECIMAL),

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/TaskMapper.java

@@ -35,11 +35,9 @@ public interface TaskMapper {
     List<TaskPO> selectTaskListByProjectId(@Param("projectId") String projectId);
 
 
-    @Update("update simulation_manual_project_task smpt,monitor_task mt\n" +
-            "set smpt.run_state = #{tickTime},\n" +
-            "    mt.run_state= #{tickTime}\n" +
-            "where smpt.id = #{id}\n" +
-            "  and mt.task_id = #{id}")
+    @Update("update simulation_manual_project_task\n" +
+            "set run_state = #{runState}\n" +
+            "where id = #{id}")
     void updateState(@Param("id") String id, @Param("runState") String runState);
 
 

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/TickScheduler.java

@@ -4,6 +4,7 @@ import api.common.pojo.constants.DictConstants;
 import api.common.util.TimeUtil;
 import com.css.simulation.resource.scheduler.mapper.TaskMapper;
 import com.css.simulation.resource.scheduler.pojo.po.TaskPO;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
@@ -13,6 +14,7 @@ import org.springframework.stereotype.Component;
 import java.util.List;
 
 @Component
+@Slf4j
 public class TickScheduler {
 
     @Value("${scheduler.manual-project.topic}")
@@ -25,16 +27,21 @@ public class TickScheduler {
 
     @Scheduled(fixedDelay = 2000)
     public void tick() {
-        //1 查询出所有执行中的任务（除了 等待中 和 已完成）
+        log.info("------- TickScheduler 查询出所有执行中的任务（'Running', 'Analysis', 'Analysing'）");
         List<TaskPO> executingTaskList = taskMapper.selectExecuting();
         //2 根据 key 查出任务的心跳时间
         executingTaskList.forEach(task -> {
             String taskId = task.getId();
             String projectId = task.getPId();
-            Long maxSimulationTime = task.getMaxSimulationTime();
             long tickTime = Long.parseLong(redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":" + taskId));
+            long maxSimulationTime = task.getMaxSimulationTime() * 1000;
+            long now = TimeUtil.getNow();
+            long difference = TimeUtil.getNow() - tickTime;
+            log.info("------- TickScheduler 任务" + taskId + "心跳时间为：" + tickTime+ "最大仿真时间为：" + tickTime+ "时间差为：" + tickTime);
+            log.info("------- TickScheduler 任务" + taskId );
+            log.info("------- TickScheduler 任务" + taskId );
             if (TimeUtil.getNow() - tickTime > maxSimulationTime) {
-                //3 判断如果心跳时间距离当前时间已经超时，则修改任务状态为失败，并销毁 pod
+            log.info("------- TickScheduler 任务" + taskId + "已超时，状态修改为："+DictConstants.TASK_ABORTED);
                 taskMapper.updateState(taskId, DictConstants.TASK_ABORTED);
             }
         });

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java

@@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 
 import java.util.*;
 import java.util.stream.Collectors;
@@ -70,33 +69,35 @@ public class TaskService {
         redisTemplate.opsForValue().set(manualProjectTopic + ":" + projectId + ":" + taskId, TimeUtil.getNowString());
     }
 
-    @Transactional
     @SneakyThrows
     public void taskState(String taskId, String state) {
-        log.info("------- /state 接收到任务 " + taskId + "的状态：" + state);
-        //1 根据 taskId 修改任务状态 taskState。
+        log.info("------- /state 修改任务 " + taskId + "的状态：" + state);
         taskMapper.updateState(taskId, state);
-        //2 如果 taskState 为完成状态，校验一下项目的已完成数量。
         ProjectPO projectPO = projectMapper.selectById(taskId);
         String projectId = projectPO.getId();
         String scenePackageId = projectPO.getScenePackageId();  // 场景测试包 id，指标根 id
+        log.info("------- /state 任务 " + taskId + " 的父项目为：" + projectId);
         int taskNum = projectMapper.selectTaskNumById(projectId);
-        int completedTaskNum = projectMapper.selectTaskNumByProjectIdAndState(projectId, DictConstants.TASK_COMPLETED);
-        if (taskNum != completedTaskNum) {  // 已完成任务数等于所有任务数量，才会准备打分；否则退出。
+        int endTaskNum = projectMapper.selectEndTaskNum(projectId);    // 查询已结束的任务 'Aborted', 'PendingAnalysis', 'Terminated'
+        log.info("------- /state 项目 " + projectId + " 完成进度为：" + endTaskNum + "/" + taskNum);
+        if (taskNum != endTaskNum) {  // 已结束任务数等于所有任务数量，才会准备打分；否则退出。
             return;
         }
+        projectMapper.updateProjectState(projectId, DictConstants.PROJECT_COMPLETED);   // 修改该 project 的状态为已完成
         List<TaskPO> taskList = taskMapper.selectTaskListByProjectId(projectId);  // 所有任务信息
         // -------------------------------- 查询叶子指标 --------------------------------
         List<IndexTemplatePO> leafIndexTemplateList = indexTemplateMapper.selectLeafIndexWithRuleDetailsByPackageId(scenePackageId);
         List<TaskIndexPO> leafTaskIndexList = new ArrayList<>();
+        log.info("------- /state 共有 " + leafIndexTemplateList.size() + "个叶子节点！");
         for (int i = 0; i < leafIndexTemplateList.size(); i++) {
             IndexTemplatePO indexTemplatePO = leafIndexTemplateList.get(i);
-            // -------------------------------- 将叶子节点对应的打分规则保存到临时目录 --------------------------------
+            String indexId = indexTemplatePO.getIndexId();
+            log.info("------- /state 开始执行对第 " + i + " 个叶子节点 " + indexId + " 进行打分！");
             String ruleName = indexTemplatePO.getRuleName();    // 打分脚本名称，例如 AEB_1-1
             String ruleDetails = indexTemplatePO.getRuleDetails();    // 打分脚本内容
-            String ruleDetailsPath = pyPath + "script/" + projectId + "_" + i;
-            FileUtil.writeInputStreamToLocalFile(IoUtil.stringToInputStream(ruleDetails), ruleDetailsPath);
-            // -------------------------------- 查询每个叶子指标包括的场景 --------------------------------
+            String ruleFilePath = pyPath + "script/" + ruleName.split("_")[0] + "/" + ruleName + ".py";
+            log.info("------- /state 将叶子节点 " + indexId + " 对应的打分规则保存到临时目录：" + ruleFilePath);
+            FileUtil.writeInputStreamToLocalFile(IoUtil.stringToInputStream(ruleDetails), ruleFilePath);
             Set<String> sceneIdSet = new HashSet<>();
             String naturalIds = indexTemplatePO.getSceneNaturalIds();
             String standardIds = indexTemplatePO.getSceneStatueIds();
@@ -114,24 +115,29 @@ public class TaskService {
                 sceneIdSet.addAll(Arrays.asList(accidentIdArray));
             }
             int resultNumberOfCurrentIndex = sceneIdSet.size();
-            // -------------------------------- 计算叶子指标的得分 --------------------------------
+            log.info("------- /state 叶子节点 " + indexId + " 包括 " + resultNumberOfCurrentIndex + " 个场景！");
+            log.info("------- /state 计算叶子节点 " + indexId + " 的得分！");
             double sum = taskList.stream()
                     .filter(task1 -> sceneIdSet.contains(task1.getSceneId()))
                     .mapToDouble(task2 -> {
+                        String task2Id = task2.getId();
+                        taskMapper.updateState(task2Id, DictConstants.TASK_ANALYSING);
                         // 计算每个任务的得分
                         ScoreTO score;
                         String runResultMinio = task2.getRunResult();
                         String runResultLinux = linuxTempPath + runResultMinio;
+//                        python3 /home/ubuntu/test/Evaluate/main.py /home/ubuntu/test/test_data.csv 4 AEB_1-2
 //                        String command = "python3 " + pyPath + " " + runResultLinux + " " + task2.getSceneType();  // 默认使用场景名称找打分脚本
-                        String command = "python3 " + pyPath + "main.py " + runResultLinux + " " + task2.getSceneType() + " " + ruleDetailsPath; // 指定打分脚本
-
+                        String command = "python3 " + pyPath + "main.py " + runResultLinux + " " + task2.getSceneType() + " " + ruleName; // 指定打分脚本
                         try {
+                            log.info("------- /state 下载 minio 上的结果文件 " + runResultMinio + " 到本地：" + runResultLinux);
                             MinioUtil.downloadToFile(minioClient, bucketName, runResultMinio, runResultLinux);
-                            log.info("------- 开始执行打分命令：" + command);
+                            log.info("------- /state 开始执行打分命令：" + command);
                             score = JsonUtil.jsonToBean(SshUtil.execute(hostname, username, password, command), ScoreTO.class);
-                            log.info("------- 打分结束，结果为：" + score);
+                            log.info("------- /state 打分结束，结果为：" + score);
                         } catch (Exception e) {
-                            log.error("------- 打分出错，命令为：" + command);
+                            log.error("------- /state 任务 " + task2Id + " 打分出错，命令为：" + command + " 修改状态为：" + DictConstants.TASK_ABORTED);
+                            taskMapper.updateState(task2Id, DictConstants.TASK_ABORTED);
                             throw new RuntimeException(e.getMessage());
                         }
                         task2.setReturnSceneId(score.getUnit_scene_ID());
@@ -183,6 +189,8 @@ public class TaskService {
         totalTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
         totalTaskIndex.setIsDeleted("0");
         taskIndexMapper.insertTotalIndex(totalTaskIndex);
+
+
     }
 
 

simulation-resource-scheduler/src/main/resources/bootstrap.yml

@@ -2,6 +2,10 @@ server:
   tomcat:
     connection-timeout: 36000000
 spring:
+  servlet:
+    multipart:
+      max-file-size: 10GB
+      max-request-size: 10GB
   application:
     name: simulation-resource-scheduler
   profiles:

simulation-resource-server/src/main/resources/bootstrap.yaml

@@ -22,4 +22,4 @@ pagehelper:
 mybatis:
   mapper-locations: classpath:/mapper/*/*.xml
   configuration:
-    map-underscore-to-camel-case: true
+    map-underscore-to-camel-case: true