Browse Source

标准化测试

LingxinMeng 2 years ago
parent
commit
c432ed8328

+ 0 - 1
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/api/model/ProjectMessageModel.java

@@ -29,7 +29,6 @@ public class ProjectMessageModel {
     private String modelType;// vtd carsim
     private String type;// 项目类型
     // -------------------------------- Comment --------------------------------
-    private Integer currentParallelism;// 当前正在使用的并行度
     private Integer taskTotal;// 任务总数量
     private Integer taskCompleted;// 任务已完成数量
 

+ 3 - 7
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/application/service/ProjectService.java

@@ -455,7 +455,6 @@ public class ProjectService {
         }
         String projectId = projectMessageModel.getProjectId();    // 项目 id
         String vehicleConfigId = projectMessageModel.getVehicleConfigId();
-        int currentParallelism = projectMessageModel.getCurrentParallelism();   // 当前并行度
         String algorithmId = projectMessageModel.getAlgorithmId();    // 算法 id
         String projectPath = linuxTempPath + "project/" + projectId + "/";
         // -------------------------------- 1 获取任务 json 列表 --------------------------------
@@ -468,9 +467,6 @@ public class ProjectService {
         Map<String, Integer> nodeMap = projectDomainService.getNodeMapToUse(isChoiceGpu, finalParallelism);
         //2 将指定 node 的并行度减少
         nodeMap.keySet().forEach(nodeName -> projectDomainService.decrementParallelism(isChoiceGpu, nodeName, nodeMap.get(nodeName)));
-        // 重新设置实际使用的并行度并保存到 redis
-        int realCurrentParallelism = nodeMap.values().stream().mapToInt(parallelism -> parallelism).sum();
-        projectMessageModel.setCurrentParallelism(realCurrentParallelism);
         log.info("项目 " + projectId + " 运行在:" + nodeMap);
         stringRedisTemplate.opsForValue().set(projectRunningKey, JsonUtil.beanToJson(projectMessageModel));
         //* -------------------------------- 3 根据算法id查询算法名称 --------------------------------
@@ -478,7 +474,7 @@ public class ProjectService {
         // -------------------------------- 4 发送任务消息 --------------------------------
         List<NodeEntity> nodeListToCount = projectDomainService.getNodeListToCount(nodeMap);
         int messageNumber = 0;
-        KafkaUtil.createTopic(kafkaAdminClient, projectId, realCurrentParallelism, (short) 1);   // 创建主题
+        KafkaUtil.createTopic(kafkaAdminClient, projectId, finalParallelism, (short) 1);   // 创建主题
         TimeUnit.SECONDS.sleep(7);
         // 需要即时启动的任务(并行度的大小)
         CopyOnWriteArrayList<String> yamlToRunRedisKeyList = new CopyOnWriteArrayList<>();
@@ -490,12 +486,12 @@ public class ProjectService {
             stringRedisTemplate.opsForValue().set(taskMessageKey, taskJson);
 
             //4-5 将 projectId 作为 topic 名称,根据 parallelism 分散发送 task 信息到 kafka
-            SendResult<String, String> stringStringSendResult = kafkaTemplate.send(projectId, messageNumber % currentParallelism, taskId, taskJson).get();
+            SendResult<String, String> stringStringSendResult = kafkaTemplate.send(projectId, messageNumber % finalParallelism, taskId, taskJson).get();
             RecordMetadata recordMetadata = stringStringSendResult.getRecordMetadata();
             String topic = recordMetadata.topic();  // 消息发送到的topic
             int partition = recordMetadata.partition(); // 消息发送到的分区
             long offset = recordMetadata.offset();  // 消息在分区内的offset
-            log.info("发送消息成功, 主题 topic 为:" + topic + " 分区 partition 为:" + partition + " 偏移量为:" + offset + " 消息体为:" + taskJson);
+            log.info("发送消息成功, 主题 topic 为项目ID:" + topic + " 分区 partition 为:" + partition + " 偏移量为:" + offset);
             //4-6 发送成功过的同时创建 pod.yaml 文件并把文件地址存到 redis
             // 选一个 count 最少的 node
             String currentNodeName = "";

+ 4 - 0
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/application/service/TaskService.java

@@ -60,6 +60,10 @@ public class TaskService {
     @Async("pool1")
     public void state(String taskId, String state, String podName) {
         TaskEntity taskEntity = taskMapper.selectById(taskId);
+        if (taskEntity == null) {
+            log.debug("收到不存在的任务的状态消息:" + taskId);
+            return;
+        }
         String projectId = taskEntity.getPId(); // 项目 id
         String lock1 = "taskId:" + taskId + ":state:" + state + ":pod-name:" + podName;
 

+ 10 - 10
simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/domain/service/TaskDomainService.java

@@ -148,7 +148,7 @@ public class TaskDomainService {
                 String task2Id = taskOfLeaf.getId();
 
                 String runState = taskOfLeaf.getRunState();
-                log.info("TaskManager--score 任务 " + task2Id + " 的运行状态为:" + runState);
+                log.debug("任务 " + task2Id + " 的运行状态为:" + runState);
                 if (DictConstants.TASK_ANALYSIS.equals(runState)) {
                     taskMapper.updateSuccessStateWithStopTime(task2Id, DictConstants.TASK_ANALYSING, TimeUtil.getNowForMysql());
                     // 计算每个任务的得分
@@ -159,10 +159,10 @@ public class TaskDomainService {
                     String scoreCommand = "python3 " + scoreDirectoryOfUser + "main.py " + result1OfLinux + " " + result2OfLinux + " " + taskOfLeaf.getSceneType() + " " + ruleName; // 指定打分脚本
                     String scoreResult;
                     ScoreEntity score = null;
-                    log.info("下载 minio 上的结果文件 " + result1OfMinio + " 和 " + result2OfMinio + " 到临时目录:" + linuxTempPath);
+                    log.debug("下载 minio 上的结果文件 " + result1OfMinio + " 和 " + result2OfMinio + " 到临时目录:" + linuxTempPath);
                     MinioUtil.downloadToFile(minioClient, bucketName, result1OfMinio, result1OfLinux);  // 也可改成下载到指定ip的服务器上,需要保证和打分脚本在一台机器上。
                     MinioUtil.downloadToFile(minioClient, bucketName, result2OfMinio, result2OfLinux);  // 也可改成下载到指定ip的服务器上,需要保证和打分脚本在一台机器上。
-                    log.info("开始执行打分命令:" + scoreCommand);
+                    log.debug("开始执行打分命令:" + scoreCommand);
                     Runtime r = Runtime.getRuntime();
                     Process p = r.exec(scoreCommand, null, new File(scoreDirectoryOfUser));
                     BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));
@@ -172,12 +172,12 @@ public class TaskDomainService {
                         sb.append(inline).append("\n");
                     }
                     scoreResult = sb.toString();
-                    log.info("项目" + projectId + " 的任务 " + task2Id + " 打分结束,结果为:" + scoreResult);
+                    log.debug("项目" + projectId + " 的任务 " + task2Id + " 打分结束,结果为:" + scoreResult);
                     String replace = StringUtil.replace(scoreResult, "'", "\"");
                     try {
                         score = JsonUtil.jsonToBean(replace, ScoreEntity.class);
                     } catch (Exception e) { // 打分失败
-                        log.info("项目" + projectId + " 的任务 " + task2Id + " 打分失败:", e);
+                        log.debug("项目" + projectId + " 的任务 " + task2Id + " 打分失败:", e);
                     }
                     if (score != null) {
                         taskOfLeaf.setReturnSceneId(score.getUnit_scene_ID());
@@ -233,7 +233,7 @@ public class TaskDomainService {
 
             // 计算任务的个数
             long taskNumberToScore = taskListOfLeafIndex.size();
-            log.info("项目 " + projectId + " 的叶子指标 " + indexId + " 下参与计算的任务总数为 " + taskNumberToScore + ":仿真异常场景个数 " + errorSceneNumber + "、未达标场景个数 " + notStandardSceneNumber + "、达标场景个数 " + standardSceneNumber);
+            log.debug("项目 " + projectId + " 的叶子指标 " + indexId + " 下参与计算的任务总数为 " + taskNumberToScore + ":仿真异常场景个数 " + errorSceneNumber + "、未达标场景个数 " + notStandardSceneNumber + "、达标场景个数 " + standardSceneNumber);
 
             // 计算叶子指标得分(任务得分总和 / 任务数量)
             double leafIndexScore = NumberUtil.cut(leafSum / taskNumberToScore, 2);
@@ -252,14 +252,14 @@ public class TaskDomainService {
         // 保存叶子指标得分
         taskIndexRepository.batchInsertLeafIndex(leafIndexList);
         // 保存一级指标分数
-        log.info("项目 " + projectId + " 的所有任务分数为:" + taskList);
+        log.debug("项目 " + projectId + " 的所有任务分数为:" + taskList);
         computeFirst(leafIndexList, allIndexTemplateList, projectId, maxLevel);
         log.info("项目 " + projectId + " 打分完成!");
     }
 
     public void computeFirst(List<LeafIndexEntity> leafIndexList, List<IndexTemplateEntity> allIndexTemplateList, String projectId, int maxLevel) {
 
-        log.info("计算父指标得分:" + leafIndexList);
+        log.debug("计算父指标得分:" + leafIndexList);
         Iterator<LeafIndexEntity> leafTaskIndexIterator = leafIndexList.iterator();
         // 把 1 级的指标得分直接保存
         while (leafTaskIndexIterator.hasNext()) {
@@ -300,12 +300,12 @@ public class TaskDomainService {
     @SneakyThrows
     public void evaluationLevel(String projectId) {
         String tokenUrl = customConfiguration.getTokenUri() + "?grant_type=client_credentials" + "&client_id=" + customConfiguration.getClientId() + "&client_secret=" + customConfiguration.getClientSecret();
-        log.info("获取仿真云平台 token:" + tokenUrl);
+        log.debug("获取仿真云平台 token:" + tokenUrl);
         String response = HttpUtil.get(tokenUrl);
         ObjectMapper objectMapper = new ObjectMapper();
         JsonNode jsonNode = objectMapper.readTree(response);
         String accessToken = jsonNode.path("access_token").asText();
-        log.info("仿真云平台 token 为:" + accessToken);
+        log.debug("仿真云平台 token 为:" + accessToken);
         Map<String, String> headers = new HashMap<>();
         headers.put("Authorization", "Bearer " + accessToken);
         Map<String, String> params = new HashMap<>();

+ 5 - 5
simulation-resource-scheduler/src/main/resources/logback-spring.xml

@@ -18,7 +18,7 @@
             <level>debug</level>
         </filter>
         <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{5}).%M\(%F:%L\) %highlight(%msg) %n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %cyan(%logger{5}).%M\(%F:%L\) %msg %n</pattern>
             <charset>utf-8</charset>
         </encoder>
     </appender>
@@ -35,7 +35,7 @@
         </rollingPolicy>
         <append>true</append>
         <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{5}).%M\(%F:%L\) %highlight(%msg) %n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %cyan(%logger{5}).%M\(%F:%L\) %msg %n</pattern>
             <charset>utf-8</charset>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印DEBUG日志 -->
@@ -57,7 +57,7 @@
         </rollingPolicy>
         <append>true</append>
         <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{5}).%M\(%F:%L\) %highlight(%msg) %n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %cyan(%logger{5}).%M\(%F:%L\) %msg %n</pattern>
             <charset>utf-8</charset>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印INFO日志 -->
@@ -79,7 +79,7 @@
         </rollingPolicy>
         <append>true</append>
         <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{5}).%M\(%F:%L\) %highlight(%msg) %n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %cyan(%logger{5}).%M\(%F:%L\) %msg %n</pattern>
             <charset>utf-8</charset>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印ERROR日志 -->
@@ -101,7 +101,7 @@
         </rollingPolicy>
         <append>true</append>
         <encoder>
-            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{5}).%M\(%F:%L\) %highlight(%msg) %n</pattern>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %cyan(%logger{5}).%M\(%F:%L\) %msg %n</pattern>
             <charset>utf-8</charset>
         </encoder>
         <filter class="ch.qos.logback.classic.filter.LevelFilter"><!-- 只打印WARN日志 -->