mlx

api-common/src/main/java/api/common/pojo/dto/Gpu.java → api-common/src/main/java/api/common/pojo/dto/GpuDTO.java

@@ -6,7 +6,7 @@ import java.util.List;
 
 
 @Data
-public class Gpu {
+public class GpuDTO {
     //名称
     private String name;
     //总内存
@@ -21,7 +21,7 @@ public class Gpu {
      */
     private int usageRate;
     //进程信息
-    private List<Process> processes;
+    private List<ProcessDTO> processDTOS;
  
     public String getName() {
         return name;
@@ -63,11 +63,11 @@ public class Gpu {
         this.usageRate = usageRate;
     }
  
-    public List<Process> getProcesses() {
-        return processes;
+    public List<ProcessDTO> getProcessDTOS() {
+        return processDTOS;
     }
  
-    public void setProcesses(List<Process> processes) {
-        this.processes = processes;
+    public void setProcessDTOS(List<ProcessDTO> processDTOS) {
+        this.processDTOS = processDTOS;
     }
 }

api-common/src/main/java/api/common/pojo/dto/Process.java → api-common/src/main/java/api/common/pojo/dto/ProcessDTO.java

@@ -1,6 +1,6 @@
 package api.common.pojo.dto;
 
-public class Process {
+public class ProcessDTO {
     private String pid;
     private String name;
     private String usedMemory;

api-common/src/main/java/api/common/pojo/po/home/SystemServerPO.java

@@ -15,14 +15,17 @@ public class SystemServerPO extends CommonPO {
     String serverId;
     String serverAddress;
     String serverType;
-    Integer cpuUsage;
     Integer memoryUsage;
     Double memoryAvailable;
+    Integer memoryTotal;
     Integer diskUsage;
     Double diskAvailable;
-    Integer taskNumber;
-    Integer weight;
+    Integer diskTotal;
+    Integer cpuUsage;
+    Integer cpuTotal;   // cpu 核数
     Integer gpuUsage;
+    Integer gpuTotal;   // gpu 内存数
+    Integer taskNumber;
 
 
 }

api-common/src/main/java/api/common/util/SshUtil.java

@@ -1,8 +1,8 @@
 package api.common.util;
 
 
-import api.common.pojo.dto.Gpu;
-import api.common.pojo.dto.Process;
+import api.common.pojo.dto.GpuDTO;
+import api.common.pojo.dto.ProcessDTO;
 import org.apache.sshd.client.SshClient;
 import org.apache.sshd.client.channel.ChannelExec;
 import org.apache.sshd.client.channel.ClientChannelEvent;
@@ -164,44 +164,44 @@ public class SshUtil {
     /**
      * 功能：CPU使用信息
      */
-    public static List<Gpu> gpuInfo(ClientSession session) throws IOException, DocumentException {
+    public static List<GpuDTO> gpuInfo(ClientSession session) throws IOException, DocumentException {
         String execute = SshUtil.execute(session, "nvidia-smi -q -x");
         String REG = "<!DOCTYPE.*.dtd\">";
         execute = execute.replaceAll(REG, "");
         Document document = DocumentHelper.parseText(execute);
         List<Element> gpu = document.getRootElement().elements("gpu");
-        List<Gpu> gpuList = new ArrayList<>();
+        List<GpuDTO> gpuDTOList = new ArrayList<>();
         gpu.forEach(element -> {
-            Gpu gpuInfo = new Gpu();
+            GpuDTO gpuDTOInfo = new GpuDTO();
             String uuid = element.element("uuid").getText();
             Element fbMemoryUsage = element.element("fb_memory_usage");
             String total = fbMemoryUsage.element("total").getText();
             String used = fbMemoryUsage.element("used").getText();
             String free = fbMemoryUsage.element("free").getText();
-            gpuInfo.setTotalMemory(total);
-            gpuInfo.setUsedMemory(used);
-            gpuInfo.setFreeMemory(free);
-            gpuInfo.setName(uuid);
+            gpuDTOInfo.setTotalMemory(total);
+            gpuDTOInfo.setUsedMemory(used);
+            gpuDTOInfo.setFreeMemory(free);
+            gpuDTOInfo.setName(uuid);
             Element processes = element.element("processes");
             List<Element> infos = processes.elements("process_info");
-            List<Process> processInfos = new ArrayList<>();
+            List<ProcessDTO> processDTOInfos = new ArrayList<>();
             infos.forEach(info -> {
-                Process process = new Process();
+                ProcessDTO processDTO = new ProcessDTO();
                 String pid = info.element("pid").getText();
                 String name = info.element("process_name").getText();
                 String usedMemory = info.element("used_memory").getText();
-                process.setPid(pid);
-                process.setName(name);
-                process.setUsedMemory(usedMemory);
-                processInfos.add(process);
+                processDTO.setPid(pid);
+                processDTO.setName(name);
+                processDTO.setUsedMemory(usedMemory);
+                processDTOInfos.add(processDTO);
             });
-            gpuInfo.setProcesses(processInfos);
+            gpuDTOInfo.setProcessDTOS(processDTOInfos);
             int intTotal = Integer.parseInt(total.split(" ")[0]);
             int intUsed = Integer.parseInt(used.split(" ")[0]);
-            gpuInfo.setUsageRate((int) ((float) intUsed / intTotal * 100));
-            gpuList.add(gpuInfo);
+            gpuDTOInfo.setUsageRate((int) ((float) intUsed / intTotal * 100));
+            gpuDTOList.add(gpuDTOInfo);
         });
-        return gpuList;
+        return gpuDTOList;
     }
 
 

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/scheduler/MyScheduler.java

@@ -1,18 +1,15 @@
 package com.css.simulation.resource.monitor.scheduler;
 
-import api.common.pojo.dto.Gpu;
 import api.common.pojo.dto.Host;
 import api.common.pojo.po.home.SystemServerPO;
 import api.common.util.SshUtil;
 import api.common.util.StringUtil;
 import lombok.Data;
 import org.apache.sshd.client.session.ClientSession;
-import org.dom4j.DocumentException;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
-import java.io.IOException;
 import java.util.List;
 
 @Component
@@ -37,7 +34,7 @@ public class MyScheduler {
             try {
                 ClientSession session = SshUtil.getSession(ip, username, password);
 //                if ("gpu".equals(type)){
-//                    List<Gpu> gpuList = SshUtil.gpuInfo(session);
+//                    List<GpuDTO> gpuList = SshUtil.gpuInfo(session);
 //                    gpuList.forEach(gpu -> {
 //                        gpu.get
 //                    });

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/kubernetes/KubernetesConfiguration.java

@@ -5,6 +5,7 @@ import io.kubernetes.client.util.ClientBuilder;
 import io.kubernetes.client.util.KubeConfig;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.util.ResourceUtils;
 
 import java.io.File;
 import java.io.FileReader;
@@ -15,9 +16,9 @@ public class KubernetesConfiguration {
 
     @Bean
     public ApiClient apiClient() throws IOException {
-//        File config = ResourceUtils.getFile("classpath:kubernetes/config");  // 开发环境可用，生产环境不行，无法从jar 包读取
+        File config = ResourceUtils.getFile("classpath:kubernetes/config");  // 开发环境可用，生产环境不行，无法从jar 包读取
 //        File config = new File("D:\\idea-project\\simulation-cloud\\simulation-resource-scheduler\\src\\main\\resources\\kubernetes\\config");  //windows
-        File config = new File("/root/.kube/config");   //linux
+//        File config = new File("/root/.kube/config");   //linux
 //
 //        ClassPathResource classPathResource = new ClassPathResource("kubernetes/config");
 //        InputStream inputStream = classPathResource.getInputStream();

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/controller/TaskController.java

@@ -65,6 +65,14 @@ public class TaskController {
         taskService.taskState(taskId, state, podName);
     }
 
+    /**
+     * 修改任务状态
+     */
+    @GetMapping("/stateTest")
+    public void taskStateTest(@RequestParam("taskId") String taskId, @RequestParam("state") String state, @RequestParam("podName") String podName) {
+        taskService.taskStateTest(taskId, state, podName);
+    }
+
     /**
      * 任务执行前调用该接口，确定该任务没有被终止
      */

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java

@@ -124,7 +124,7 @@ public class TaskService {
         for (int i = 0; i < leafIndexTemplateList.size(); i++) {
             IndexTemplatePO indexTemplatePO = leafIndexTemplateList.get(i);
             String indexId = indexTemplatePO.getIndexId();
-            log.info("------- /state 开始执行对第 " + i + " 个叶子节点 " + indexId + " 进行打分！");
+            log.info("------- /state 开始执行对第 " + (i+1) + " 个叶子节点 " + indexId + " 进行打分！");
             String ruleName = indexTemplatePO.getRuleName();    // 打分脚本名称，例如 AEB_1-1
             String ruleDetails = indexTemplatePO.getRuleDetails();    // 打分脚本内容
             String ruleFilePath = pyPath + "script/" + ruleName.split("_")[0] + "/" + ruleName + ".py";
@@ -246,6 +246,155 @@ public class TaskService {
     }
 
 
+    @SneakyThrows
+    public void taskStateTest(String taskId, String state, String podName) {
+        log.info("------- /state 修改任务 " + taskId + "的状态：" + state + "，pod 名称为：" + podName);
+        taskMapper.updateState(taskId, state);
+
+        ProjectPO projectPO = projectMapper.selectById(taskId);
+        String projectId = projectPO.getId();
+        String scenePackageId = projectPO.getScenePackageId();  // 场景测试包 id，指标根 id
+        log.info("------- /state 任务 " + taskId + " 的父项目为：" + projectId);
+        int taskNum = projectMapper.selectTaskNumById(projectId);
+        int endTaskNum = projectMapper.selectEndTaskNum(projectId);    // 查询已结束的任务 'Aborted', 'PendingAnalysis', 'Terminated'
+        projectMapper.updateTaskCompleted(projectId, endTaskNum);
+        log.info("------- /state 项目 " + projectId + " 完成进度为：" + endTaskNum + "/" + taskNum);
+        if (taskNum != endTaskNum) {  // 已结束任务数等于所有任务数量，才会准备打分；否则退出。
+            return;
+        }
+        projectMapper.updateProjectState(projectId, DictConstants.PROJECT_COMPLETED);   // 修改该 project 的状态为已完成
+        List<TaskPO> taskList = taskMapper.selectTaskListByProjectId(projectId);  // 所有任务信息
+        // -------------------------------- 查询叶子指标 --------------------------------
+        List<IndexTemplatePO> leafIndexTemplateList = indexTemplateMapper.selectLeafIndexWithRuleDetailsByPackageId(scenePackageId);
+        List<TaskIndexPO> leafTaskIndexList = new ArrayList<>();
+        log.info("------- /state 共有 " + leafIndexTemplateList.size() + "个叶子节点！");
+
+        ClientSession session = SshUtil.getSession(hostname, username, password);
+        for (int i = 0; i < leafIndexTemplateList.size(); i++) {
+            IndexTemplatePO indexTemplatePO = leafIndexTemplateList.get(i);
+            String indexId = indexTemplatePO.getIndexId();
+            log.info("------- /state 开始执行对第 " + (i+1) + " 个叶子节点 " + indexId + " 进行打分！");
+            String ruleName = indexTemplatePO.getRuleName();    // 打分脚本名称，例如 AEB_1-1
+            String ruleDetails = indexTemplatePO.getRuleDetails();    // 打分脚本内容
+            String ruleFilePath = pyPath + "script/" + ruleName.split("_")[0] + "/" + ruleName + ".py";
+            log.info("------- /state 将叶子节点 " + indexId + " 对应的打分规则保存到临时目录：" + ruleFilePath);
+            FileUtil.writeInputStreamToLocalFile(IoUtil.stringToInputStream(ruleDetails), ruleFilePath);
+            Set<String> sceneIdSet = new HashSet<>();
+            String naturalIds = indexTemplatePO.getSceneNaturalIds();
+            String standardIds = indexTemplatePO.getSceneStatueIds();
+            String accidentIds = indexTemplatePO.getSceneTrafficIds();
+            if (StringUtil.isNotEmpty(naturalIds)) {
+                String[] naturalIdArray = naturalIds.split(",");
+                sceneIdSet.addAll(Arrays.asList(naturalIdArray));
+            }
+            if (StringUtil.isNotEmpty(standardIds)) {
+                String[] standardArray = standardIds.split(",");
+                sceneIdSet.addAll(Arrays.asList(standardArray));
+            }
+            if (StringUtil.isNotEmpty(accidentIds)) {
+                String[] accidentIdArray = accidentIds.split(",");
+                sceneIdSet.addAll(Arrays.asList(accidentIdArray));
+            }
+            int resultNumberOfCurrentIndex = sceneIdSet.size();
+            log.info("------- /state 叶子节点 " + indexId + " 包括 " + resultNumberOfCurrentIndex + " 个场景！");
+            log.info("------- /state 计算叶子节点 " + indexId + " 的得分！");
+            double sum = taskList.stream()
+                    .filter(task1 -> sceneIdSet.contains(task1.getSceneId()))
+                    .mapToDouble(task2 -> {
+                        String task2Id = task2.getId();
+                        taskMapper.updateState(task2Id, DictConstants.TASK_ANALYSING);
+                        // 计算每个任务的得分
+                        ScoreTO score;
+                        String runResultMinio = task2.getRunResultFilePath() + "/Ego.csv";
+                        String runResultLinux = "D:\\css-project\\仿真云平台\\Ego.csv";
+//                        python3 /home/ubuntu/test/Evaluate/main.py /home/ubuntu/test/test_data.csv 4 AEB_1-2
+//                        String command = "python3 " + pyPath + " " + runResultLinux + " " + task2.getSceneType();  // 默认使用场景名称找打分脚本
+                        String command = "python3 " + pyPath + "main.py " + runResultLinux + " " + task2.getSceneType() + " " + ruleName; // 指定打分脚本
+                        try {
+                            try {
+                                log.info("------- /state 下载 minio 上的结果文件 " + runResultMinio + " 到本地：" + runResultLinux);
+                                MinioUtil.downloadToFile(minioClient, bucketName, runResultMinio, runResultLinux);
+                            } catch (Exception e) {
+                                throw new RuntimeException("------- /state 下载 minio 上的结果文件出错：" + e.getMessage());
+                            }
+                            try {
+                                log.info("------- /state 开始执行打分命令：" + command);
+                                score = JsonUtil.jsonToBean(SshUtil.execute(session, command), ScoreTO.class);
+                                log.info("------- /state 打分结束，结果为：" + score);
+                            } catch (IOException e) {
+                                throw new RuntimeException("------- /state 任务 " + task2Id + " 打分出错，命令为：" + command + " 修改状态为：" + DictConstants.TASK_ABORTED + "\n" + e.getMessage());
+                            }
+                        } catch (Exception e) {
+                            taskMapper.updateState(task2Id, DictConstants.TASK_ABORTED);
+                            throw new RuntimeException(e.getMessage());
+                        }
+                        task2.setReturnSceneId(score.getUnit_scene_ID());
+                        task2.setScore(score.getUnit_scene_score());
+                        task2.setTargetEvaluate(score.getEvaluate_item());
+                        task2.setScoreExplain(score.getScore_description());
+                        task2.setModifyUserId(USER_ID);
+                        task2.setModifyTime(TimeUtil.getNowForMysql());
+                        return score.getUnit_scene_score();
+                    }).sum();
+            // 计算不合格的任务数（不到100分就是不合格）
+            long notStandardSceneNum = taskList.stream().filter(task1 -> sceneIdSet.contains(task1.getSceneId()) && task1.getScore() < 100).count();
+            // 叶子指标的得分（叶子指标下所有场景得分的平均值）
+            double leafIndexScore = sum / resultNumberOfCurrentIndex;
+            // -------------------------------- 保存叶子指标得分 --------------------------------
+            indexTemplatePO.setTempScore(leafIndexScore);
+            TaskIndexPO leafTaskIndex = TaskIndexPO.builder()
+                    .id(StringUtil.getRandomUUID())
+                    .pId(projectId)
+                    .target(indexTemplatePO.getIndexId())
+                    .notStandardSceneNum((int) notStandardSceneNum)
+                    .score(leafIndexScore)
+                    .build();
+            leafTaskIndex.setCreateUserId(USER_ID);
+            leafTaskIndex.setCreateTime(TimeUtil.getNowForMysql());
+            leafTaskIndex.setModifyUserId(USER_ID);
+            leafTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
+            leafTaskIndex.setIsDeleted("0");
+            leafTaskIndexList.add(leafTaskIndex);
+        }
+        SshUtil.stop(session);
+
+        // 根据每个指标的得分和权重算出 project 的总得分。
+        double totalScore = compute(leafIndexTemplateList);
+        // 保存分数
+        // 保存任务分数
+        taskManager.batchUpdateByScoreResult(taskList);
+        // 保存指标分数
+        taskIndexManager.batchInsertLeafIndex(leafTaskIndexList);
+        // 保存总分数
+        TaskIndexPO totalTaskIndex = TaskIndexPO.builder()
+                .id(StringUtil.getRandomUUID())
+                .pId(projectId)
+                .target(scenePackageId)
+                .score(totalScore)
+                .build();
+        totalTaskIndex.setCreateUserId(USER_ID);
+        totalTaskIndex.setCreateTime(TimeUtil.getNowForMysql());
+        totalTaskIndex.setModifyUserId(USER_ID);
+        totalTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
+        totalTaskIndex.setIsDeleted("0");
+        taskIndexMapper.insertTotalIndex(totalTaskIndex);
+
+        // 调用 server 的接口，计算评价等级
+        String tokenUrl = tokenUri + "?grant_type=client_credentials"
+                + "&client_id=" + clientId
+                + "client_secret" + clientSecret;
+        String response = HttpUtil.get(closeableHttpClient, requestConfig, tokenUrl);
+        ObjectMapper objectMapper = new ObjectMapper();
+        JsonNode jsonNode = objectMapper.readTree(response);
+        String accessToken = jsonNode.path("access_token").asText();
+        Map<String, String> headers = new HashMap<>();
+        headers.put("Authorization", "Bearer " + accessToken);
+        HttpUtil.post(closeableHttpClient, requestConfig, evaluationLevelUri, headers, null);
+
+
+    }
+
+
     public Boolean taskConfirm(String taskId) {
         // 查询 task 如果不是 pending 则不执行
         String state = taskMapper.selectStateById(taskId);