mlx

simulation-resource-scheduler/pom.xml

@@ -142,12 +142,12 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.13.2</version>
-            <scope>test</scope>
-        </dependency>
+<!--        <dependency>-->
+<!--            <groupId>junit</groupId>-->
+<!--            <artifactId>junit</artifactId>-->
+<!--            <version>4.13.2</version>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
     </dependencies>
     <build>
         <plugins>

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ProjectConsumer.java

@@ -29,6 +29,22 @@ import java.util.Set;
 @Slf4j
 public class ProjectConsumer {
 
+    @Value("${scheduler.minio-path.project-result}")
+    String resultPathMinio;
+    @Value("${scheduler.linux-path.job-template}")
+    String jobTemplate;
+    @Value("${scheduler.linux-path.job-yaml}")
+    String jobYaml;
+
+    @Value("${scheduler.host.hostname}")
+    String hostname;
+    @Value("${scheduler.host.username}")
+    String username;
+    @Value("${scheduler.host.password}")
+    String password;
+
+    // -------------------------------- Comment --------------------------------
+
     @Autowired
     KafkaTemplate<String, String> kafkaTemplate;
     @Autowired
@@ -63,19 +79,7 @@ public class ProjectConsumer {
     ManualProjectService manualProjectService;
     @Autowired
     ProjectUtil projectUtil;
-    @Value("${scheduler.project.result-path-minio}")
-    String resultPathMinio;
-    @Value("${scheduler.project.job-template}")
-    String jobTemplate;
-    @Value("${scheduler.project.job-yaml}")
-    String jobYaml;
 
-    @Value("${scheduler.score.hostname}")
-    String hostname;
-    @Value("${scheduler.score.username}")
-    String username;
-    @Value("${scheduler.score.password}")
-    String password;
 
 
     /**

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/manager/TaskManager.java

@@ -34,20 +34,30 @@ import java.util.stream.Collectors;
 @Component
 @Slf4j
 public class TaskManager {
+    @Value("${minio.bucket-name}")
+    String bucketName;
+    @Value("${scheduler.linux-path.score-py}")
+    String pyPath;
+    @Value("${scheduler.linux-path.temp}")
+    String linuxTempPath;
+    @Value("${simulation-cloud.client-id}")
+    String clientId;
+    @Value("${simulation-cloud.client-secret}")
+    String clientSecret;
+    @Value("${simulation-cloud.token-uri}")
+    String tokenUri;
+    @Value("${simulation-cloud.evaluation-level-uri}")
+    String evaluationLevelUri;
+    @Value("${scheduler.minio-path.project-result}")
+    String resultPathMinio;
     @Autowired
     ClusterMapper clusterMapper;
     @Autowired
     StringRedisTemplate stringRedisTemplate;
-    @Value("${scheduler.manual-project.topic}")
-    String manualProjectTopic;
     @Autowired
     TaskMapper taskMapper;
-    @Value("${scheduler.manual-project.result-path-minio}")
-    String resultPathMinio;
     @Autowired
     MinioClient minioClient;
-    @Value("${minio.bucket-name}")
-    String bucketName;
     @Autowired
     ManualProjectMapper manualProjectMapper;
     @Autowired
@@ -56,18 +66,6 @@ public class TaskManager {
     TaskIndexManager taskIndexManager;
     @Autowired
     IndexMapper indexMapper;
-    @Value("${scheduler.score.py-path}")
-    String pyPath;
-    @Value("${scheduler.linux-temp-path}")
-    String linuxTempPath;
-    @Value("${simulation-cloud.client-id}")
-    String clientId;
-    @Value("${simulation-cloud.client-secret}")
-    String clientSecret;
-    @Value("${simulation-cloud.token-uri}")
-    String tokenUri;
-    @Value("${simulation-cloud.evaluation-level-uri}")
-    String evaluationLevelUri;
     @Autowired
     CloseableHttpClient closeableHttpClient;
     @Autowired

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/ProjectScheduler.java

@@ -34,6 +34,15 @@ import java.util.Set;
 @Slf4j
 public class ProjectScheduler {
 
+    @Value("${scheduler.host.hostname}")
+    String hostname;
+    @Value("${scheduler.host.username}")
+    String username;
+    @Value("${scheduler.host.password}")
+    String password;
+    @Value("${scheduler.linux-path.job-yaml}")
+    String jobYaml;
+    // -------------------------------- Comment --------------------------------
     @Autowired
     StringRedisTemplate redisTemplate;
     @Autowired
@@ -44,14 +53,6 @@ public class ProjectScheduler {
     ClusterMapper clusterMapper;
     @Autowired
     ManualProjectMapper manualProjectMapper;
-    @Value("${scheduler.project.job-yaml}")
-    String jobYaml;
-    @Value("${scheduler.score.hostname}")
-    String hostname;
-    @Value("${scheduler.score.username}")
-    String username;
-    @Value("${scheduler.score.password}")
-    String password;
     @Autowired
     ApiClient apiClient;
     @Autowired

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/AlgorithmService.java

@@ -21,7 +21,7 @@ public class AlgorithmService {
     MinioClient minioClient;
     @Value("${minio.bucket-name}")
     String bucketName;
-    @Value("${scheduler.linux-temp-path}")
+    @Value("${scheduler.linux-path.temp}")
     String linuxTempPath;
 
     @SneakyThrows

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ManualProjectService.java

@@ -14,7 +14,6 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.ibatis.session.SqlSessionFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
@@ -33,25 +32,30 @@ import java.util.stream.Collectors;
 @Slf4j
 public class ManualProjectService {
 
-    @Autowired
-    private SqlSessionFactory sqlSessionFactory;
+    @Value("${scheduler.minio-path.project-result}")
+    String projectResultPathOfMinio;
+    @Value("${scheduler.linux-temp-path}")
+    String linuxTempPath;
+    @Value("${minio.bucket-name}")
+    String bucketName;
+    @Value("${scheduler.score.hostname}")
+    String hostname;
+    @Value("${scheduler.score.username}")
+    String username;
+    @Value("${scheduler.score.password}")
+    String password;
+
+    // -------------------------------- Comment --------------------------------
     @Autowired
     StringRedisTemplate stringRedisTemplate;
-    @Value("${scheduler.manual-project.result-path-minio}")
-    String resultPathMinio;
     @Autowired
     KafkaTemplate<String, String> kafkaTemplate;
-    @Value("${scheduler.linux-temp-path}")
-    String linuxTempPath;
     @Autowired
     CloseableHttpClient closeableHttpClient;
     @Autowired
     RequestConfig requestConfig;
     @Autowired
     MinioClient minioClient;
-    @Value("${minio.bucket-name}")
-    String bucketName;
-
     @Autowired
     ManualProjectMapper manualProjectMapper;
     @Autowired
@@ -67,13 +71,6 @@ public class ManualProjectService {
     @Autowired
     AlgorithmMapper algorithmMapper;
 
-    @Value("${scheduler.score.hostname}")
-    String hostname;
-    @Value("${scheduler.score.username}")
-    String username;
-    @Value("${scheduler.score.password}")
-    String password;
-
     // -------------------------------- Comment --------------------------------
 
     @Transactional
@@ -158,7 +155,7 @@ public class ManualProjectService {
                         .sceneName(scenePO.getName())
                         .sceneType(scenePO.getType())
                         .runState(DictConstants.TASK_PENDING)
-                        .runResultFilePath(resultPathMinio + projectId + "/" + taskId)
+                        .runResultFilePath(projectResultPathOfMinio + projectId + "/" + taskId)
                         .build();
                 taskPO.setCreateTime(TimeUtil.getNowForMysql());
                 taskPO.setCreateUserId(userId);

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java

@@ -25,24 +25,26 @@ import org.springframework.stereotype.Service;
 public class TaskService {
 
 
+    @Value("${minio.bucket-name}")
+    String bucketName;
+    @Value("${scheduler.host.hostname}")
+    String hostname;
+    @Value("${scheduler.host.username}")
+    String username;
+    @Value("${scheduler.host.password}")
+    String password;
+    @Value("${spring.kafka.delete-command}")
+    String kafkaDeleteCommand;
+    // -------------------------------- Comment --------------------------------
+
     @Autowired
     StringRedisTemplate stringRedisTemplate;
     @Autowired
     TaskManager taskManager;
     @Autowired
     IndexTemplateMapper indexTemplateMapper;
-    @Value("${scheduler.score.hostname}")
-    String hostname;
-    @Value("${scheduler.score.username}")
-    String username;
-    @Value("${scheduler.score.password}")
-    String password;
-    @Value("${spring.kafka.delete-command}")
-    String kafkaDeleteCommand;
     @Autowired
     MinioClient minioClient;
-    @Value("${minio.bucket-name}")
-    String bucketName;
     @Autowired
     TaskMapper taskMapper;
     @Autowired

simulation-resource-scheduler/src/test/java/com/css/simulation/resource/scheduler/SchedulerTest.java

@@ -1,191 +0,0 @@
-package com.css.simulation.resource.scheduler;
-
-
-import api.common.util.CollectionUtil;
-import api.common.util.StringUtil;
-import api.common.util.TimeUtil;
-import com.css.simulation.resource.scheduler.mapper.IndexTemplateMapper;
-import com.css.simulation.resource.scheduler.mapper.IndexMapper;
-import com.css.simulation.resource.scheduler.mapper.TaskMapper;
-import com.css.simulation.resource.scheduler.pojo.po.IndexTemplatePO;
-import com.css.simulation.resource.scheduler.pojo.po.LeafIndexPO;
-import io.kubernetes.client.openapi.ApiClient;
-import io.kubernetes.client.openapi.ApiException;
-import lombok.extern.slf4j.Slf4j;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.data.redis.core.StringRedisTemplate;
-import org.springframework.test.context.junit4.SpringRunner;
-
-import java.util.*;
-import java.util.stream.Collectors;
-
-
-@Slf4j
-@SpringBootTest
-@RunWith(SpringRunner.class)
-public class SchedulerTest {
-
-    @Autowired
-    IndexMapper indexMapper;
-    @Autowired
-    TaskMapper taskMapper;
-    @Autowired
-    IndexTemplateMapper indexTemplateMapper;
-    @Autowired
-    StringRedisTemplate redisTemplate;
-    @Autowired
-    ApiClient apiClient;
-
-    private final String USER_ID = "simulation-resource-scheduler";
-
-
-    @Test
-    public void redisTemplate() throws ApiException {
-        redisTemplate.opsForValue().set("a:b", "test");
-        redisTemplate.opsForValue().set("a:b:c", "test");
-    }
-
-    @Test
-    public void compute() {
-
-        /*
-[LeafIndexPO(id=3783c4757af54a289c35d9a5da05d1aa, pId=0af13ea53152480fa692efb04e6b45ee, indexId=105df6a0d1de4bb7ba08d02338a64a67, parentId=4241063c9c3846a2982c6383df9e2661, rootId=d6fbf031543541ee8f688e68ce42796a, target=105df6a0d1de4bb7ba08d02338a64a67, notStandardSceneNum=0, score=0.0, weight=30, scoreExplain=null, packageLevel=4),
-LeafIndexPO(id=466eebd8a3c44441b401692f2ff68a4d, pId=0af13ea53152480fa692efb04e6b45ee, indexId=19cd6ea94c0e4ae5b79c65bb1d242b58, parentId=bc733db732e449c9858631e97331d49c, rootId=d6fbf031543541ee8f688e68ce42796a, target=19cd6ea94c0e4ae5b79c65bb1d242b58, notStandardSceneNum=0, score=100.0, weight=55, scoreExplain=1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。, packageLevel=5),
-LeafIndexPO(id=b07f85a2334e48dfb864032fb1cae94f, pId=0af13ea53152480fa692efb04e6b45ee, indexId=4a45b92909064f0c97a058262a4a8881, parentId=4c27f4e9318843a2911f054e8df6a39c, rootId=d6fbf031543541ee8f688e68ce42796a, target=4a45b92909064f0c97a058262a4a8881, notStandardSceneNum=0, score=100.0, weight=40, scoreExplain=1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。, packageLevel=3),
-LeafIndexPO(id=36a119d748b64f23b5bd7120e8e6e2d7, pId=0af13ea53152480fa692efb04e6b45ee, indexId=58a8c47500954c41b3c4ec8ffe553da4, parentId=1abb262bdd684645b757283c8641b406, rootId=d6fbf031543541ee8f688e68ce42796a, target=58a8c47500954c41b3c4ec8ffe553da4, notStandardSceneNum=0, score=0.0, weight=40, scoreExplain=null, packageLevel=3),
-LeafIndexPO(id=af4cc25bc998455eae6efdad6779534e, pId=0af13ea53152480fa692efb04e6b45ee, indexId=72ec17bcc5234cb3a5c623ed2cecf77a, parentId=0628958aa4584a41bdbcdc4e97afb151, rootId=d6fbf031543541ee8f688e68ce42796a, target=72ec17bcc5234cb3a5c623ed2cecf77a, notStandardSceneNum=0, score=0.0, weight=70, scoreExplain=null, packageLevel=2),
-LeafIndexPO(id=72066cb6d0b043159d0afdcb3eb18514, pId=0af13ea53152480fa692efb04e6b45ee, indexId=8bd982970a3843ffb40373b638d80b8f, parentId=d6fbf031543541ee8f688e68ce42796a, rootId=d6fbf031543541ee8f688e68ce42796a, target=8bd982970a3843ffb40373b638d80b8f, notStandardSceneNum=0, score=0.0, weight=40, scoreExplain=null, packageLevel=1),
-LeafIndexPO(id=bd1964e9701e4591a7c064b4ac1134bd, pId=0af13ea53152480fa692efb04e6b45ee, indexId=8f54b0ac76754ee6b1383236cee6a9da, parentId=bc733db732e449c9858631e97331d49c, rootId=d6fbf031543541ee8f688e68ce42796a, target=8f54b0ac76754ee6b1383236cee6a9da, notStandardSceneNum=2, score=0.0, weight=45, scoreExplain=1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。, packageLevel=5),
-LeafIndexPO(id=d53b2b01edb84683bfcf01f038f035d3, pId=0af13ea53152480fa692efb04e6b45ee, indexId=9afc65292007439ba3b7e8f11fa61417, parentId=4241063c9c3846a2982c6383df9e2661, rootId=d6fbf031543541ee8f688e68ce42796a, target=9afc65292007439ba3b7e8f11fa61417, notStandardSceneNum=0, score=0.0, weight=70, scoreExplain=null, packageLevel=4),
-LeafIndexPO(id=2b5eda8b6811479eb0c101118235a93f, pId=0af13ea53152480fa692efb04e6b45ee, indexId=9f4dea73c08e40fda5a50c445f22ed36, parentId=1f18b24d33424236bec09b8ced88be33, rootId=d6fbf031543541ee8f688e68ce42796a, target=9f4dea73c08e40fda5a50c445f22ed36, notStandardSceneNum=1, score=0.0, weight=70, scoreExplain=1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。, packageLevel=2),
-LeafIndexPO(id=b882d0b3aed84737ae176a52ab6bd761, pId=0af13ea53152480fa692efb04e6b45ee, indexId=be9814fedc0341ea9441f44f2fcd8757, parentId=d6fbf031543541ee8f688e68ce42796a, rootId=d6fbf031543541ee8f688e68ce42796a, target=be9814fedc0341ea9441f44f2fcd8757, notStandardSceneNum=0, score=100.0, weight=40, scoreExplain=1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。, packageLevel=1),
- LeafIndexPO(id=2e2d91683b07428183012631ef60736f, pId=0af13ea53152480fa692efb04e6b45ee, indexId=f96f144f451a4dd298e0be9dfbeebf3a, parentId=40acb88fbe6344b6bbb34e697278968d, rootId=d6fbf031543541ee8f688e68ce42796a, target=f96f144f451a4dd298e0be9dfbeebf3a, notStandardSceneNum=1, score=50.0, weight=30, scoreExplain=1) 未发生碰撞，得分 100；
-2) 发生碰撞，得分 0。, packageLevel=4)]
-         */
-
-
-        List<LeafIndexPO> leafTaskIndexList = new ArrayList<>();
-
-        LeafIndexPO leafIndexPO1 = LeafIndexPO.builder()
-                .id("0a8cd6c061a24ed2bbb36a45b3a003d4")
-                .pId("e8337795555541639659d773e28eafd5")
-                .indexId("433838e404a74f87ba8f78c617134eec")
-                .parentId("018ae110ca51413ba22268f36c125a03")
-                .rootId("018ae110ca51413ba22268f36c125a03")
-                .target("433838e404a74f87ba8f78c617134eec")
-                .notStandardSceneNum(0)
-                .score(100.0)
-                .weight("40")
-                .scoreExplain("1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。)")
-                .packageLevel(1)
-                .build();
-        leafTaskIndexList.add(leafIndexPO1);
-
-        LeafIndexPO leafIndexPO2 = LeafIndexPO.builder()
-                .id("24a53877c1b34b3bb61dc9d7e28035d2")
-                .pId("e8337795555541639659d773e28eafd5")
-                .indexId("a2a1866e5c7047c3bddc905971540043")
-                .parentId("148c7e1fb6474cdf8c106215f92b0d9d")
-                .rootId("018ae110ca51413ba22268f36c125a03")
-                .target("a2a1866e5c7047c3bddc905971540043")
-                .notStandardSceneNum(1)
-                .score(0.0)
-                .weight("70")
-                .scoreExplain("1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。)")
-                .packageLevel(2)
-                .build();
-        leafTaskIndexList.add(leafIndexPO2);
-
-        LeafIndexPO leafIndexPO3 = LeafIndexPO.builder()
-                .id("94a9225873534e97be79b25c98861b83")
-                .pId("e8337795555541639659d773e28eafd5")
-                .indexId("df913a81a34742808cefc76829bfa398")
-                .parentId("e28c9687cfdd4fec956d4c3a7fa061ec")
-                .rootId("018ae110ca51413ba22268f36c125a03")
-                .target("df913a81a34742808cefc76829bfa398")
-                .notStandardSceneNum(0)
-                .score(100.0)
-                .weight("60")
-                .scoreExplain("1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。)")
-                .packageLevel(3)
-                .build();
-        leafTaskIndexList.add(leafIndexPO3);
-
-
-        LeafIndexPO leafIndexPO4 = LeafIndexPO.builder()
-                .id("14d0017b489844bcb33118f7cfeee04c")
-                .pId("e8337795555541639659d773e28eafd5")
-                .indexId("eefbf9234bad410b801ed860b4add56f")
-                .parentId("e28c9687cfdd4fec956d4c3a7fa061ec")
-                .rootId("018ae110ca51413ba22268f36c125a03")
-                .target("eefbf9234bad410b801ed860b4add56f")
-                .notStandardSceneNum(1)
-                .score(0.0)
-                .weight("40")
-                .scoreExplain("1) 未发生碰撞，得分 100；2) 发生碰撞，得分 0。)")
-                .packageLevel(3)
-                .build();
-        leafTaskIndexList.add(leafIndexPO4);
-
-
-        computeFirst(leafTaskIndexList, "e8337795555541639659d773e28eafd5", 3);
-    }
-
-    public void computeFirst(List<LeafIndexPO> leafTaskIndexList, String projectId, int maxLevel) {
-
-        log.info("------- /state computeFirst 计算父指标得分：" + leafTaskIndexList);
-        Iterator<LeafIndexPO> leafTaskIndexIterator = leafTaskIndexList.iterator();
-        // 把 1 级的指标得分直接保存
-        while (leafTaskIndexIterator.hasNext()) {
-            LeafIndexPO leafTaskIndex = leafTaskIndexIterator.next();
-            if (leafTaskIndex.getPackageLevel() == 1) {
-                leafTaskIndex.setCreateUserId(USER_ID);
-                leafTaskIndex.setCreateTime(TimeUtil.getNowForMysql());
-                leafTaskIndex.setModifyUserId(USER_ID);
-                leafTaskIndex.setModifyTime(TimeUtil.getNowForMysql());
-                leafTaskIndex.setIsDeleted("0");
-                indexMapper.insertFirstIndex(leafTaskIndex);
-                leafTaskIndexIterator.remove();
-            }
-        }
-        if (leafTaskIndexList.size() > 0) {
-
-            List<LeafIndexPO> nextLevelIndexList = new ArrayList<>();
-            // 找出等级和 maxLevel 不相同的指标暂时不计算
-            leafTaskIndexList.stream()
-                    .filter(po -> maxLevel != po.getPackageLevel())
-                    .forEach(nextLevelIndexList::add);
-            // 找出等级和 maxLevel 相同的指标并根据父指标分组
-            Map<String, List<LeafIndexPO>> sonTaskIndexMap = leafTaskIndexList.stream()
-                    .filter(po -> maxLevel == po.getPackageLevel())
-                    .collect(Collectors.groupingBy(LeafIndexPO::getParentId));
-            Set<String> parentIdSet = sonTaskIndexMap.keySet();
-            List<String> parentIdList = CollectionUtil.setToList(parentIdSet);
-            List<IndexTemplatePO> parentIndexTemplateList = indexTemplateMapper.selectByIdList(parentIdList);
-            // 计算父指标得分
-            parentIndexTemplateList.forEach(indexTemplate -> {
-                String weight = indexTemplate.getWeight();
-                List<LeafIndexPO> sonTaskIndexList = sonTaskIndexMap.get(indexTemplate.getIndexId());
-                double parentScore = sonTaskIndexList.stream().mapToDouble(taskIndex -> taskIndex.getScore() * Double.parseDouble(taskIndex.getWeight()) / 100).sum();
-                LeafIndexPO parentTaskIndex = LeafIndexPO.builder()
-                        .id(StringUtil.getRandomUUID())
-                        .pId(projectId)
-                        .target(indexTemplate.getIndexId())
-                        .score(parentScore)
-                        .indexId(indexTemplate.getIndexId())
-                        .parentId(indexTemplate.getParentId())
-                        .rootId(indexTemplate.getRootId())
-                        .weight(weight)
-                        .packageLevel(maxLevel - 1)
-                        .build();
-                nextLevelIndexList.add(parentTaskIndex);
-            });
-            // 将父指标作为叶子指标递归
-            computeFirst(nextLevelIndexList, projectId, maxLevel - 1);
-        }
-    }
-
-}