标准化测试

api-common/src/main/java/api/common/pojo/constants/DictConstants.java

@@ -3,6 +3,8 @@ package api.common.pojo.constants;
 @SuppressWarnings("all")
 public class DictConstants {
 
+    public static final String PROJECT_RUN_TYPE_EXECUTE = "1";  // 等待执行
+    public static final String PROJECT_RUN_TYPE_EXPAND = "2";  // 等待扩充
     public static final String PROJECT_WAIT_QUEUE_KEY = "project-wait-queue";  // 等待执行
     public static final String PROJECT_WAIT_TYPE_EXECUTE = "1";  // 等待执行
     public static final String PROJECT_WAIT_TYPE_EXPAND = "2";  // 等待扩充

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/application/entity/ProjectWaitQueueEntity.java

@@ -1,12 +1,15 @@
 package com.css.simulation.resource.scheduler.application.entity;
 
 import com.css.simulation.resource.scheduler.api.model.ProjectMessageModel;
+import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 @Data
 @EqualsAndHashCode
+@Builder
 public class ProjectWaitQueueEntity {
     private String waitingType; //1等待执行 2等待扩充
+    private Integer waitingParallelism; // 等待扩充或执行的并行度
     private ProjectMessageModel projectMessageModel; // 项目详情
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/application/service/ProjectService.java

@@ -124,7 +124,7 @@ public class ProjectService {
         //1 创建任务文件并固定场景数据
         createTaskAndFixData(projectMessageModel);
         //2 校验证书和并行度
-        checkIfCanRun(projectMessageModel);
+        checkIfCanRun(projectMessageModel, DictConstants.PROJECT_RUN_TYPE_EXECUTE);
     }
 
     /**
@@ -324,107 +324,110 @@ public class ProjectService {
      * 任务运行前首先判断用户是否拥有可分配资源
      *
      * @param projectMessageModel 项目启动消息
+     * @param runType             运行类型 1执行 2扩充
      */
     @SneakyThrows
-    public void checkIfCanRun(ProjectMessageModel projectMessageModel) {
-        log.debug("判断用户是否拥有可分配资源：" + projectMessageModel);
-        //1 读取 kafka 的 project 信息
-        final String modelType = projectMessageModel.getModelType();
-        String projectId = projectMessageModel.getProjectId();    // 手动执行项目 id 或 自动执行子项目 id
-        long parallelism = projectMessageModel.getParallelism();   // 项目并行度
-        String projectType = projectMessageModel.getType(); // 项目类型
-        final String isChoiceGpu = projectDomainService.getIsChoiceGpuByProjectId(projectId);
-        final int remainderParallelism = projectDomainService.getRemainderParallelism(isChoiceGpu);
-        //2 获取用户信息（管理员账户、管理员子账户、普通账户、普通子账户）（独占、共享）
-        final UserEntity userEntity = projectDomainService.getUserEntityByProjectIdAndProjectType(projectId, projectType);
-        String projectUserId = userEntity.getId();
-        log.debug("项目 " + projectId + " 的创建人为：" + userEntity);
-        String roleCode = userEntity.getRoleCode();
-        String useType = userEntity.getUseType();
-        ClusterEntity clusterEntity;
-        String clusterUserId;  // 项目实际运行使用的用户集群
-        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
-            clusterUserId = DictConstants.SYSTEM_USER_ID;
-            log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为管理员账户或管理员子账户，直接判断服务器能否执行。");
-            PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(DictConstants.SYSTEM_CLUSTER_ID, projectId);
-            final String projectRunningKey = redisPrefix.getProjectRunningKey();
-            if (remainderParallelism <= 0) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXECUTE, projectMessageModel);
-            } else if (remainderParallelism < parallelism) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXPAND, projectMessageModel);
-                run(clusterUserId, remainderParallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
-            } else {
-                run(clusterUserId, (int) parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
-            }
-            return;
-        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
-            clusterUserId = projectUserId;
-            clusterEntity = clusterMapper.selectByUserId(clusterUserId);
-            log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通账户（包括独占或共享都在自己的集群），集群为：" + clusterEntity);
-        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
-            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
+    public void checkIfCanRun(ProjectMessageModel projectMessageModel, String runType) {
+        if (DictConstants.PROJECT_RUN_TYPE_EXECUTE.equals(runType)) {
+            log.debug("判断用户是否拥有可分配资源：" + projectMessageModel);
+            //1 读取 kafka 的 project 信息
+            String modelType = projectMessageModel.getModelType();
+            String projectId = projectMessageModel.getProjectId();    // 手动执行项目 id 或 自动执行子项目 id
+            int parallelism = projectMessageModel.getParallelism();   // 项目并行度
+            String projectType = projectMessageModel.getType(); // 项目类型
+            String isChoiceGpu = projectDomainService.getIsChoiceGpuByProjectId(projectId);
+            int remainderParallelism = projectDomainService.getRemainderParallelism(isChoiceGpu);
+            //2 获取用户信息（管理员账户、管理员子账户、普通账户、普通子账户）（独占、共享）
+            final UserEntity userEntity = projectDomainService.getUserEntityByProjectIdAndProjectType(projectId, projectType);
+            String projectUserId = userEntity.getId();
+            log.debug("项目 " + projectId + " 的创建人为：" + userEntity);
+            String roleCode = userEntity.getRoleCode();
+            String useType = userEntity.getUseType();
+            ClusterEntity clusterEntity;
+            String clusterUserId;  // 项目实际运行使用的用户集群
+            if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
+                clusterUserId = DictConstants.SYSTEM_USER_ID;
+                log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为管理员账户或管理员子账户，直接判断服务器能否执行。");
+                PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(DictConstants.SYSTEM_CLUSTER_ID, projectId);
+                final String projectRunningKey = redisPrefix.getProjectRunningKey();
+                if (remainderParallelism <= 0) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectMessageModel(projectMessageModel).build());
+                } else if (remainderParallelism < parallelism) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - remainderParallelism).projectMessageModel(projectMessageModel).build());
+                    run(clusterUserId, remainderParallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                } else {
+                    run(clusterUserId, parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                }
+                return;
+            } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
                 clusterUserId = projectUserId;
                 clusterEntity = clusterMapper.selectByUserId(clusterUserId);
-                log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通独占子账户（自己的集群），集群为：" + clusterEntity);
-            } else if (DictConstants.USER_TYPE_PUBLIC.equals(useType)) {    //3-4 共享子账户，根据父账户的共享节点排队
-                clusterUserId = userEntity.getCreateUserId();
-                clusterEntity = clusterMapper.selectByUserId(clusterUserId);
-                log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通共享子账户（父账户的集群），集群为：" + clusterEntity);
-            } else {
-                throw new RuntimeException("用户" + projectUserId + "未知占用类型：" + useType);
-            }
-        } else {
-            throw new RuntimeException("未知角色类型：" + roleCode);
-        }
-        // 获取仿真软件证书数量和动力学软件证书数量（vtd占一个仿真证书，carsim各占一个）
-        PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(clusterEntity.getId(), projectId);
-        final Integer usingSimulationLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_SIMULATION);
-        final Integer usingDynamicLicenseNumber;
-        final Integer numSimulationLicense = clusterEntity.getNumSimulationLicense();
-        final Integer numDynamicLicense = clusterEntity.getNumDynamicLicense();
-        final String projectRunningKey = redisPrefix.getProjectRunningKey();
-        //1 判断仿真证书是否够用，如果证书为0则将项目加入等待队列；如果证书小于并行度则加入扩充队列，并用现有证书执行；如果证书够用，直接执行。
-        final int remainderSimulationLicense = numSimulationLicense - usingSimulationLicenseNumber;
-        final int remainderDynamicLicense;
-
-        // 如果执行后的并行度总和小于最大节点数则执行，否则不执行
-        if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
-            if (remainderSimulationLicense <= 0 || remainderParallelism <= 0) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXECUTE, projectMessageModel);
-            } else if (remainderSimulationLicense < parallelism || remainderParallelism <= parallelism) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXPAND, projectMessageModel);
-                run(clusterUserId, Math.min(remainderSimulationLicense, remainderParallelism), projectMessageModel, projectRunningKey, isChoiceGpu);
+                log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通账户（包括独占或共享都在自己的集群），集群为：" + clusterEntity);
+            } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
+                if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
+                    clusterUserId = projectUserId;
+                    clusterEntity = clusterMapper.selectByUserId(clusterUserId);
+                    log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通独占子账户（自己的集群），集群为：" + clusterEntity);
+                } else if (DictConstants.USER_TYPE_PUBLIC.equals(useType)) {    //3-4 共享子账户，根据父账户的共享节点排队
+                    clusterUserId = userEntity.getCreateUserId();
+                    clusterEntity = clusterMapper.selectByUserId(clusterUserId);
+                    log.debug("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通共享子账户（父账户的集群），集群为：" + clusterEntity);
+                } else {
+                    throw new RuntimeException("用户" + projectUserId + "未知占用类型：" + useType);
+                }
             } else {
-                run(clusterUserId, (int) parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                throw new RuntimeException("未知角色类型：" + roleCode);
             }
-        } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
-            usingDynamicLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_DYNAMIC);
-            remainderDynamicLicense = numDynamicLicense - usingDynamicLicenseNumber;
-            if (remainderSimulationLicense <= 0 || remainderDynamicLicense <= 0 || remainderParallelism <= 0) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXECUTE, projectMessageModel);
-            } else if (remainderSimulationLicense < parallelism || remainderDynamicLicense < parallelism || remainderParallelism < parallelism) {
-                wait(DictConstants.PROJECT_WAIT_TYPE_EXPAND, projectMessageModel);
-                run(clusterUserId, Math.min(remainderSimulationLicense, Math.min(remainderDynamicLicense, remainderParallelism)), projectMessageModel, projectRunningKey, isChoiceGpu);
+            // 获取仿真软件证书数量和动力学软件证书数量（vtd占一个仿真证书，carsim各占一个）
+            PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(clusterEntity.getId(), projectId);
+            final Integer usingSimulationLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_SIMULATION);
+            final Integer usingDynamicLicenseNumber;
+            final Integer numSimulationLicense = clusterEntity.getNumSimulationLicense();
+            final Integer numDynamicLicense = clusterEntity.getNumDynamicLicense();
+            final String projectRunningKey = redisPrefix.getProjectRunningKey();
+            //1 判断仿真证书是否够用，如果证书为0则将项目加入等待队列；如果证书小于并行度则加入扩充队列，并用现有证书执行；如果证书够用，直接执行。
+            final int remainderSimulationLicense = numSimulationLicense - usingSimulationLicenseNumber;
+            final int remainderDynamicLicense;
+
+            // 如果执行后的并行度总和小于最大节点数则执行，否则不执行
+            if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
+                if (remainderSimulationLicense <= 0 || remainderParallelism <= 0) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectMessageModel(projectMessageModel).build());
+                } else if (remainderSimulationLicense < parallelism || remainderParallelism <= parallelism) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - Math.min(remainderSimulationLicense, remainderParallelism)).projectMessageModel(projectMessageModel).build());
+                    run(clusterUserId, Math.min(remainderSimulationLicense, remainderParallelism), projectMessageModel, projectRunningKey, isChoiceGpu);
+                } else {
+                    run(clusterUserId, parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                }
+            } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
+                usingDynamicLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_DYNAMIC);
+                remainderDynamicLicense = numDynamicLicense - usingDynamicLicenseNumber;
+                if (remainderSimulationLicense <= 0 || remainderDynamicLicense <= 0 || remainderParallelism <= 0) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectMessageModel(projectMessageModel).build());
+                } else if (remainderSimulationLicense < parallelism || remainderDynamicLicense < parallelism || remainderParallelism < parallelism) {
+                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - Math.min(Math.min(remainderSimulationLicense, remainderDynamicLicense), remainderParallelism)).projectMessageModel(projectMessageModel).build());
+                    run(clusterUserId, Math.min(remainderSimulationLicense, Math.min(remainderDynamicLicense, remainderParallelism)), projectMessageModel, projectRunningKey, isChoiceGpu);
+                } else {
+                    run(clusterUserId, parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                }
             } else {
-                run(clusterUserId, (int) parallelism, projectMessageModel, projectRunningKey, isChoiceGpu);
+                throw new RuntimeException("未知模型类型：" + modelType);
             }
+        } else if (DictConstants.PROJECT_RUN_TYPE_EXPAND.equals(runType)) {
+
         } else {
-            throw new RuntimeException("未知模型类型：" + modelType);
+            throw new RuntimeException("未知运行类型：" + runType);
         }
+
     }
 
     //* -------------------------------- 等待 --------------------------------
 
     /**
-     * @param waitType            等待类型 1等待执行 2等待扩充
-     * @param projectMessageModel 项目信息
+     * @param projectWaitQueueEntity 等待对象
      */
-    public void wait(String waitType, ProjectMessageModel projectMessageModel) {
+    public void wait(ProjectWaitQueueEntity projectWaitQueueEntity) {
         try {
-            //1 创建等待队列元素对象
-            final ProjectWaitQueueEntity projectWaitQueueEntity = new ProjectWaitQueueEntity();
-            projectWaitQueueEntity.setWaitingType(waitType);
-            projectWaitQueueEntity.setProjectMessageModel(projectMessageModel);
             //2 创建等待列表对象
             final String waitingQueueJson = customRedisClient.get(DictConstants.PROJECT_WAIT_QUEUE_KEY);
             List<ProjectWaitQueueEntity> waitingQueue;
@@ -481,7 +484,8 @@ public class ProjectService {
         KafkaUtil.createTopic(kafkaAdminClient, projectId, finalParallelism, (short) 1);   // 创建主题
         TimeUnit.SECONDS.sleep(7);
         // 需要即时启动的任务（并行度的大小）
-        CopyOnWriteArrayList<String> yamlToRunRedisKeyList = new CopyOnWriteArrayList<>();
+        ArrayList<String> yamlToRunRedisKeyList = new ArrayList<>();
+        ArrayList<String> yamlToWaitRedisKeyList = new ArrayList<>();
         for (String taskJsonPath : taskJsonList) {
             String taskId = FileUtil.getFilenameWithoutSuffix(taskJsonPath);
             // 保存运行中的任务信息
@@ -529,7 +533,7 @@ public class ProjectService {
 
             if (currentCount == 0) {
                 String podName = yamlRedisKey.split(":")[yamlRedisKey.split(":").length - 1];
-                log.info("将 pod 加入到启动列表 " + podName);
+                log.info("将 POD ：{} 加入到启动列表 ", podName);
                 yamlToRunRedisKeyList.add(yamlRedisKey);
             }
             messageNumber++;
@@ -628,18 +632,18 @@ public class ProjectService {
             }
 
             if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
-                log.info("createTempYaml() k8s参数为：" + kubernetesConfiguration);
-                log.info("createTempYaml() yaml模板为：" + replace12);
+                log.info("k8s参数为：" + kubernetesConfiguration);
+                log.info("yaml模板为：" + replace12);
                 String replace20 = replace19.replace("vtd-image", kubernetesConfiguration.getImageVtdGpu());
                 finalYaml = replace20.replace("vtd-command", kubernetesConfiguration.getCommandVtdCarsimGpu());
             } else if (DictConstants.NOT_USE_GPU.equals(isChoiceGpu)) {
                 String replace20 = replace19.replace("vtd-image", kubernetesConfiguration.getImageVtdNogpu());
                 finalYaml = replace20.replace("vtd-command", kubernetesConfiguration.getCommandVtdCarsimNogpu());
             } else {
-                throw new RuntimeException("createTempYaml() 是否使用 gpu：" + isChoiceGpu);
+                throw new RuntimeException("是否使用 gpu：" + isChoiceGpu);
             }
         } else {
-            throw new RuntimeException("createTempYaml() 模型类型错误：" + modelType);
+            throw new RuntimeException("模型类型错误：" + modelType);
         }
         log.info("保存项目 " + projectId + " 的 yaml 文件：" + yamlPath);
         FileUtil.writeStringToLocalFile(finalYaml, yamlPath);
@@ -736,17 +740,17 @@ public class ProjectService {
                 }
                 algorithmMapper.updateDockerImportAndDockerImageById("1", dockerImage, algorithmId);
                 if (algorithmTarLinuxTempPath == null) {
-                    throw new RuntimeException("handleAlgorithm 算法" + algorithmId + "下载失败。");
+                    throw new RuntimeException("算法" + algorithmId + "下载失败。");
                 }
                 LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
                 LinuxUtil.execute("docker push " + dockerImage);
                 FileUtil.rm(algorithmTarLinuxTempPath);
-                log.info("handleAlgorithm 已删除算法临时文件：" + algorithmTarLinuxTempPath);
+                log.info("已删除算法临时文件：" + algorithmTarLinuxTempPath);
             } else {
                 throw new RuntimeException("算法 " + algorithmId + " 的 mysql 数据有误！");
             }
         } else {
-            log.info("handleAlgorithm 项目" + projectId + "需要使用索为平台算法 " + algorithmId);
+            log.info("项目" + projectId + "需要使用索为平台算法 " + algorithmId);
             algorithmTarLinuxTempPath = linuxTempPath + "algorithm/" + algorithmId + ".tar";
             String dockerImageWithoutVersion = dockerConfiguration.getRegistry() + "/algorithm_" + algorithmId;
             dockerImage = dockerImageWithoutVersion + ":latest";
@@ -767,9 +771,9 @@ public class ProjectService {
                 LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
                 LinuxUtil.execute("docker push " + dockerImage);
                 FileUtil.rm(algorithmTarLinuxTempPath);
-                log.info("handleAlgorithm 已删除算法临时文件：" + algorithmTarLinuxTempPath);
+                log.info("已删除算法临时文件：" + algorithmTarLinuxTempPath);
             } else {
-                log.info("handleAlgorithm 算法镜像" + dockerImageWithoutVersion + "已导入。");
+                log.info("算法镜像" + dockerImageWithoutVersion + "已导入。");
             }
         }
         log.info("项目 " + projectId + " 使用的算法镜像为：" + dockerImage);

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/application/service/TaskService.java

@@ -57,6 +57,14 @@ public class TaskService {
 
     // -------------------------------- Comment --------------------------------
 
+    /**
+     * state 接口不能阻塞，因为会导致 pod 里调用接口超时
+     *
+     * @param taskId  任务id
+     * @param state   任务状态
+     * @param podName podName
+     */
+    @Async
     public void state(String taskId, String state, String podName) {
         TaskEntity taskEntity = taskMapper.selectById(taskId);
         if (taskEntity == null) {

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/infrastructure/scheduler/ProjectScheduler.java

@@ -12,7 +12,6 @@ import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
-import java.util.Collection;
 import java.util.List;
 
 @Component
@@ -26,16 +25,16 @@ public class ProjectScheduler {
     /**
      * 调度项目启动
      */
-    @Scheduled(fixedDelay = 60000)
+    @Scheduled(fixedDelay = 1000)
     public void dispatchProject() {
         List<ProjectWaitQueueEntity> projectWaitQueue = projectDomainService.getWaitQueue();
         if (CollectionUtil.isNotEmpty(projectWaitQueue)) {
             ProjectWaitQueueEntity projectWaitQueueEntity = projectWaitQueue.get(0);
             String waitingType = projectWaitQueueEntity.getWaitingType();
             if (DictConstants.PROJECT_WAIT_TYPE_EXECUTE.equals(waitingType)) {
-                log.info("尝试启动等待中的项目：{}", projectWaitQueueEntity);
+//                log.info("尝试启动等待中的项目：{}", projectWaitQueueEntity);
                 final ProjectMessageModel projectMessageModel = projectWaitQueueEntity.getProjectMessageModel();
-                projectService.checkIfCanRun(projectMessageModel);
+                projectService.checkIfCanRun(projectMessageModel, DictConstants.PROJECT_RUN_TYPE_EXECUTE);
             } else if (DictConstants.PROJECT_WAIT_TYPE_EXPAND.equals(waitingType)) {
                 log.info("尝试扩充运行中的项目：{}", projectWaitQueueEntity);
                 ProjectMessageModel projectMessageModel = projectWaitQueueEntity.getProjectMessageModel();
@@ -45,7 +44,9 @@ public class ProjectScheduler {
                     log.info("等待扩充的项目" + projectWaitQueueEntity + "已运行完成，无需扩充。");
                     projectWaitQueue.remove(0);
                     projectDomainService.setWaitQueue(projectWaitQueue);
+                    return;
                 }
+                projectService.checkIfCanRun(projectMessageModel, DictConstants.PROJECT_RUN_TYPE_EXPAND);
             }
         }
     }