package com.css.simulation.resource.scheduler.app.service;

import api.common.pojo.constants.DictConstants;
import api.common.pojo.enums.MultiSimulationStatusEnum;
import api.common.pojo.enums.MultiSimulationTaskStatusEnum;
import api.common.pojo.param.project.MultiCreateYamlRet;
import api.common.pojo.param.project.MultiSimulationProjectKafkaParam;
import api.common.pojo.param.project.MultiSimulationProjectParam;
import api.common.pojo.param.project.MultiSimulationSceneKafkaParam;
import api.common.pojo.po.project.MultiSimulationProjectTaskRecordPO;
import api.common.pojo.vo.map.SimulationMapVO;
import api.common.pojo.vo.project.MultiSimulationProjectVO;
import api.common.pojo.vo.project.MultiSimulationSceneCarVO;
import api.common.util.*;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.css.simulation.resource.scheduler.adapter.entity.ProjectStartMessageEntity;
import com.css.simulation.resource.scheduler.adapter.entity.ProjectStopMessageEntity;
import com.css.simulation.resource.scheduler.app.entity.MultiProjectWaitQueueEntity;
import com.css.simulation.resource.scheduler.app.entity.ProjectWaitQueueEntity;
import com.css.simulation.resource.scheduler.app.entity.VehicleEntity;
import com.css.simulation.resource.scheduler.domain.service.ProjectDomainService;
import com.css.simulation.resource.scheduler.domain.service.TaskDomainService;
import com.css.simulation.resource.scheduler.infra.configuration.custom.CustomConfiguration;
import com.css.simulation.resource.scheduler.infra.configuration.docker.DockerConfiguration;
import com.css.simulation.resource.scheduler.infra.configuration.entity.DynamicsEntity;
import com.css.simulation.resource.scheduler.infra.configuration.git.GitConfiguration;
import com.css.simulation.resource.scheduler.infra.configuration.git.GitUtil;
import com.css.simulation.resource.scheduler.infra.db.entity.AlgorithmExpandEntity;
import com.css.simulation.resource.scheduler.infra.db.mysql.mapper.*;
import com.css.simulation.resource.scheduler.infra.db.redis.CustomRedisClient;
import com.css.simulation.resource.scheduler.infra.db.redis.RedisUtil;
import com.css.simulation.resource.scheduler.infra.entity.*;
import com.css.simulation.resource.scheduler.infra.fs.minio.MinioUtil;
import com.css.simulation.resource.scheduler.infra.mq.kafka.KafkaUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.minio.MinioClient;
import lombok.SneakyThrows;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.admin.Admin;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;

import javax.annotation.Resource;
import java.io.File;
import java.io.InputStream;
import java.math.BigDecimal;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

@Service
@Slf4j
public class ProjectApplicationService {

    @Value("${scheduler.minio-path.project-result}")
    private String projectResultPathOfMinio;
    @Value("${scheduler.linux-path.temp}")
    private String linuxTempPath;
    @Value("${scheduler.linux-path.pod-yaml-directory}")
    private String podYamlDirectory;

    @Value("${scheduler.linux-path.multi-pod-yaml-directory}")
    private String multiPodYamlDirectory;
    @Value("${scheduler.linux-path.multi-vtd-xml-generator}")
    private String multiVtdXmlGenerator;
    @Value("${minio.bucket-name}")
    private String bucketName;

    // -------------------------------- Comment --------------------------------
    @Resource
    private MinioClient minioClient;
    @Resource
    private StringRedisTemplate stringRedisTemplate;
    @Resource
    private VehicleMapper vehicleMapper;
    @Resource
    private SensorCameraMapper sensorCameraMapper;
    @Resource
    private SensorOgtMapper sensorOgtMapper;
    @Resource
    private AlgorithmMapper algorithmMapper;
    @Resource
    private ClusterMapper clusterMapper;
    @Resource
    private IndexMapper indexMapper;
    @Resource
    private SimulationManualProjectTaskMapper simulationManualProjectTaskMapper;
    @Resource
    private TaskDomainService taskDomainService;
    @Resource
    private KafkaTemplate<String, String> kafkaTemplate;
    @Resource(name = "myKafkaAdmin")
    private Admin kafkaAdminClient;
    @Resource
    private CustomConfiguration customConfiguration;
    @Resource
    private ProjectDomainService projectDomainService;
    @Resource
    private IndexTemplateMapper indexTemplateMapper;
    @Resource
    private SceneMapper sceneMapper;
    @Resource
    private DockerConfiguration dockerConfiguration;
    @Resource
    private GitConfiguration gitConfiguration;
    @Resource
    private SimulationManualProjectMapper simulationManualProjectMapper;
    @Resource
    private SimulationAutomaticSubprojectMapper simulationAutomaticSubprojectMapper;
    @Resource
    private CustomRedisClient customRedisClient;
    @Resource
    private AlgorithmExpandMapper algorithmExpandMapper;
    @Resource
    private SimulationMapMapper mapMapper;
    @Resource
    private MultiSimulationProjectMapper multiSimulationProjectMapper;

    @Resource
    private MultiSimulationProjectTaskRecordMapper taskRecordMapper;
    @Resource
    private MultiSimulationProjectResultMapper resultMapper;


    // -------------------------------- Comment --------------------------------

    @Async("pool1")
    public void runProject(ProjectStartMessageEntity projectStartMessageEntity) {
        //1 创建任务文件并固定场景数据
        createTaskAndFixData(projectStartMessageEntity);
        //2 校验证书和并行度
        checkIfCanRun(
                ProjectWaitQueueEntity.builder()
                        .waitingType(DictConstants.PROJECT_RUN_TYPE_EXECUTE)
                        .waitingParallelism(projectStartMessageEntity.getParallelism())
                        .projectStartMessageEntity(projectStartMessageEntity)
                        .build()
        );
    }

    /**
     * 接收到运行信息立即复制一份数据作为据运行数
     *
     * @param projectStartMessageEntity 项目启动消息
     */
    public void createTaskAndFixData(ProjectStartMessageEntity projectStartMessageEntity) {
        //* -------------------------------- 0 读取消息，创建临时目录 --------------------------------
        String projectId = projectStartMessageEntity.getProjectId();                // 手动执行项目 id 或 自动执行子项目 id
        String projectType = projectStartMessageEntity.getType();                   // 项目类型
        try {
            String modelType = projectStartMessageEntity.getModelType();                // 模型类型，1 动力学模型 2 carsim模型
            String packageId = projectStartMessageEntity.getScenePackageId();           // 场景测试包 id
            String vehicleConfigId = projectStartMessageEntity.getVehicleConfigId();    // 模型配置 id
            String algorithmId = projectStartMessageEntity.getAlgorithmId();            // 模型配置 id
            long videoTime = projectStartMessageEntity.getMaxSimulationTime();          // 结果视频的时长
            String userId = projectDomainService.getUserIdByProjectIdAndProjectType(projectId, projectType);
            String projectPath = linuxTempPath + "project/" + projectId + "/";
            FileUtil.mkdir(projectPath);
            //5 将该 project 下所有旧的指标得分删除。
            simulationManualProjectTaskMapper.deleteByProject(projectId);
            indexMapper.deleteFirstTargetScoreByProjectId(projectId);
            indexMapper.deleteLastTargetScoreByProjectId(projectId);
            // -------------------------------- 1 查询场景 --------------------------------
            log.info("项目 " + projectId + " 开始查询场景。");
            //根据场景测试包 packageId，拿到场景集合（包括重复场景），重复场景会在发送消息时根据叶子指标发送多次。
            List<SceneEntity> sceneEntityList = getSceneList(projectId, packageId);
            int taskTotal = sceneEntityList.size();
            projectStartMessageEntity.setTaskTotal(taskTotal);
            projectStartMessageEntity.setTaskCompleted(0);
            //去重，之后发送消息的时候会补全指标，如果不去重的话会出现多个场景重复关联多个指标
            Set<SceneEntity> sceneEntitySet = new HashSet<>(sceneEntityList);
            log.info("项目 " + projectId + " 场景包括：" + sceneEntitySet);
            // -------------------------------- 2 算法导入 --------------------------------
            log.info("项目 " + projectId + " 开始算法导入。");
            String algorithmDockerImage = handleAlgorithm(projectId, algorithmId);
            log.info("项目 " + projectId + " 算法已导入：" + algorithmDockerImage);
            customRedisClient.set("project:" + projectId + ":docker-image", algorithmDockerImage);
            // -------------------------------- 3 查询模型 --------------------------------
            log.info("项目 " + projectId + " 开始查询模型。");
            //1 根据车辆配置id vehicleConfigId, 获取 模型信息和传感器信息
            com.css.simulation.resource.scheduler.infra.entity.VehicleEntity vehicleEntity = vehicleMapper.selectByVehicleConfigId(vehicleConfigId);   // 车辆
            List<CameraEntity> cameraEntityList = sensorCameraMapper.selectCameraByVehicleConfigId(vehicleConfigId);    // 摄像头
            List<OgtEntity> ogtEntityList = sensorOgtMapper.selectOgtByVehicleId(vehicleConfigId); // 完美传感器
            log.info("项目 " + projectId + " 开始保存任务消息。");
            if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
                // -------------------------------- 4 保存任务消息 --------------------------------
                List<TaskEntity> taskList = new ArrayList<>();
                for (SceneEntity sceneEntity : sceneEntitySet) {
                    String sceneId = sceneEntity.getId();
                    //3-1 可能会存在多个指标下有同样的场景，所以会查出多个指标，多个指标的场景需要发送多次
                    List<String> lastTargetIdList = null;
                    if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
                        lastTargetIdList = indexMapper.selectLeafIndexIdByManualProjectIdAndSceneId(projectId, "%" + sceneId + "%");
                    } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
                        lastTargetIdList = indexMapper.selectLeafIndexIdByAutoSubProjectIdAndSceneId(projectId, "%" + sceneId + "%");
                    }
                    if (CollectionUtil.isEmpty(lastTargetIdList)) {
                        throw new RuntimeException("项目 " + projectId + " 使用的场景测试包 " + packageId + " 不存在场景：" + sceneId);
                    }
                    for (String lastTargetId : lastTargetIdList) {
                        String taskId = StringUtil.getRandomUUID();
                        // 保存任务信息
                        TaskEntity taskEntity = TaskEntity.builder() // run_start_time 和 run_end_time 不填
                                .id(taskId).pId(projectId).sceneId(sceneId).lastTargetId(lastTargetId).sceneName(sceneEntity.getName()).sceneType(sceneEntity.getType()).runState(DictConstants.TASK_PENDING).runResultFilePath(projectResultPathOfMinio + projectId + "/" + taskId).build();
                        taskEntity.setCreateTime(TimeUtil.getNowForMysql());
                        taskEntity.setCreateUserId(userId);
                        taskEntity.setModifyTime(TimeUtil.getNowForMysql());
                        taskEntity.setModifyUserId(userId);
                        taskEntity.setModifyTime(TimeUtil.getNowForMysql());
                        taskEntity.setIsDeleted("0");
                        taskList.add(taskEntity);
                        // 将 xosc、xodr、osgb 全部上传到仿真结果路径
                        String scenarioOsc = sceneEntity.getScenarioOsc();
                        String[] splitXosc = scenarioOsc.split("/");
                        String xoscName = splitXosc[splitXosc.length - 1];
                        String[] xoscNameSplit = xoscName.split("\\.");
                        String xoscSuffix = xoscNameSplit[xoscNameSplit.length - 1];
                        String xoscPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + xoscName;
                        String xoscPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + xoscSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOsc, xoscPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, xoscPathOfLinux, bucketName, xoscPathOfMinio);
                        FileUtil.rm(xoscPathOfLinux);   // 删除临时文件

                        String scenarioOdr = sceneEntity.getScenarioOdr();
                        String[] splitXodr = scenarioOdr.split("/");
                        String xodrName = splitXodr[splitXodr.length - 1];
                        String[] xodrNameSplit = xodrName.split("\\.");
                        String xodrSuffix = xodrNameSplit[xodrNameSplit.length - 1];
                        String xodrPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + xodrName;
                        String xodrPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + xodrSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOdr, xodrPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, xodrPathOfLinux, bucketName, xodrPathOfMinio);
                        FileUtil.rm(xodrPathOfLinux);   // 删除临时文件

                        String scenarioOsgb = sceneEntity.getScenarioOsgb();
                        String[] splitOsgb = scenarioOsgb.split("/");
                        String osgbName = splitOsgb[splitOsgb.length - 1];
                        String[] osgbNameSplit = osgbName.split("\\.");
                        String osgbSuffix = osgbNameSplit[osgbNameSplit.length - 1];
                        String osgbPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + osgbName;
                        String osgbPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + osgbSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOsgb, osgbPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, osgbPathOfLinux, bucketName, osgbPathOfMinio);
                        FileUtil.rm(osgbPathOfLinux);   // 删除临时文件

                        // 组装 task 消息
                        TaskMessageEntity taskMessageEntity = TaskMessageEntity.builder().info(InfoEntity.builder().project_id(taskEntity.getPId()).task_id(taskEntity.getId()).task_path(taskEntity.getRunResultFilePath()).default_time(videoTime).build()).scenario(ScenarioEntity.builder().scenario_osc(xoscPathOfMinio).scenario_odr(xodrPathOfMinio).scenario_osgb(osgbPathOfMinio).build()).vehicle(VehicleEntity.builder().model(ModelEntity.builder().model_label(vehicleEntity.getModelLabel()).build()).dynamics(DynamicsEntity.builder().dynamics_maxspeed(vehicleEntity.getMaxSpeed()).dynamics_enginepower(vehicleEntity.getEnginePower()).dynamics_maxdecel(vehicleEntity.getMaxDeceleration()).dynamics_maxsteering(vehicleEntity.getMaxSteeringAngle()).dynamics_mass(vehicleEntity.getMass()).dynamics_frontsurfaceeffective(vehicleEntity.getFrontSurfaceEffective()).dynamics_airdragcoefficient(vehicleEntity.getAirDragCoefficient()).dynamics_rollingresistance(vehicleEntity.getRollingResistanceCoefficient()).dynamics_wheeldiameter(vehicleEntity.getWheelDiameter()).dynamics_wheeldrive(vehicleEntity.getWheelDrive()).dynamics_overallefficiency(vehicleEntity.getOverallEfficiency()).dynamics_distfront(vehicleEntity.getFrontDistance()).dynamics_distrear(vehicleEntity.getRearDistance()).dynamics_distleft(vehicleEntity.getLeftDistance()).dynamics_distright(vehicleEntity.getRightDistance()).dynamics_distheight(vehicleEntity.getHeightDistance()).dynamics_wheelbase(vehicleEntity.getWheelbase()).build()).sensors(SensorsEntity.builder()   // 根据 vehicleId 查询绑定的传感器列表
                                .camera(cameraEntityList).OGT(ogtEntityList).build()).build()).build();
                        FileUtil.writeStringToLocalFile(JsonUtil.beanToJson(taskMessageEntity), projectPath + taskId + ".json");
                        log.info("项目 " + projectId + " 将任务消息转成 json 保存到临时目录等待资源分配后执行：" + taskMessageEntity.getInfo().getTask_id());
                    }
                }
                taskDomainService.batchInsertTask(taskList);
                log.info("项目 " + projectId + " 共有 " + taskList.size() + " 个任务，已保存到数据库");
            } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {

                List<TaskEntity> taskList = new ArrayList<>();
                for (SceneEntity sceneEntity : sceneEntitySet) {
                    String sceneId = sceneEntity.getId();
                    //3-1 可能会存在多个指标下有同样的场景，所以会查出多个指标，多个指标的场景需要发送多次
                    List<String> lastTargetIdList = null;
                    if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
                        lastTargetIdList = indexMapper.selectLeafIndexIdByManualProjectIdAndSceneId(projectId, "%" + sceneId + "%");
                    } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
                        lastTargetIdList = indexMapper.selectLeafIndexIdByAutoSubProjectIdAndSceneId(projectId, "%" + sceneId + "%");
                    }
                    if (CollectionUtil.isEmpty(lastTargetIdList)) {
                        throw new RuntimeException("项目 " + projectId + " 使用的场景测试包 " + sceneId + " 不存在指标。");
                    }
                    for (String lastTargetId : lastTargetIdList) {
                        String taskId = StringUtil.getRandomUUID();
                        // 保存任务信息
                        TaskEntity taskEntity = TaskEntity.builder() // run_start_time 和 run_end_time 不填
                                .id(taskId).pId(projectId).sceneId(sceneId).lastTargetId(lastTargetId).sceneName(sceneEntity.getName()).sceneType(sceneEntity.getType()).runState(DictConstants.TASK_PENDING).runResultFilePath(projectResultPathOfMinio + projectId + "/" + taskId).build();
                        taskEntity.setCreateTime(TimeUtil.getNowForMysql());
                        taskEntity.setCreateUserId(userId);
                        taskEntity.setModifyTime(TimeUtil.getNowForMysql());
                        taskEntity.setModifyUserId(userId);
                        taskEntity.setModifyTime(TimeUtil.getNowForMysql());
                        taskEntity.setIsDeleted("0");
                        taskList.add(taskEntity);
                        // 将 xosc、xodr、osgb 全部上传到仿真结果路径
                        String scenarioOsc = sceneEntity.getScenarioOsc();
                        String[] splitXosc = scenarioOsc.split("/");
                        String xoscName = splitXosc[splitXosc.length - 1];
                        String[] xoscNameSplit = xoscName.split("\\.");
                        String xoscSuffix = xoscNameSplit[xoscNameSplit.length - 1];
                        String xoscPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + xoscName;
                        String xoscPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + xoscSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOsc, xoscPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, xoscPathOfLinux, bucketName, xoscPathOfMinio);

                        String scenarioOdr = sceneEntity.getScenarioOdr();
                        String[] splitXodr = scenarioOdr.split("/");
                        String xodrName = splitXodr[splitXodr.length - 1];
                        String[] xodrNameSplit = xodrName.split("\\.");
                        String xodrSuffix = xodrNameSplit[xodrNameSplit.length - 1];
                        String xodrPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + xodrName;
                        String xodrPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + xodrSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOdr, xodrPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, xodrPathOfLinux, bucketName, xodrPathOfMinio);

                        String scenarioOsgb = sceneEntity.getScenarioOsgb();
                        String[] splitOsgb = scenarioOsgb.split("/");
                        String osgbName = splitOsgb[splitOsgb.length - 1];
                        String[] osgbNameSplit = osgbName.split("\\.");
                        String osgbSuffix = osgbNameSplit[osgbNameSplit.length - 1];
                        String osgbPathOfLinux = linuxTempPath + "video/" + projectId + "/" + taskId + "/" + osgbName;
                        String osgbPathOfMinio = projectResultPathOfMinio + projectId + "/" + taskId + "/" + taskId + "." + osgbSuffix;
                        MinioUtil.downloadToFile(minioClient, bucketName, scenarioOsgb, osgbPathOfLinux);
                        MinioUtil.uploadFromFile(minioClient, osgbPathOfLinux, bucketName, osgbPathOfMinio);
                        log.info("已经将 xosc、xodr、osgb 上传到 minio 的结果文件目录：" + projectResultPathOfMinio);

                        // 组装 task 消息
                        // carsim 不需要查询模型参数
                        TaskMessageEntity taskMessageEntity = TaskMessageEntity.builder().info(InfoEntity.builder().project_id(taskEntity.getPId()).task_id(taskEntity.getId()).task_path(taskEntity.getRunResultFilePath()).default_time(videoTime).build()).scenario(ScenarioEntity.builder().scenario_osc(xoscPathOfMinio).scenario_odr(xodrPathOfMinio).scenario_osgb(osgbPathOfMinio).build()).vehicle(VehicleEntity.builder().model(ModelEntity.builder().model_label(vehicleEntity.getModelLabel()).build()).dynamics(null).sensors(SensorsEntity.builder()   // 根据 vehicleId 查询绑定的传感器列表
                                .camera(cameraEntityList).OGT(ogtEntityList).build()).build()).build();

                        FileUtil.writeStringToLocalFile(JsonUtil.beanToJson(taskMessageEntity), projectPath + taskId + ".json");
                        log.info("项目 " + projectId + " 将任务消息转成 json 保存到临时目录等待资源分配后执行：" + taskMessageEntity.getInfo().getTask_id());
                    }
                }
                taskDomainService.batchInsertTask(taskList);
                log.info("项目 " + projectId + " 共有 " + taskList.size() + " 个任务，已保存到数据库");
            }

        } catch (Exception e) {
            log.error("项目报错。", e);
            throw new RuntimeException(e);
        }

    }

    /**
     * 任务运行前首先判断用户是否拥有可分配资源
     *
     * @param projectWaitQueueEntity 项目启动消息
     */
    @SneakyThrows
    public void checkIfCanRun(ProjectWaitQueueEntity projectWaitQueueEntity) {
        final String waitingType = projectWaitQueueEntity.getWaitingType();
        final Integer waitingParallelism = projectWaitQueueEntity.getWaitingParallelism();
        final ProjectStartMessageEntity projectStartMessageEntity = projectWaitQueueEntity.getProjectStartMessageEntity();
        log.info("判断用户是否拥有可分配资源：" + projectStartMessageEntity);
        //1 项目信息
        String modelType = projectStartMessageEntity.getModelType();
        String projectId = projectStartMessageEntity.getProjectId();    // 手动执行项目 id 或 自动执行子项目 id
        int parallelism = projectStartMessageEntity.getParallelism();   // 项目并行度
        String projectType = projectStartMessageEntity.getType(); // 项目类型
        String isChoiceGpu = projectDomainService.getIsChoiceGpuByProjectId(projectId);
        //2 剩余并行度
        int remainderParallelism = projectDomainService.getRemainderParallelism(isChoiceGpu);
        //3 用户信息（管理员账户、管理员子账户、普通账户、普通子账户）（独占、共享）
        final UserEntity userEntity = projectDomainService.getUserEntityByProjectIdAndProjectType(projectId, projectType);
        String projectUserId = userEntity.getId();
        String roleCode = userEntity.getRoleCode();
        String useType = userEntity.getUseType();
        ClusterEntity clusterEntity;
        String clusterUserId;  // 项目实际运行使用的用户集群
        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
            clusterUserId = DictConstants.SYSTEM_USER_ID;
            log.info("项目 " + projectId + " 的创建人 " + projectUserId + " 为管理员账户或管理员子账户，直接判断服务器能否执行。");
            PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(DictConstants.SYSTEM_CLUSTER_ID, projectId);
            final String projectRunningKey = redisPrefix.getProjectRunningKey();
            if (remainderParallelism <= 0) {
                wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectStartMessageEntity(projectStartMessageEntity).build());
            } else if (remainderParallelism < parallelism) {
                wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - remainderParallelism).projectStartMessageEntity(projectStartMessageEntity).build());
                run(clusterUserId, remainderParallelism, projectStartMessageEntity, projectRunningKey, isChoiceGpu);
            } else {
                run(clusterUserId, parallelism, projectStartMessageEntity, projectRunningKey, isChoiceGpu);
            }
            return;
        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
            clusterUserId = projectUserId;
            clusterEntity = clusterMapper.selectByUserId(clusterUserId);
            log.info("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通账户（包括独占或共享都在自己的集群），集群为：" + clusterEntity);
        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
                clusterUserId = projectUserId;
                clusterEntity = clusterMapper.selectByUserId(clusterUserId);
                log.info("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通独占子账户（自己的集群），集群为：" + clusterEntity);
            } else if (DictConstants.USER_TYPE_PUBLIC.equals(useType)) {    //3-4 共享子账户，根据父账户的共享节点排队
                clusterUserId = userEntity.getCreateUserId();
                clusterEntity = clusterMapper.selectByUserId(clusterUserId);
                log.info("项目 " + projectId + " 的创建人 " + projectUserId + " 为普通共享子账户（父账户的集群），集群为：" + clusterEntity);
            } else {
                throw new RuntimeException("用户" + projectUserId + "未知占用类型：" + useType);
            }
        } else {
            throw new RuntimeException("未知角色类型：" + roleCode);
        }

        // 获取仿真软件证书数量和动力学软件证书数量（vtd占一个仿真证书，carsim各占一个）
        PrefixEntity redisPrefix = projectDomainService.getRedisPrefixByClusterIdAndProjectId(clusterEntity.getId(), projectId);
        final Integer usingSimulationLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_SIMULATION);
        final Integer usingDynamicLicenseNumber;
        final Integer numSimulationLicense = clusterEntity.getNumSimulationLicense();
        final Integer numDynamicLicense = clusterEntity.getNumDynamicLicense();
        final String projectRunningKey = redisPrefix.getProjectRunningKey();
        // 判断仿真证书是否够用，如果证书为0则将项目加入等待队列；如果证书小于并行度则加入扩充队列，并用现有证书执行；如果证书够用，直接执行。
        final int remainderSimulationLicense = numSimulationLicense - usingSimulationLicenseNumber;
        final int remainderDynamicLicense;
        if (DictConstants.PROJECT_WAIT_TYPE_EXECUTE.equals(waitingType)) {
            if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
                if (remainderSimulationLicense <= 0 || remainderParallelism <= 0) {
                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectStartMessageEntity(projectStartMessageEntity).build());
                } else if (remainderSimulationLicense < parallelism || remainderParallelism <= parallelism) {
                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - Math.min(remainderSimulationLicense, remainderParallelism)).projectStartMessageEntity(projectStartMessageEntity).build());
                    run(clusterUserId, Math.min(remainderSimulationLicense, remainderParallelism), projectStartMessageEntity, projectRunningKey, isChoiceGpu);
                } else {
                    run(clusterUserId, parallelism, projectStartMessageEntity, projectRunningKey, isChoiceGpu);
                }
            } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
                usingDynamicLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_DYNAMIC);
                remainderDynamicLicense = numDynamicLicense - usingDynamicLicenseNumber;
                if (remainderSimulationLicense <= 0 || remainderDynamicLicense <= 0 || remainderParallelism <= 0) {
                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXECUTE).waitingParallelism(parallelism).projectStartMessageEntity(projectStartMessageEntity).build());
                } else if (remainderSimulationLicense < parallelism || remainderDynamicLicense < parallelism || remainderParallelism < parallelism) {
                    wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(parallelism - Math.min(Math.min(remainderSimulationLicense, remainderDynamicLicense), remainderParallelism)).projectStartMessageEntity(projectStartMessageEntity).build());
                    run(clusterUserId, Math.min(remainderSimulationLicense, Math.min(remainderDynamicLicense, remainderParallelism)), projectStartMessageEntity, projectRunningKey, isChoiceGpu);
                } else {
                    run(clusterUserId, parallelism, projectStartMessageEntity, projectRunningKey, isChoiceGpu);
                }
            } else {
                throw new RuntimeException("未知模型类型：" + modelType);
            }
        } else if (DictConstants.PROJECT_WAIT_TYPE_EXPAND.equals(waitingType)) {
            if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
                int expandParallelism;
                if (remainderSimulationLicense <= 0 || remainderParallelism <= 0) {
                    log.info("无可用证书或并行度。");
                } else {
                    if (remainderSimulationLicense < waitingParallelism || remainderParallelism <= waitingParallelism) {
                        expandParallelism = Math.min(remainderSimulationLicense, remainderParallelism);
                        wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(waitingParallelism - expandParallelism).projectStartMessageEntity(projectStartMessageEntity).build());
                    } else {
                        expandParallelism = waitingParallelism;
                    }
                    expand(clusterUserId, modelType, projectId, isChoiceGpu, parallelism, expandParallelism, expandParallelism == waitingParallelism);
                }
            } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
                usingDynamicLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_DYNAMIC);
                remainderDynamicLicense = numDynamicLicense - usingDynamicLicenseNumber;
                int expandParallelism;
                if (remainderSimulationLicense <= 0 || remainderDynamicLicense <= 0 || remainderParallelism <= 0) {
                    log.info("无可用证书或并行度。");
                } else {
                    if (remainderSimulationLicense < waitingParallelism || remainderDynamicLicense < waitingParallelism || remainderParallelism < waitingParallelism) {
                        expandParallelism = Math.min(Math.min(remainderSimulationLicense, remainderDynamicLicense), remainderParallelism);
                        wait(ProjectWaitQueueEntity.builder().waitingType(DictConstants.PROJECT_WAIT_TYPE_EXPAND).waitingParallelism(waitingParallelism - expandParallelism).projectStartMessageEntity(projectStartMessageEntity).build());
                    } else {
                        expandParallelism = waitingParallelism;
                    }
                    expand(clusterUserId, modelType, projectId, isChoiceGpu, parallelism, expandParallelism, expandParallelism == waitingParallelism);
                }
            } else {
                throw new RuntimeException("未知模型类型：" + modelType);
            }
        } else {
            throw new RuntimeException("未知运行类型：" + waitingType);
        }

    }

    //* -------------------------------- 等待 --------------------------------

    /**
     * @param projectWaitQueueEntity 等待对象
     */
    public void wait(ProjectWaitQueueEntity projectWaitQueueEntity) {
        try {
            //2 创建等待列表对象
            final String waitingQueueJson = customRedisClient.get(DictConstants.PROJECT_WAIT_QUEUE_KEY);
            List<ProjectWaitQueueEntity> waitingQueue;
            if (StringUtil.isEmpty(waitingQueueJson)) {
                waitingQueue = new ArrayList<>();
            } else {
                waitingQueue = JsonUtil.jsonToList(waitingQueueJson, ProjectWaitQueueEntity.class);
            }
            boolean contains = false;
            for (ProjectWaitQueueEntity waitQueueEntity : waitingQueue) {
                if (waitQueueEntity.getProjectStartMessageEntity().getProjectId().equals(projectWaitQueueEntity.getProjectStartMessageEntity().getProjectId())) {
                    contains = true;
                    waitQueueEntity.setWaitingParallelism(projectWaitQueueEntity.getWaitingParallelism());
                }
            }
            if (!contains) {
                waitingQueue.add(projectWaitQueueEntity);
            }
            String newWaitingQueueJson = JsonUtil.listToJson(waitingQueue);
            customRedisClient.set(DictConstants.PROJECT_WAIT_QUEUE_KEY, newWaitingQueueJson);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
    }

    public void waitMulti(MultiProjectWaitQueueEntity multiProjectWaitQueue) {
        try {
            //2 创建等待列表对象
            final String waitingQueueJson = customRedisClient.get(DictConstants.MULTI_PROJECT_WAIT_QUEUE_KEY);
            List<MultiProjectWaitQueueEntity> waitingQueue;
            if (StringUtil.isEmpty(waitingQueueJson)) {
                waitingQueue = new ArrayList<>();
            } else {
                waitingQueue = JsonUtil.jsonToList(waitingQueueJson, MultiProjectWaitQueueEntity.class);
            }
            boolean contains = false;
            for (MultiProjectWaitQueueEntity waitQueueEntity : waitingQueue) {
                if (waitQueueEntity.getProjectId().equals(multiProjectWaitQueue.getProjectId())) {
                    contains = true;
                    if (multiProjectWaitQueue.getWaitingParallelism() > 0){
                        waitQueueEntity.setWaitingParallelism(multiProjectWaitQueue.getWaitingParallelism());
                        waitQueueEntity.setMultiTaskMessageEntityList(multiProjectWaitQueue.getMultiTaskMessageEntityList());
                        waitQueueEntity.setRunState(multiProjectWaitQueue.getRunState());
                    }else {
                        // 项目等待为0,则删除
                        waitingQueue.remove(waitQueueEntity);
                    }
                }
            }
            if (!contains && multiProjectWaitQueue.getWaitingParallelism() > 0) {
                waitingQueue.add(multiProjectWaitQueue);
            }
            String newWaitingQueueJson = JsonUtil.listToJson(waitingQueue);
            customRedisClient.set(DictConstants.MULTI_PROJECT_WAIT_QUEUE_KEY, newWaitingQueueJson);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
    }

    public void removeMulti(String projectId) {
        try {
            //2 创建等待列表对象
            final String waitingQueueJson = customRedisClient.get(DictConstants.MULTI_PROJECT_WAIT_QUEUE_KEY);
            List<MultiProjectWaitQueueEntity> waitingQueue;
            if (StringUtil.isEmpty(waitingQueueJson)) {
                waitingQueue = new ArrayList<>();
            } else {
                waitingQueue = JsonUtil.jsonToList(waitingQueueJson, MultiProjectWaitQueueEntity.class);
            }
            for (MultiProjectWaitQueueEntity waitQueueEntity : waitingQueue) {
                if (waitQueueEntity.getProjectId().equals(projectId)) {
                    waitingQueue.remove(waitQueueEntity);
                }
            }
            String newWaitingQueueJson = JsonUtil.listToJson(waitingQueue);
            customRedisClient.set(DictConstants.MULTI_PROJECT_WAIT_QUEUE_KEY, newWaitingQueueJson);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
    }

    @SneakyThrows
    public void runMulti(int parallelism, MultiProjectWaitQueueEntity multiProjectWaitQueue, String isChoiceGpu) {
        // 然后再判断是否执行完，未执行完的话则塞入redis，执行完则删除redis
        // 启动成功之后，更新mysql
        List<MultiTaskMessageEntity> multiTaskMessageEntityList = multiProjectWaitQueue.getMultiTaskMessageEntityList();
        String projectId = multiProjectWaitQueue.getProjectId();

        int parallel = parallelism;
        if (multiTaskMessageEntityList.size()- multiProjectWaitQueue.getRunState()-1 < parallelism){
            log.info("出现奇怪情况需要使用的并行度大于剩余任务数量parallelism:{},projectId:{},剩余:{}", parallelism, projectId, JSONObject.toJSONString(multiProjectWaitQueue));
            parallel = multiTaskMessageEntityList.size();
        }
        Integer runState = multiProjectWaitQueue.getRunState();
        // 使用完塞入
        Map<String, Integer> multiNodeMapToUse = projectDomainService.getMultiNodeMapToUse(isChoiceGpu, parallel);
        List<MultiCreateYamlRet> yamlList = new ArrayList<>();
        for (int i = runState + 1; i < parallel + runState + 1; i++) {
            MultiTaskMessageEntity messageEntity = multiTaskMessageEntityList.get(i);
            String taskId = messageEntity.getInfo().getTask_id();
            // 发送kafka消息
//            SendResult<String, String> stringStringSendResult = kafkaTemplate.send(projectId, i % multiTaskMessageEntityList.size(),
//                taskId, JSONObject.toJSONString(messageEntity)).get();
            SendResult<String, String> stringStringSendResult = kafkaTemplate.send(projectId, i % multiTaskMessageEntityList.size(),
                taskId, JsonUtil.beanToJson(messageEntity)).get();

            RecordMetadata recordMetadata = stringStringSendResult.getRecordMetadata();
            String topic = recordMetadata.topic();  // 消息发送到的topic
            int partition = recordMetadata.partition(); // 消息发送到的分区
            long offset = recordMetadata.offset();  // 消息在分区内的offset
            log.info("多模式仿真任务发送消息成功， 主题 topic 为项目ID：" + topic + " 分区 partition 为：" + partition + " 偏移量为：" + offset);
//            String sceneId = messageEntity.getInfo().getScene_id();
            String modelName =null;
            for (String name: multiNodeMapToUse.keySet()) {
                Integer integer = multiNodeMapToUse.get(name);
                if (integer > 0){
                    modelName = name;
                    multiNodeMapToUse.put(name, integer -1);
                }
            }
            if (modelName == null){
                throw new RuntimeException("未选取到可用的节点");
            }
            MultiSimulationSceneKafkaParam multiSimulationSceneKafkaParam = multiProjectWaitQueue.getKafkaParamList().get(i);
            MultiCreateYamlRet multiTempYaml = projectDomainService.createMultiTempYaml(projectId, multiSimulationSceneKafkaParam, messageEntity, modelName, partition, offset, isChoiceGpu);
            multiTempYaml.setTaskId(messageEntity.getInfo().getTask_id());
            multiTempYaml.setNodeName(modelName);
            yamlList.add(multiTempYaml);
        }
        TimeUnit.SECONDS.sleep(10);
        log.info("项目" + projectId + "共发送了" + yamlList.size() + "条消息，准备首先启动" + yamlList);
        for (MultiCreateYamlRet redisKey : yamlList) {
            projectDomainService.createMultiPodBegin(redisKey, redisKey.getYamlRedisKey());
            // 存nodeName
            stringRedisTemplate.opsForValue().set("multi-taskId:" + redisKey.getTaskId(), JSONObject.toJSONString(redisKey));
            // 并行度减1
            projectDomainService.decrementParallelism(isChoiceGpu, redisKey.getNodeName(), 1);
        }
        log.info("项目" + projectId + "已经启动" + yamlList.size());
        for (MultiCreateYamlRet redisKey : yamlList) {
            taskRecordMapper.updateMultiSimulationProjectTaskRecordStatus(MultiSimulationTaskStatusEnum.RUN_STATUS.getProjectStatus(), redisKey.getTaskId(), 0);
        }
//         保存每个机器的并行度
//        projectDomainService.setMultiNodeMapUse(isChoiceGpu, multiNodeMapToUse);
    }

    /**
     * 运行项目
     *
     * @param projectStartMessageEntity 初始接收到的项目启动信息
     * @param projectRunningKey         projectRunningKey
     */
    @SneakyThrows
    public void run(String clusterUserId, int finalParallelism, ProjectStartMessageEntity projectStartMessageEntity, String projectRunningKey, String isChoiceGpu) {
        Map<String, Integer> remainderNodeMap = projectDomainService.getRemainderNodeMap(isChoiceGpu);
        String modelType = projectStartMessageEntity.getModelType();
        if (!DictConstants.SYSTEM_USER_ID.equals(clusterUserId)) {
            projectDomainService.useLicense(clusterUserId, modelType, finalParallelism);
        }
        String projectId = projectStartMessageEntity.getProjectId();    // 项目 id
        final String projectType = projectStartMessageEntity.getType();
        String vehicleConfigId = projectStartMessageEntity.getVehicleConfigId();
        String projectPath = linuxTempPath + "project/" + projectId + "/";
        // -------------------------------- 1 获取任务 json 列表 --------------------------------
        List<String> taskJsonList = FileUtil.listAbsolutePathByTypeAndLength(projectPath, "json", (StringUtil.getRandomUUID() + ".json").length());
        int taskTotal = taskJsonList.size();
        projectStartMessageEntity.setTaskTotal(taskTotal);
        projectStartMessageEntity.setTaskCompleted(0);
        // 设置任务数量之后，获取运行节点，并将项目运行信息放入 redis
        //1 获取剩余并行度和即将使用的各node的并行度
        Map<String, Integer> nodeMap = projectDomainService.getNodeMapToUse(isChoiceGpu, finalParallelism);
        //2 将指定 node 的并行度减少
        nodeMap.keySet().forEach(nodeName -> projectDomainService.decrementParallelism(isChoiceGpu, nodeName, nodeMap.get(nodeName)));
        log.info("项目 " + projectId + " 运行在：" + nodeMap);
        stringRedisTemplate.opsForValue().set(projectRunningKey, JsonUtil.beanToJson(projectStartMessageEntity));
        //* -------------------------------- 3 根据算法id查询算法名称 --------------------------------
        String algorithmDockerImage = projectDomainService.getAlgorithmDockerImageByProjectId(projectId);
        // -------------------------------- 4 发送任务消息 --------------------------------
        List<NodeEntity> nodeListToCount = projectDomainService.getNodeListToCount(nodeMap);
        int messageNumber = 0;
        KafkaUtil.createTopic(kafkaAdminClient, projectId, finalParallelism, (short) 1);   // 创建主题
        TimeUnit.SECONDS.sleep(3);
        // 需要即时启动的任务（并行度的大小）
        ArrayList<String> yamlToRunRedisKeyList = new ArrayList<>();
        for (String taskJsonPath : taskJsonList) {
            String taskId = FileUtil.getFilenameWithoutSuffix(taskJsonPath);
            // 保存运行中的任务信息
            String taskMessageKey = projectRunningKey + ":task:" + taskId + ":message";
            String taskJson = FileUtil.read(taskJsonPath);
            stringRedisTemplate.opsForValue().set(taskMessageKey, taskJson);

            // 将 projectId 作为 topic 名称，根据 parallelism 分散发送 task 信息到 kafka
            SendResult<String, String> stringStringSendResult = kafkaTemplate.send(projectId, messageNumber % finalParallelism, taskId, taskJson).get();
            RecordMetadata recordMetadata = stringStringSendResult.getRecordMetadata();
            String topic = recordMetadata.topic();  // 消息发送到的topic
            int partition = recordMetadata.partition(); // 消息发送到的分区
            long offset = recordMetadata.offset();  // 消息在分区内的offset
            log.info("发送消息成功， 主题 topic 为项目ID：" + topic + " 分区 partition 为：" + partition + " 偏移量为：" + offset);
            // 发送成功过的同时创建 pod.yaml 文件并把文件地址存到 redis
            // 选一个 count 最少的 node
            String currentNodeName = "";
            NodeEntity currentNodeEntity = null;
            int currentCount = Integer.MAX_VALUE;
            log.info("各节点已经预定的任务个数为：" + nodeListToCount);
            for (NodeEntity nodeEntity : nodeListToCount) {
                int tempCount = nodeEntity.getCount();
                String tempNodeName = nodeEntity.getNodeName();
                if (tempCount < currentCount) {
                    currentCount = tempCount;
                    currentNodeName = tempNodeName;
                    currentNodeEntity = nodeEntity;
                }
            }
            if (currentNodeEntity == null) {
                String errorMessage = "挑选节点失败。";
                log.info(errorMessage);
                throw new RuntimeException(errorMessage);
            }
            currentNodeEntity.setCount(currentNodeEntity.getCount() + 1);
            Integer cpuOrder = null;
            if (currentCount == 0) {
                // 根据各节点剩余并行度，倒序获取 cpu 编号
                cpuOrder = remainderNodeMap.get(currentNodeName) - 1;
                remainderNodeMap.put(currentNodeName, cpuOrder);
            }
            // 只有准备启动（即 currentCount == 0）的时候才指定 cpu 编号
            log.info("创建任务 " + taskId + " 的 yaml：是否使用 gpu （0是1否）" + isChoiceGpu + "，当前节点已创建 yaml 个数为：" + currentCount + "，当前节点名称为：" + currentNodeName + "，当前 cpu 编号为：" + cpuOrder + "，镜像名：" + algorithmDockerImage);
            String yamlRedisKey = projectDomainService.createTempYaml(projectId, vehicleConfigId, modelType, algorithmDockerImage, currentNodeName, partition, offset, isChoiceGpu, cpuOrder);
            if (currentCount == 0) {
                yamlToRunRedisKeyList.add(yamlRedisKey);
            }
            messageNumber++;
        }
        TimeUnit.SECONDS.sleep(3);
        log.info("项目 " + projectId + " 共发送了 " + messageNumber + " 条消息，准备首先启动 " + yamlToRunRedisKeyList);
        for (String redisKey : yamlToRunRedisKeyList) {
            projectDomainService.createPodBegin(projectId, redisKey);
        }
        log.info("项目 " + projectId + " 已经启动 " + yamlToRunRedisKeyList);
        // 项目启动之后删除等待队列中的该项目
        projectDomainService.removeWaitQueue(DictConstants.PROJECT_WAIT_TYPE_EXECUTE, projectId);

    }


    @Synchronized
    public void expand(String clusterUserId, String modelType, String projectId, String isChoiceGpu, int totalParallelism, int expandParallelism, boolean isDone) {
        log.info("扩充项目 {} {} 个并行度", projectId, expandParallelism);
        if (!DictConstants.SYSTEM_USER_ID.equals(clusterUserId)) {
            projectDomainService.useLicense(clusterUserId, modelType, expandParallelism);
        }
        //1 获取剩余并行度和即将使用的各node的并行度
        Map<String, Integer> remainderNodeMap = projectDomainService.getRemainderNodeMap(isChoiceGpu);
        log.info("剩余并行度为：" + remainderNodeMap);
        Map<String, Integer> nodeMapToUse = projectDomainService.getNodeMapToUse(isChoiceGpu, expandParallelism);
        log.info("即将使用的并行度为：" + nodeMapToUse);
        //2 将指定 node 的并行度减少
        nodeMapToUse.keySet().forEach(nodeName -> projectDomainService.decrementParallelism(isChoiceGpu, nodeName, nodeMapToUse.get(nodeName)));
        //3 获取还未运行的任务 ("project:" + projectId + ":node:" + nodeName + ":yaml")
        final Set<String> yamlPathCacheKeySet = customRedisClient.getKeySetByPrefixAndContent(stringRedisTemplate, "project:" + projectId + ":node", "yaml");
        log.info("项目 {} 还未运行的 yaml 在缓存中的 key 有 {}", projectId, yamlPathCacheKeySet);
        if (CollectionUtil.isNotEmpty(yamlPathCacheKeySet)) {
            // 根据节点名分组
            final Map<String, List<String>> yamlPathCacheKeyMapGroupByNodeName = yamlPathCacheKeySet.stream().collect(Collectors.groupingBy(key -> {
                final String[] split = key.split(":");
                return split[3];
            }));
            log.info("yaml缓存key根据节点分组之后为：" + yamlPathCacheKeyMapGroupByNodeName);
            // 每个节点分出一部分给两个节点
            yamlPathCacheKeyMapGroupByNodeName.forEach((nodeNameBefore, yamlPathCacheKeySetGroupByNodeName) -> {
                final int yamlCount = yamlPathCacheKeySetGroupByNodeName.size();
                // 修改全部yaml
                nodeMapToUse.forEach((nodeNameAfter, parallelismToUse) -> {
                    log.info("修改yaml执行节点 {} -> {}：{}", nodeNameBefore, nodeNameAfter, yamlPathCacheKeySetGroupByNodeName);
                    int shareNum = (int) Math.ceil(yamlCount * 1.0 / totalParallelism * parallelismToUse);
                    log.info("需要分享的yaml个数为：{}", shareNum);
                    for (int i = 0; i < shareNum; i++) {
                        log.info("将yaml按比例均分给节点 {}", nodeNameAfter);
                        final String yamlPathCacheKeyBefore = yamlPathCacheKeySetGroupByNodeName.get(i);
                        final String yamlPathCacheKeyAfter = yamlPathCacheKeyBefore.replace(nodeNameBefore, nodeNameAfter);
                        final String yamlPath = customRedisClient.get(yamlPathCacheKeyBefore);
                        final String yamlContentBefore = FileUtil.read(new File(yamlPath));
                        final String yamlContentAfter = yamlContentBefore.replace(nodeNameBefore, nodeNameAfter);
                        FileUtil.rm(yamlPath);
                        FileUtil.writeStringToLocalFile(yamlContentAfter, yamlPath);
                        customRedisClient.delete(yamlPathCacheKeyBefore);
                        customRedisClient.set(yamlPathCacheKeyAfter, yamlPath);
                        if (i < parallelismToUse) {
                            log.info("均分之后可直接执行部分yaml：{}", yamlPath);
                            // 修改 cpu 编号
                            // 根据各节点剩余并行度，倒序获取 cpu 编号
                            int cpuOrderAfter = remainderNodeMap.get(nodeNameAfter) - 1 - i;
                            log.info("CPU 编号 {}", cpuOrderAfter);
                            String cpuOrderString = String.valueOf(cpuOrderAfter);
                            final String read = FileUtil.read(yamlPath);
                            final String replace = read.replace("cpu-order", "\"" + cpuOrderString + "\"");
                            FileUtil.writeStringToLocalFile(replace, yamlPath);
                            // 创建 pod
                            log.info("扩充项目{}的一个并行度成功。", projectId);
                            projectDomainService.createPod(projectId, yamlPathCacheKeyAfter, cpuOrderString);
                        }
                    }
                });
            });
        }
        if (isDone) {
            projectDomainService.removeWaitQueue(DictConstants.PROJECT_WAIT_TYPE_EXPAND, projectId);
        }
    }


    //* -------------------------------- 逻辑 --------------------------------


    @SneakyThrows
    public List<SceneEntity> getSceneList(String projectId, String packageId) {

        String allIndexPrefix = "project:" + projectId + ":package:" + packageId + ":all";
        String leafIndexPrefix = "project:" + projectId + ":package:" + packageId + ":leaf";
        //1 查询该场景包的所有指标列表，包删了无所谓，但要过滤删掉的指标。并保存成 json 文件。
        List<IndexTemplateEntity> allIndexList = indexTemplateMapper.selectByPackageIdIncludeDeleted(packageId);
        stringRedisTemplate.opsForValue().set(allIndexPrefix, JsonUtil.listToJson(allIndexList));
        //2 查询场景包叶子指标
        List<IndexTemplateEntity> leafIndexList = allIndexList.stream().filter(index -> StringUtil.isNotEmpty(index.getRuleId())).collect(Collectors.toList());
        log.info("项目 " + projectId + " 的叶子指标为：" + leafIndexList);
        stringRedisTemplate.opsForValue().set(leafIndexPrefix, JsonUtil.listToJson(leafIndexList));
        List<SceneEntity> sceneList = new ArrayList<>();
        leafIndexList.forEach(leafIndex -> {
            String naturalIds = leafIndex.getSceneNaturalIds();
            String standardIds = leafIndex.getSceneStatueIds();
            String accidentIds = leafIndex.getSceneTrafficIds();
            String generalizationIds = leafIndex.getSceneGeneralizationIds();
            String sceneReferenceIds = leafIndex.getSceneReferenceLibIds();
            if (StringUtil.isNotEmpty(naturalIds)) {
                List<String> naturalIdList = new ArrayList<>(Arrays.asList(naturalIds.split(",")));
                sceneList.addAll(sceneMapper.selectNaturalByIdList(naturalIdList));
            }
            if (StringUtil.isNotEmpty(standardIds)) {
                List<String> standardIdList = new ArrayList<>(Arrays.asList(standardIds.split(",")));
                sceneList.addAll(sceneMapper.selectStandardByIdList(standardIdList));
            }
            if (StringUtil.isNotEmpty(accidentIds)) {
                List<String> accidentIdList = new ArrayList<>(Arrays.asList(accidentIds.split(",")));
                sceneList.addAll(sceneMapper.selectAccidentByIdList(accidentIdList));
            }
            if (StringUtil.isNotEmpty(generalizationIds)) {
                List<String> generalizationIdList = new ArrayList<>(Arrays.asList(generalizationIds.split(",")));
                sceneList.addAll(sceneMapper.selectGeneralizationByIdList(generalizationIdList));
            }
            if (StringUtil.isNotEmpty(sceneReferenceIds)) {
                List<String> sceneReferenceIdList = new ArrayList<>(Arrays.asList(sceneReferenceIds.split(",")));
                sceneList.addAll(sceneMapper.selectReferenceLibByIdList(sceneReferenceIdList));
            }
        });
        log.info("项目" + projectId + " 共有 " + sceneList.size() + " 个任务：" + sceneList);
        return sceneList;
    }


    /**
     * 将 master 节点设置成镜像仓库，导入镜像的同时 commit 到仓库当中，供其他节点 pull
     *
     * @param projectId   项目 id
     * @param algorithmId 算法 id
     * @return 镜像名称
     */
    @SneakyThrows
    public String handleAlgorithm(String projectId, String algorithmId) {
        //4-1 根据算法 id 获取算法文件地址、是否已导入成镜像。
        AlgorithmEntity algorithmEntity = algorithmMapper.selectById(algorithmId);
        String dockerImage;
        String algorithmDirectoryLinuxTempPath;
        String algorithmTarLinuxTempPath = null;
        if (algorithmEntity != null) {
            log.info("项目" + projectId + "使用仿真平台自己的算法 " + algorithmEntity);
            String algorithmCode = algorithmEntity.getAlgorithmCode();
            String dockerImport = algorithmEntity.getDockerImport();
            dockerImage = dockerConfiguration.getRegistry() + "/algorithm_" + algorithmCode + ":latest";
            if ("1".equals(dockerImport) && StringUtil.isNotEmpty(dockerImport)) { // 已经导入同时存在镜像名称
                return algorithmEntity.getDockerImage();
            } else if (dockerImport == null || "0".equals(dockerImport)) {  // 未导入
                String uploadMode = algorithmEntity.getUploadMode();
                if (DictConstants.ALGORITHM_UPLOAD_MODE_FILE.equals(uploadMode)) {
                    algorithmTarLinuxTempPath = linuxTempPath + "algorithm-file/" + algorithmCode + "/" + algorithmCode + ".tar";
                    String minioPath = algorithmEntity.getMinioPath();
                    log.info("下载 minio 算法文件 " + minioPath + " 到本地 " + algorithmTarLinuxTempPath);
                    MinioUtil.downloadToFile(minioClient, bucketName, minioPath, algorithmTarLinuxTempPath);
                } else if (DictConstants.ALGORITHM_UPLOAD_MODE_GIT.equals(uploadMode)) {
                    algorithmDirectoryLinuxTempPath = linuxTempPath + "algorithm-git/" + algorithmCode + "/";
                    String gitUrl = algorithmEntity.getGitUrl().replace(gitConfiguration.getName(), gitConfiguration.getUrl());
                    String gitUserName = algorithmEntity.getGitUserName();
                    String gitPassword = algorithmEntity.getGitPassword();
                    log.info("下载 git 算法文件 " + gitUrl + " 到本地 " + algorithmDirectoryLinuxTempPath);
                    GitUtil.clone(gitUrl, gitUserName, gitPassword, algorithmDirectoryLinuxTempPath, true);
                    for (String filename : Objects.requireNonNull(new File(algorithmDirectoryLinuxTempPath).list())) {
                        if (filename.endsWith(".tar")) {
                            algorithmTarLinuxTempPath = algorithmDirectoryLinuxTempPath + filename;
                        }
                    }
                } else {
                    throw new RuntimeException("算法 " + algorithmId + " 的 upload_mode 字段数据有误！");
                }
                algorithmMapper.updateDockerImportAndDockerImageById("1", dockerImage, algorithmId);
                if (algorithmTarLinuxTempPath == null) {
                    throw new RuntimeException("算法" + algorithmId + "下载失败。");
                }
                OsUtil.exec("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
                OsUtil.exec("docker push " + dockerImage);
                FileUtil.rm(algorithmTarLinuxTempPath);
                log.info("已删除算法临时文件：" + algorithmTarLinuxTempPath);
            } else {
                throw new RuntimeException("算法 " + algorithmId + " 的 mysql 数据有误！");
            }
        } else {
            log.info("项目" + projectId + "使用索为平台算法 " + algorithmId);
            algorithmTarLinuxTempPath = linuxTempPath + "algorithm/" + algorithmId + ".tar";
            String dockerImageWithoutVersion = dockerConfiguration.getRegistry() + "/algorithm_" + algorithmId;
            dockerImage = dockerImageWithoutVersion + ":latest";
            //0 查看算法是否已经导入
            if (!projectDomainService.isImported(dockerImageWithoutVersion)) {
                //1 获取 token
                String tokenUrl = customConfiguration.getAlgorithmPlatformTokenUri() + "?grant_type=client_credential&appid=" + customConfiguration.getAlgorithmPlatformAppid() + "&secret=" + customConfiguration.getAlgorithmPlatformSecret();
                String tokenJson = HttpUtil.get(tokenUrl);
                String token = new ObjectMapper().readTree(tokenJson).path("data").path("access_token").asText();
                //2 获取 下载地址
                String idOfAlgorithm = algorithmExpandMapper.selectIdByAlgorithmId(AlgorithmExpandEntity.builder().algorithmId(algorithmId).build());
                String downloadUrl = customConfiguration.getAlgorithmPlatformAlgorithmAddrUri() + "?access_token=" + token + "&id=" + idOfAlgorithm;
                //3 下载算法包
                String downloadUrlJson = HttpUtil.get(downloadUrl);
                String tempDownloadUrl = new ObjectMapper().readTree(downloadUrlJson).path("data").asText();
                InputStream inputStream = HttpUtil.getInputStream(tempDownloadUrl);
                FileUtil.writeInputStreamToLocalFile(inputStream, algorithmTarLinuxTempPath);
                // 本地执行 docker import 算法文件成镜像（可改成用 docker-java 操作仓库）
                OsUtil.exec("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
                OsUtil.exec("docker push " + dockerImage);
                FileUtil.rm(algorithmTarLinuxTempPath); // 删除算法临时文件
                algorithmExpandMapper.updateDockerImportAndDockerImageById("1", dockerImage, algorithmId);
                algorithmExpandMapper.updateStatusByAlgorithmId(AlgorithmExpandEntity.builder().algorithmId(algorithmId).status(DictConstants.ALGORITHM_EXPAND_STATUS_TESTING).build());
            } else {
                log.info("算法镜像" + dockerImageWithoutVersion + "已导入。");
            }
        }
        log.info("项目 " + projectId + " 使用的算法镜像为：" + dockerImage);

        return dockerImage;
    }

    /**
     * @param projectStopMessageEntity 项目终止消息实体
     */
    @SneakyThrows
    public void stopProject(ProjectStopMessageEntity projectStopMessageEntity) {
        final String projectId = projectStopMessageEntity.getProjectId();
        final String projectType = projectStopMessageEntity.getType();

        // 删除等待队列中的项目
        projectDomainService.removeWaitQueue(DictConstants.PROJECT_WAIT_TYPE_ALL, projectId);
        String isChoiceGpu = projectDomainService.getIsChoiceGpuByProjectId(projectId);
        //* -------------------------------- Comment --------------------------------
        ProjectEntity projectEntity = projectDomainService.getProjectByProjectId(projectId);
        if (projectEntity == null) {
            log.info("项目未运行：" + projectId);
            return;
        }
        String projectUserId, clusterUserId, modelType, projectRunningKeyPrefix;
        int parallelism;
        PrefixEntity redisPrefix;
        Set<String> projectRunningKeySet;
        boolean isRunning;
        //* -------------------------------- Comment --------------------------------
        parallelism = Integer.parseInt(projectEntity.getParallelism());
        redisPrefix = projectDomainService.getRedisPrefixByProjectIdAndProjectType(projectId, projectType);
        projectRunningKeyPrefix = redisPrefix.getProjectRunningKey();
        //* -------------------------------- Comment --------------------------------
        //1 判断项目是否已经运行
        projectRunningKeySet = RedisUtil.getKeySetByPrefix(stringRedisTemplate, projectRunningKeyPrefix);
        isRunning = !CollectionUtil.isEmpty(projectRunningKeySet);
        //2 将项目状态修改为终止中
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            projectEntity = simulationManualProjectMapper.selectById(projectId);
            simulationManualProjectMapper.updateProjectState(projectId, DictConstants.PROJECT_TERMINATING, TimeUtil.getNowForMysql());
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            projectEntity = simulationAutomaticSubprojectMapper.selectById(projectId);
            simulationAutomaticSubprojectMapper.updateProjectState(projectId, DictConstants.PROJECT_TERMINATING, TimeUtil.getNowForMysql());
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }

        projectUserId = projectEntity.getCreateUserId();
        clusterUserId = projectDomainService.getClusterUserIdByProjectUserId(projectUserId);

        //3 删除 kafka 消息
        KafkaUtil.deleteTopic(kafkaAdminClient, projectId);
        //4 删除项目所有任务
        simulationManualProjectTaskMapper.deleteByProject(projectId);

        //5 根据 pod 前缀删除所有 pod
        modelType = projectDomainService.getModelTypeByProjectIdAndProjectType(projectId, projectType);
        if (isRunning) {
            Set<String> nodeOfPodKeySet = RedisUtil.getKeySetByPrefix(stringRedisTemplate, "project:" + projectId + ":pod:" + "project-" + projectId);
            Set<String> podNameSet = nodeOfPodKeySet.stream().map(key -> key.split(":")[3]).collect(Collectors.toSet());
            for (String podName : podNameSet) {
                String nodeName;
                try {
                    nodeName = projectDomainService.getNodeNameOfPod(projectId, podName);
                } catch (Exception e) {
                    break;
                }
                // 删除 pod
                projectDomainService.deletePod(podName);
                // 节点并行度加一
                projectDomainService.incrementOneParallelism(isChoiceGpu, nodeName);
                // 释放证书
                projectDomainService.releaseLicense(clusterUserId, modelType, parallelism);
            }
        }

        //5 其他 redis key
        RedisUtil.deleteByPrefix(stringRedisTemplate, "project:" + projectId);
        RedisUtil.deleteByPrefix(stringRedisTemplate, redisPrefix.getProjectRunningKey());
        RedisUtil.deleteByPrefix(stringRedisTemplate, redisPrefix.getProjectWaitingKey());

        //6 删除项目 pod 启动文件
        FileUtil.deleteFileBySubstring(podYamlDirectory, projectId);

        //7 删除项目临时文件
        FileUtil.rm(linuxTempPath + "project/" + projectId + "/");

        //8 删除 minio 残留文件
        MinioUtil.rmR(minioClient, bucketName, projectResultPathOfMinio + projectId + "/");

        log.info("项目 " + projectId + " 终止成功！");
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            simulationManualProjectMapper.updateProjectState(projectId, DictConstants.PROJECT_TERMINATED, TimeUtil.getNowForMysql());
        } else {
            simulationAutomaticSubprojectMapper.updateProjectState(projectId, DictConstants.PROJECT_TERMINATED, TimeUtil.getNowForMysql());
        }

        //9 如果在打分过程中被终止需要删除锁
        customRedisClient.delete("project:" + projectId + ":completed-lock");

        //10 如果使用算法平台算法则修改算法状态为未测试
        projectDomainService.checkAlgorithmIsExpand(projectType, projectId, DictConstants.ALGORITHM_EXPAND_STATUS_NOT_TESTED);
    }

    @Async("pool1")
    public void runMultiProject(MultiSimulationProjectKafkaParam projectStartMessageEntity) {
        MultiProjectWaitQueueEntity multiTaskAndFixData = createMultiTaskAndFixData(projectStartMessageEntity);
        checkIfCanRunMulti(multiTaskAndFixData);
    }
    @SneakyThrows
//    @Transactional
    public void stopMultiProject(MultiSimulationProjectKafkaParam projectKafkaParam) {
        String projectId = projectKafkaParam.getProjectId();
        // 删除等待队列中的项目
        removeMulti(projectId);
        String isChoiceGpu = DictConstants.USE_GPU;
        MultiSimulationProjectVO projectVO = multiSimulationProjectMapper.selectMultiSimulationProjectById(projectId);
        if (projectVO.getProjectStatus() == MultiSimulationStatusEnum.TERMINATED_STATUS.getProjectStatus() ||
            projectVO.getProjectStatus() == MultiSimulationStatusEnum.COMPLETED_STATUS.getProjectStatus()){
            // 已完成或者已经是终止状态,不再处理
            log.info("目前该多模式仿真任务已经是结束状态,不再进行停止操作,projectId:{},目前状态:{}", projectId,
                MultiSimulationStatusEnum.matchStatusEnumByStatus(projectVO.getProjectStatus()).getProjectStatusValue());
            return;
        }
        Integer status = projectVO.getProjectStatus();
        MultiSimulationProjectParam multiSimulationProjectParam = new MultiSimulationProjectParam();
        multiSimulationProjectParam.setProjectId(projectId);
        String projectUserId = projectVO.getProjectUserId();
        String clusterUserId = projectDomainService.getClusterUserIdByProjectUserId(projectUserId);

        // 查看mysql项目是否已经运行
        // 如果不为空,则代表已经执行,为空则代表kafka还未被消费
        List<MultiSimulationProjectTaskRecordPO> recordPOList = taskRecordMapper.selectMultiSimulationProjectTaskRecordList(projectId);
        if (!CollectionUtils.isEmpty(recordPOList)){
            for (MultiSimulationProjectTaskRecordPO po: recordPOList) {
                Integer recordStatus = po.getStatus();
                if (recordStatus == MultiSimulationTaskStatusEnum.RUN_STATUS.getProjectStatus()){
                    taskRecordMapper.updateMultiSimulationProjectTaskRecordStatus(MultiSimulationTaskStatusEnum.PERSON_CANCEL_STATUS.getProjectStatus(), po.getId(), 0);
                    // 删除pod
                    String nodeNameKey = "multi-taskId:" + po.getId();
                    String value = stringRedisTemplate.opsForValue().get(nodeNameKey);
                    MultiCreateYamlRet multiCreateYamlRet = JSONObject.parseObject(value, MultiCreateYamlRet.class);

                    String nodeName = multiCreateYamlRet.getNodeName();
                    String podName = multiCreateYamlRet.getPodName();
                    // 删除 pod
                    projectDomainService.deleteMultiProjectPod(podName);
                    // 节点并行度加一
                    projectDomainService.incrementOneParallelism(isChoiceGpu, nodeName);
                    // 释放证书
                    projectDomainService.releaseLicense(clusterUserId, DictConstants.MODEL_TYPE_VTD, 1);

                } else if (recordStatus == MultiSimulationTaskStatusEnum.INIT_STATUS.getProjectStatus()) {
                    // 初始状态时
                    taskRecordMapper.updateMultiSimulationProjectTaskRecordStatus(MultiSimulationTaskStatusEnum.PERSON_CANCEL_STATUS.getProjectStatus(), po.getId(), 0);
                }else if (recordStatus == MultiSimulationTaskStatusEnum.NEED_ANALYSIS_STATUS.getProjectStatus() || recordStatus == MultiSimulationTaskStatusEnum.AUTO_TERMINATED_STATUS.getProjectStatus()) {
                    // 不作处理了
                }
            }
            // 设置整个仿真项目
            if (status == MultiSimulationTaskStatusEnum.RUN_STATUS.getProjectStatus()){
                multiSimulationProjectParam.setProjectStatus(MultiSimulationStatusEnum.TERMINATED_STATUS.getProjectStatus());
                multiSimulationProjectMapper.updateMultiSimulationProjectStatus(multiSimulationProjectParam);
            }
            // 删除kafka topic
            KafkaUtil.deleteTopic(kafkaAdminClient, projectId);
            //6 删除项目 pod 启动文件
            FileUtil.deleteFileBySubstring(multiPodYamlDirectory, projectId);
            //7 删除项目临时文件
            FileUtil.rm(linuxTempPath + "multiProject/" + projectId + "/");
            // 删除minio临时文件
            MinioUtil.rmR(minioClient, bucketName, projectResultPathOfMinio + projectId + "/");
            // 删除算法key
            // 删除yaml路径redis
            // 删除记录podNamekey
            RedisUtil.deleteByPrefix(stringRedisTemplate, "multi_project:" + projectId);
        }

    }

    public MultiProjectWaitQueueEntity createMultiTaskAndFixData(MultiSimulationProjectKafkaParam projectStartMessageEntity) {
        //* -------------------------------- 0 读取消息，创建临时目录 --------------------------------
        String projectId = projectStartMessageEntity.getProjectId();                // 手动执行项目 id 或 自动执行子项目 id
        try {
            // 首先删掉历史的projectId记录
            taskRecordMapper.deleteProjectByProjectId(projectId);
            resultMapper.deleteProjectResultByProjectId(projectId);
            KafkaUtil.deleteTopic(kafkaAdminClient, projectId);

            List<MultiSimulationSceneKafkaParam> kafkaParamList = projectStartMessageEntity.getKafkaParamList();
            List<MultiTaskMessageEntity> entityList = new ArrayList<>();
            for (MultiSimulationSceneKafkaParam kafkaParam: kafkaParamList) {
                String taskId = StringUtil.getRandomUUID();
                String mapId = kafkaParam.getMapId();
                String sceneId = kafkaParam.getId();
                String minioUploadPath = "multiResult/" + projectId + "/" + taskId + "/";
                SimulationMapVO simulationMapVO = mapMapper.selectMapByMapId(mapId);
                if (Objects.isNull(simulationMapVO)){
                    throw new RuntimeException("地图" + mapId + "不存在");
                }
                String mapPath = simulationMapVO.getMapPath();
                String mapOsgbPath = simulationMapVO.getMapOsgbPath();

//                String mapMinioPath = mapPath.substring(mapPath.indexOf("/mapFile"), mapPath.indexOf("?"));
                String mapMinioPath = mapPath;
                String[] mapDriverSp = mapMinioPath.split("/");
                String mapDriverSpName = mapDriverSp[mapDriverSp.length - 1];
                String[] mapDriverNameSp = mapDriverSpName.split("\\.");
                String mapDriverLast = mapDriverNameSp[mapDriverNameSp.length - 1];
                String mapDriverLinuxPath = linuxTempPath + "multiProject/" + projectId + "/" + taskId + "/" + mapDriverSpName;
                String mapDriverPathOfMinio = projectResultPathOfMinio + minioUploadPath + taskId + "." + mapDriverLast;
                MinioUtil.downloadToFile(minioClient, bucketName, mapMinioPath, mapDriverLinuxPath);
                MinioUtil.uploadFromFile(minioClient, mapDriverLinuxPath, bucketName, mapDriverPathOfMinio);
                FileUtil.rm(mapDriverLinuxPath);   // 删除临时文件

//                String mapOsgMinioPath = mapOsgbPath.substring(mapOsgbPath.indexOf("/mapFile"), mapOsgbPath.indexOf("?"));
                String mapOsgMinioPath = mapOsgbPath;
                String[] mapOsgSp = mapOsgMinioPath.split("/");
                String mapOsgSpName = mapOsgSp[mapOsgSp.length - 1];
                String[] mapOsgSpNameSp = mapOsgSpName.split("\\.");
                String mapOsgSpNameSpLast = mapOsgSpNameSp[mapOsgSpNameSp.length - 1];
                String mapOsgLinuxPath = linuxTempPath + "multiProject/" + projectId + "/" + taskId + "/" + mapOsgSpName;
                String mapOsgPathOfMinio = projectResultPathOfMinio + minioUploadPath + taskId + "." + mapOsgSpNameSpLast;
                MinioUtil.downloadToFile(minioClient, bucketName, mapOsgMinioPath, mapOsgLinuxPath);
                MinioUtil.uploadFromFile(minioClient, mapOsgLinuxPath, bucketName, mapOsgPathOfMinio);
                FileUtil.rm(mapOsgLinuxPath);   // 删除临时文件

                // 生成并上传xml
                JSONObject carJson = new JSONObject();
                carJson.put("numOfVehicle", kafkaParam.getSimulationSceneCarVOList().size());
                JSONArray carArray = new JSONArray();

                List<MultiSimulationSceneCarVO> simulationSceneCarVOList = kafkaParam.getSimulationSceneCarVOList();
                if (CollectionUtils.isEmpty(simulationSceneCarVOList)){
                    throw new RuntimeException("未配置车辆");
                }
                VehicleEntity vehicle = null;
                for (MultiSimulationSceneCarVO sceneCar: simulationSceneCarVOList) {
                    String sceneCarId = sceneCar.getId();
                    // 处理算法id
                    String algorithmId = sceneCar.getAlgorithmId();
                    log.info("项目:" + projectId + ",场景:" + sceneId + ",车辆id:" + sceneCarId + ",开始算法导入。");
                    String algorithmDockerImage = handleAlgorithm(sceneId, algorithmId);
                    log.info("项目:" + projectId + ",场景:" + sceneId + ",车辆id:" + sceneCarId + ",算法已导入" + algorithmDockerImage);
//                    result.put("sceneCarId-" + sceneCarId + "-docker-image", algorithmDockerImage);
                    customRedisClient.set(projectDomainService.getMultiAlgorithmIdRedisKey(algorithmId, projectId), algorithmDockerImage);
                    // 处理车辆,一个场景的车辆是一样的，只需取第一个车辆即可
                    if (vehicle == null){
                        String carId = sceneCar.getCarId();
                        //1 根据车辆配置id vehicleConfigId, 获取 模型信息和传感器信息
                        com.css.simulation.resource.scheduler.infra.entity.VehicleEntity vehicleEntity = vehicleMapper.selectByVehicleConfigId(carId);   // 车辆
                        List<CameraEntity> cameraEntityList = sensorCameraMapper.selectCameraByVehicleConfigId(carId);    // 摄像头
                        List<OgtEntity> ogtEntityList = sensorOgtMapper.selectOgtByVehicleId(carId); // 完美传感器
                        vehicle = VehicleEntity.builder().model(ModelEntity.builder().model_label(vehicleEntity.getModelLabel()).build())
                            .dynamics(DynamicsEntity.builder().dynamics_maxspeed(vehicleEntity.getMaxSpeed()).dynamics_enginepower(vehicleEntity.getEnginePower()).dynamics_maxdecel(vehicleEntity.getMaxDeceleration()).dynamics_maxsteering(vehicleEntity.getMaxSteeringAngle()).dynamics_mass(vehicleEntity.getMass()).dynamics_frontsurfaceeffective(vehicleEntity.getFrontSurfaceEffective()).dynamics_airdragcoefficient(vehicleEntity.getAirDragCoefficient()).dynamics_rollingresistance(vehicleEntity.getRollingResistanceCoefficient()).dynamics_wheeldiameter(vehicleEntity.getWheelDiameter()).dynamics_wheeldrive(vehicleEntity.getWheelDrive()).dynamics_overallefficiency(vehicleEntity.getOverallEfficiency()).dynamics_distfront(vehicleEntity.getFrontDistance()).dynamics_distrear(vehicleEntity.getRearDistance()).dynamics_distleft(vehicleEntity.getLeftDistance()).dynamics_distright(vehicleEntity.getRightDistance()).dynamics_distheight(vehicleEntity.getHeightDistance()).dynamics_wheelbase(vehicleEntity.getWheelbase()).build())
                            .sensors(SensorsEntity.builder()   // 根据 vehicleId 查询绑定的传感器列表
                                .camera(cameraEntityList).OGT(ogtEntityList).build()).build();
                    }
                    String pathStart = sceneCar.getPathStart();
                    JSONObject jsonObject = JSONObject.parseObject(pathStart);
                    jsonObject.remove("name");
                    carArray.add(jsonObject);
                }
                carJson.put("position", carArray);
                // 生成xml文件
                String mapXmlLinuxPath = linuxTempPath + "multiProject/" + projectId + "/" + taskId + "/" + taskId + ".xml";
                String mapJsonLinuxPath = linuxTempPath + "multiProject/" + projectId + "/" + taskId + "/" + taskId + ".json";
                FileUtil.writeStringToLocalFile(carJson.toJSONString(), mapJsonLinuxPath);
                String linuxComm = multiVtdXmlGenerator + " " + mapJsonLinuxPath + " " + mapXmlLinuxPath;
                LinuxUtil.execute(linuxComm);
                String mapXmlPathOfMinio = projectResultPathOfMinio + minioUploadPath + taskId + ".xml";
                MinioUtil.uploadFromFile(minioClient, mapXmlLinuxPath, bucketName, mapXmlPathOfMinio);
                FileUtil.rm(mapXmlLinuxPath);
                FileUtil.rm(mapJsonLinuxPath);

                MultiTaskMessageEntity build = MultiTaskMessageEntity.builder().info(MultiInfoEntity.builder().project_id(projectId).task_id(taskId).scene_id(sceneId)
                        .default_time(Long.valueOf(projectStartMessageEntity.getDefaultTime()))
                        .task_path(projectResultPathOfMinio + minioUploadPath).build())
                    .scenario(ScenarioEntity.builder().scenario_osc(mapXmlPathOfMinio).scenario_odr(mapDriverPathOfMinio).scenario_osgb(mapOsgPathOfMinio).build())
                    .vehicle(vehicle)
                    .build();
                entityList.add(build);
            }
            int sort = 0;
            for (MultiTaskMessageEntity entity: entityList) {
                int i = taskRecordMapper.addMultiSimulationProjectTaskRecord(entity.getInfo().getTask_id(), entity.getInfo().getScene_id(), entity.getInfo().getProject_id(),
                    JSONObject.toJSONString(entity), sort);
                sort ++;
            }
            log.info("project:{},共插入{}条数据", projectId, entityList.size());
            MultiProjectWaitQueueEntity build = MultiProjectWaitQueueEntity.builder()
                .multiTaskMessageEntityList(entityList)
                .projectId(projectId)
                .waitingParallelism(entityList.size())
                .kafkaParamList(kafkaParamList)
                .runState(-1)
                .build();
            log.info("MultiProjectWaitQueueEntity返回结果:{}", JSONObject.toJSONString(build));
            return build;
        } catch (Exception e) {
            log.error("项目报错。", e);
            // TODO 项目报错之后状态设置一下
            throw new RuntimeException(e);
        }
    }



    @SneakyThrows
    // TODO 此处加锁
    public void checkIfCanRunMulti(MultiProjectWaitQueueEntity projectWaitQueueEntity) {
//        List<MultiTaskMessageEntity> multiTaskMessageEntityList = projectWaitQueueEntity.getMultiTaskMessageEntityList();
        //1 项目信息
        int parallelism = projectWaitQueueEntity.getWaitingParallelism();
        if (parallelism <=0){
            log.info("需要只需的项目并行度为0");
        }
        String isChoiceGpu = DictConstants.USE_GPU;
        final UserEntity userEntity = projectDomainService.getUserEntityByMultiProjectId(projectWaitQueueEntity.getProjectId());
        String projectUserId = userEntity.getId();
        String roleCode = userEntity.getRoleCode();
        String useType = userEntity.getUseType();

        ClusterEntity clusterEntity = null;
        String clusterUserId;  // 项目实际运行使用的用户集群

        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
            clusterUserId = DictConstants.SYSTEM_USER_ID;
        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
            clusterUserId = projectUserId;
            clusterEntity = clusterMapper.selectByUserId(clusterUserId);

        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
                clusterUserId = projectUserId;
                clusterEntity = clusterMapper.selectByUserId(clusterUserId);
            } else if (DictConstants.USER_TYPE_PUBLIC.equals(useType)) {    //3-4 共享子账户，根据父账户的共享节点排队
                clusterUserId = userEntity.getCreateUserId();
                clusterEntity = clusterMapper.selectByUserId(clusterUserId);
            } else {
                throw new RuntimeException("用户" + projectUserId + "未知占用类型：" + useType);
            }
        } else {
            throw new RuntimeException("未知角色类型：" + roleCode);
        }
        int remainderSimulationLicense = Integer.MAX_VALUE;
        if (!clusterUserId.equals(DictConstants.SYSTEM_USER_ID)){
            // 获取仿真软件证书数量和动力学软件证书数量（vtd占一个仿真证书，carsim各占一个）
            Integer usingSimulationLicenseNumber = projectDomainService.getUsingLicenseNumber(clusterUserId, DictConstants.LICENSE_TYPE_SIMULATION);
            Integer numSimulationLicense = clusterEntity.getNumSimulationLicense();
            // 判断仿真证书是否够用，如果证书为0则将项目加入等待队列；如果证书小于并行度则加入扩充队列，并用现有证书执行；如果证书够用，直接执行。
            remainderSimulationLicense = numSimulationLicense - usingSimulationLicenseNumber;
        }

        //2 剩余并行度
        // 考虑仿真证书数量
        // 剩余并行度
        int remainderParallelism = projectDomainService.getRemainderMultiParallelism(isChoiceGpu);
        log.info("计算出剩余可执行的并行度:{}", remainderParallelism);
        boolean needWait = false;
        if (DictConstants.SYSTEM_USER_ID.equals(clusterUserId)){
            // 不需要判断证书
        } else{
            if (remainderSimulationLicense <= 0){
                log.info("multiProjectId:{},仿真数量不够用clusterUserId:{}", projectWaitQueueEntity.getProjectId(), clusterUserId);
                needWait = true;
            }
        }
//        else if (projectWaitQueueEntity.getRunState() <0 && remainderParallelism >0){
//            // 首次执行，且可以执行，不进入等待队列
//            if (remainderSimulationLicense <= 0){
//                needWait = true;
//            } else {
//                // 消耗一个证书
//                if (!DictConstants.SYSTEM_USER_ID.equals(clusterUserId)){
//                    projectDomainService.useLicense(clusterUserId, 1);
//                }
//            }
//        }
        if (remainderParallelism <= 0 || needWait) {
            waitMulti(projectWaitQueueEntity);
        } else if (remainderParallelism < parallelism) {
            // 初始执行
            if (projectWaitQueueEntity.getRunState() <0){
                log.info("多模式仿真初始执行,创建kafka topic:{}", projectWaitQueueEntity.getProjectId());
                KafkaUtil.createTopic(kafkaAdminClient, projectWaitQueueEntity.getProjectId(), projectWaitQueueEntity.getMultiTaskMessageEntityList().size(), (short) 1);   // 创建主题
                TimeUnit.SECONDS.sleep(5);
                if (!DictConstants.SYSTEM_USER_ID.equals(clusterUserId)){
                    projectDomainService.useLicense(clusterUserId, 1);
                }
            }
            // 执行完之后删除之前的list.
            // 现有逻辑不需要不需要再删除list了
            runMulti(remainderParallelism, projectWaitQueueEntity, isChoiceGpu);
            Integer runState = projectWaitQueueEntity.getRunState();
            int runSt = remainderParallelism + runState;
            waitMulti(MultiProjectWaitQueueEntity.builder().waitingParallelism(parallelism - remainderParallelism)
                .runState(runSt)
                .multiTaskMessageEntityList(projectWaitQueueEntity.getMultiTaskMessageEntityList())
                .kafkaParamList(projectWaitQueueEntity.getKafkaParamList())
                .build());
        } else {
            if (projectWaitQueueEntity.getRunState() <0){
                log.info("多模式仿真初始执行,剩余并行度够用,创建kafka topic:{}", projectWaitQueueEntity.getProjectId());
                KafkaUtil.createTopic(kafkaAdminClient, projectWaitQueueEntity.getProjectId(), projectWaitQueueEntity.getMultiTaskMessageEntityList().size(), (short) 1);   // 创建主题
                TimeUnit.SECONDS.sleep(5);
                if (!DictConstants.SYSTEM_USER_ID.equals(clusterUserId)){
                    projectDomainService.useLicense(clusterUserId, 1);
                }
            }
            runMulti(parallelism, projectWaitQueueEntity, isChoiceGpu);
            // 能执行完也需要删除之前redis key
            waitMulti(MultiProjectWaitQueueEntity.builder().waitingParallelism(0)
                    .runState(projectWaitQueueEntity.getMultiTaskMessageEntityList().size() -1)
                    .kafkaParamList(projectWaitQueueEntity.getKafkaParamList())
                    .multiTaskMessageEntityList(projectWaitQueueEntity.getMultiTaskMessageEntityList()).build()
            );
        }
    }

}
