package com.css.simulation.resource.scheduler.domain.service;

import api.common.pojo.common.ResponseBodyVO;
import api.common.pojo.constants.DictConstants;
import api.common.pojo.param.project.MultiCreateYamlRet;
import api.common.pojo.param.project.MultiSimulationSceneKafkaParam;
import api.common.pojo.vo.project.MultiSimulationSceneCarVO;
import api.common.util.*;
import com.alibaba.fastjson.JSONObject;
import com.css.simulation.resource.scheduler.adapter.entity.ProjectStartMessageEntity;
import com.css.simulation.resource.scheduler.app.entity.ProjectWaitQueueEntity;
import com.css.simulation.resource.scheduler.infra.configuration.custom.CustomConfiguration;
import com.css.simulation.resource.scheduler.infra.configuration.entity.NodeEntity;
import com.css.simulation.resource.scheduler.infra.configuration.kubernetes.KubernetesConfiguration;
import com.css.simulation.resource.scheduler.infra.configuration.minio.MinioConfiguration;
import com.css.simulation.resource.scheduler.infra.db.entity.AlgorithmExpandEntity;
import com.css.simulation.resource.scheduler.infra.db.entity.SimulationAutomaticProjectEntity;
import com.css.simulation.resource.scheduler.infra.db.entity.SimulationAutomaticSubprojectEntity;
import com.css.simulation.resource.scheduler.infra.db.entity.SimulationManualProjectEntity;
import com.css.simulation.resource.scheduler.infra.db.mysql.mapper.*;
import com.css.simulation.resource.scheduler.infra.entity.MultiTaskMessageEntity;
import com.css.simulation.resource.scheduler.infra.entity.PrefixEntity;
import com.css.simulation.resource.scheduler.infra.entity.ProjectEntity;
import com.css.simulation.resource.scheduler.infra.entity.UserEntity;
import com.css.simulation.resource.scheduler.infra.configuration.kubernetes.KubernetesUtil;
import com.css.simulation.resource.scheduler.infra.db.redis.CustomRedisClient;
import com.css.simulation.resource.scheduler.infra.db.redis.RedisUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import lombok.SneakyThrows;
import lombok.Synchronized;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.File;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * 任务结果打分
 */
@Component
@Slf4j
public class ProjectDomainService {

    @Value("${scheduler.linux-path.pod-yaml-directory}")
    private String podYamlDirectory;
    @Value("${scheduler.linux-path.vtd-pod-template-yaml}")
    private String vtdPodTemplateYaml;
    @Value("${scheduler.linux-path.carsim-pod-template-yaml}")
    private String carsimPodTemplateYaml;
    @Value("${scheduler.simulation-cloud-ip}")
    private String simulationCloudIp;
    @Value("${spring.kafka.bootstrap-servers}")
    private String kafkaIp;


    /**
     * 以下是多模式仿真任务
     */
    @Value("${scheduler.linux-path.multi-pod-yaml-directory}")
    private String multiPodYamlDirectory;
    @Value("${scheduler.linux-path.multi-vtd-pod-template-first-yaml}")
    private String multiVtdPodTemplateFirstYaml;
    @Value("${scheduler.linux-path.multi-vtd-pod-template-last-yaml}")
    private String multiVtdPodTemplateLastYaml;

    @Value("${scheduler.linux-path.multi-vtd-pod-template-init-yaml}")
    private String multiVtdPodTemplateInitYaml;

    @Value("${scheduler.linux-path.multi-vtd-xml-generator}")
    private String multiVtdXmlGenerator;

    // -------------------------------- Comment --------------------------------
    @Resource
    private StringRedisTemplate stringRedisTemplate;
    @Resource
    private VehicleMapper vehicleMapper;
    @Resource
    private ClusterMapper clusterMapper;
    @Resource
    private CustomConfiguration customConfiguration;
    @Resource
    private KubernetesConfiguration kubernetesConfiguration;
    @Resource
    private MinioConfiguration minioConfiguration;
    @Resource
    private SimulationManualProjectMapper simulationManualProjectMapper;
    @Resource
    private SimulationAutomaticProjectMapper simulationAutomaticProjectMapper;
    @Resource
    private SimulationAutomaticSubprojectMapper simulationAutomaticSubprojectMapper;
    @Resource
    private CustomRedisClient customRedisClient;
    @Resource
    private UserMapper userMapper;
    @Resource
    private ApiClient apiClient;
    @Resource
    private AlgorithmMapper algorithmMapper;
    @Resource
    private AlgorithmExpandMapper algorithmExpandMapper;


    /**
     * 创建一个临时 yaml，node 在最后用 # 号隔开
     */
    @SneakyThrows
    public String createTempYaml(String projectId, String vehicleConfigId, String modelType, String algorithmDockerImage, String nodeName, int kafkaPartition, long kafkaOffset, String isChoiceGpu, Integer cpuOrder) {
        String podName = getRandomPodName(projectId);   // 生成 podName
        String podYaml = getPodYamlName(nodeName, podName);     // 模板文件名称
        String yamlPath = podYamlDirectory + podYaml;
        String finalYaml;
        String podString;

        if ("1".equals(modelType)) {
            podString = FileUtil.read(new File(vtdPodTemplateYaml));
            String replace0 = podString.replace("vtd-container", "vtd-" + projectId);
            String replace1 = replace0.replace("simulation-cloud-ip", simulationCloudIp);
            String replace2 = replace1.replace("kafka-ip", kafkaIp);
            String replace3 = replace2.replace("kafka-topic", projectId);     // 消息主题名称为 projectId
            String replace4 = replace3.replace("kafka-partition", "\"" + kafkaPartition + "\"");
            String replace5 = replace4.replace("kafka-offset", "\"" + kafkaOffset + "\"");
            String replace6 = replace5.replace("minio-ip", minioConfiguration.getEndpointWithoutHttp());
            String replace7 = replace6.replace("minio-bucket", minioConfiguration.getBucketName());
            String replace8 = replace7.replace("minio-access-key", minioConfiguration.getAccessKey());
            String replace9 = replace8.replace("minio-secret-key", minioConfiguration.getSecretKey());

            String replace10 = replace9.replace("algorithm-container", "algorithm-" + projectId);
            String replace11 = replace10.replaceAll("algorithm-image", algorithmDockerImage);

            String replace12 = replace11.replace("pod-name", podName); // pod 名称包括 projectId 和 随机字符串
            String replace13 = replace12.replace("namespace-name", kubernetesConfiguration.getNamespace()); // pod 名称包括 projectId 和 随机字符串
            String replace14 = replace13.replace("node-name", nodeName);     // 指定 pod 运行节点

            String replace15;
            if (cpuOrder != null) {
                replace15 = replace14.replace("cpu-order", "\"" + cpuOrder + "\"");     // 指定 cpu 编号
                stringRedisTemplate.opsForValue().set("project:" + projectId + ":pod:" + podName + ":cpu", String.valueOf(cpuOrder));    //  pod 运行使用的 cpu编号
            } else {
                replace15 = replace14;
            }


            if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
                String replace16 = replace15.replace("vtd-image", kubernetesConfiguration.getImageVtdGpu());
                finalYaml = replace16.replace("vtd-command", kubernetesConfiguration.getCommandVtdGpu());
            } else if (DictConstants.NOT_USE_GPU.equals(isChoiceGpu)) {
                String replace16 = replace15.replace("vtd-image", kubernetesConfiguration.getImageVtdNogpu());
                finalYaml = replace16.replace("vtd-command", kubernetesConfiguration.getCommandVtdNogpu());
            } else {
                throw new RuntimeException("是否使用 gpu：" + isChoiceGpu);
            }
        } else if ("2".equals(modelType)) {
            podString = FileUtil.read(new File(carsimPodTemplateYaml));
            String replace0 = podString.replace("vtd-container", "vtd-" + projectId);
            String replace1 = replace0.replace("simulation-cloud-ip", simulationCloudIp);
            String replace2 = replace1.replace("kafka-ip", kafkaIp);
            String replace3 = replace2.replace("kafka-topic", projectId);     // 消息主题名称为 projectId
            String replace4 = replace3.replace("kafka-partition", "\"" + kafkaPartition + "\"");
            String replace5 = replace4.replace("kafka-offset", "\"" + kafkaOffset + "\"");
            String replace6 = replace5.replaceAll("minio-ip", minioConfiguration.getEndpointWithoutHttp());
            String replace7 = replace6.replaceAll("minio-access-key", minioConfiguration.getAccessKey());
            String replace8 = replace7.replaceAll("minio-secret-key", minioConfiguration.getSecretKey());

            String replace9 = replace8.replace("algorithm-container", "algorithm-" + projectId);
            String replace10 = replace9.replaceAll("algorithm-image", algorithmDockerImage);

            String replace11 = replace10.replace("carsim-container", "carsim-" + projectId);
            String replace12 = replace11.replace("carsim-image", kubernetesConfiguration.getCarsimImage());
            String replace13 = replace12.replace("carsim-command", kubernetesConfiguration.getCarsimCommand());
            String replace14 = replace13.replace("minio-bucket", minioConfiguration.getBucketName());
            String replace15 = replace14.replace("par-path", vehicleMapper.selectParPathByVehicleConfigId(vehicleConfigId));

            String replace16 = replace15.replace("pod-name", podName); // pod 名称包括 projectId 和 随机字符串
            String replace17 = replace16.replace("namespace-name", kubernetesConfiguration.getNamespace()); // pod 名称包括 projectId 和 随机字符串
            String replace18 = replace17.replace("node-name", nodeName);     // 指定 pod 运行节点

            String replace19;
            if (cpuOrder != null) {
                replace19 = replace18.replace("cpu-order", "\"" + cpuOrder + "\"");     // 指定 cpu 编号
                stringRedisTemplate.opsForValue().set("project:" + projectId + ":pod:" + podName + ":cpu", String.valueOf(cpuOrder));    //  pod 运行使用的 cpu编号
            } else {
                replace19 = replace18;
            }

            if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
                log.info("k8s参数为：" + kubernetesConfiguration);
                log.info("yaml模板为：" + replace12);
                String replace20 = replace19.replace("vtd-image", kubernetesConfiguration.getImageVtdGpu());
                finalYaml = replace20.replace("vtd-command", kubernetesConfiguration.getCommandVtdCarsimGpu());
            } else if (DictConstants.NOT_USE_GPU.equals(isChoiceGpu)) {
                String replace20 = replace19.replace("vtd-image", kubernetesConfiguration.getImageVtdNogpu());
                finalYaml = replace20.replace("vtd-command", kubernetesConfiguration.getCommandVtdCarsimNogpu());
            } else {
                throw new RuntimeException("是否使用 gpu：" + isChoiceGpu);
            }
        } else {
            throw new RuntimeException("模型类型错误：" + modelType);
        }
        log.info("保存项目 " + projectId + " 的 yaml 文件：" + yamlPath);
        FileUtil.writeStringToLocalFile(finalYaml, yamlPath);
        // 保存 yaml 地址
        String yamlRedisKey = "project:" + projectId + ":node:" + nodeName + ":yaml:" + podName;
        stringRedisTemplate.opsForValue().set(yamlRedisKey, yamlPath);
        return yamlRedisKey;
    }

    @SneakyThrows
    public MultiCreateYamlRet createMultiTempYaml(String projectId, MultiSimulationSceneKafkaParam multiSimulationSceneKafkaParam, MultiTaskMessageEntity messageEntity, String nodeName, int kafkaPartition, long kafkaOffset, String isChoiceGpu) {
        String podName = getMultiRandomPodName(projectId);   // 生成 podName
        String podYaml = getMultiPodYamlName(nodeName, podName);     // 模板文件名称
        String yamlPath = multiPodYamlDirectory + podYaml;
        String podString = FileUtil.read(new File(multiVtdPodTemplateFirstYaml));
        String algorithmStr = FileUtil.read(new File(multiVtdPodTemplateLastYaml));
        String algorithmInitStr = FileUtil.read(new File(multiVtdPodTemplateInitYaml));

        podString = podString.replace("pod-name", podName);
        podString = podString.replace("namespace-name", kubernetesConfiguration.getMultiNamespace());
        podString = podString.replace("node-name", nodeName);
        podString = podString.replace("vtd-container", "vtd-multi-" + messageEntity.getInfo().getScene_id());
        podString = podString.replace("vtd-image", kubernetesConfiguration.getMultiImageVtdGpu());
        podString = podString.replace("simulation-cloud-ip", simulationCloudIp);
        podString = podString.replace("kafka-ip", kafkaIp);
        podString = podString.replace("minio-ip", minioConfiguration.getEndpointWithoutHttp());
        podString = podString.replace("minio-access-key", minioConfiguration.getAccessKey());
        podString = podString.replace("minio-secret-key", minioConfiguration.getSecretKey());
        podString = podString.replace("minio-bucket", minioConfiguration.getBucketName());
        podString = podString.replace("kafka-partition", "\"" + kafkaPartition + "\"");
        podString = podString.replace("kafka-offset", "\"" + kafkaOffset + "\"");
        podString = podString.replace("cpu-order", "\""+1+"\"");
        podString = podString.replace("vtd-command", kubernetesConfiguration.getMultiCommandVtdGpu());
        podString = podString.replace("kafka-topic", projectId);

        List<MultiSimulationSceneCarVO> simulationSceneCarVOList = multiSimulationSceneKafkaParam.getSimulationSceneCarVOList();

        podString = podString.replace("num-of-vehicle", "\"" + simulationSceneCarVOList.size() + "\"");
        int cameraVehicleId = 0;
        for (int i = 0; i < simulationSceneCarVOList.size(); i++) {
            Integer isSimulationCar = simulationSceneCarVOList.get(i).getIsSimulationCar();
            if (isSimulationCar == 1){
                cameraVehicleId = i + 1;
            }
        }
        podString = podString.replace("camera-vehicle-id", "\"" + cameraVehicleId + "\"");

        StringBuffer stringBuffer = new StringBuffer();
        StringBuffer stringBufferInit = new StringBuffer();
        int basePort = 11310;
        for (int i = 0; i < simulationSceneCarVOList.size(); i++) {
            MultiSimulationSceneCarVO car = simulationSceneCarVOList.get(i);
            String temp = algorithmStr.replace("algorithm-container", "algorithm-multi-" + car.getId());
            String algorithmId = car.getAlgorithmId();
            String algorithmImage = customRedisClient.get(getMultiAlgorithmIdRedisKey(algorithmId, projectId));
            temp = temp.replace("algorithm-image", algorithmImage);
            temp = temp.replace("ros-master-uri", "http://localhost:" + String.valueOf(basePort + i + 1));
            temp = temp.replace("vehicle-id", "\"" + String.valueOf(i+1) + "\"");
            String pathEnd = car.getPathEnd();
            JSONObject object = JSONObject.parseObject(pathEnd);
            temp = temp.replace("end-coord", String.valueOf(object.get("x")) + "," + String.valueOf(object.get("y")));
            stringBuffer.append(temp);

            String tempInit = algorithmInitStr.replace("algorithm-image", algorithmImage).replace("init-name", "init-" + car.getId());
            stringBufferInit.append(tempInit);
        }
        podString = podString.replace("algorithmContainerDemo", stringBuffer.toString());
        podString = podString.replace("algorithmContainerInit", stringBufferInit.toString());

        log.info("保存项目projectId:" + projectId + "sceneId:" + messageEntity.getInfo().getScene_id() + "的 yaml 文件：" + yamlPath);
        FileUtil.writeStringToLocalFile(podString, yamlPath);
        FileUtil.writeStringToLocalFile(podString, multiPodYamlDirectory + "bak/" + podYaml);
        // 保存 yaml 地址
        String yamlRedisKey = "multi_project:" + projectId+":scene:" + messageEntity.getInfo().getScene_id() + ":node:" + nodeName + ":yaml:" + podName;
        stringRedisTemplate.opsForValue().set(yamlRedisKey, yamlPath);
        MultiCreateYamlRet multiCreateYamlRet = new MultiCreateYamlRet();
        multiCreateYamlRet.setProjectId(projectId);
        multiCreateYamlRet.setSceneId(messageEntity.getInfo().getScene_id());
        multiCreateYamlRet.setYamlRedisKey(yamlRedisKey);
        multiCreateYamlRet.setPodName(podName);
        return multiCreateYamlRet;
    }

    public String getMultiAlgorithmIdRedisKey(String algorithmId,String projectId){
        return "multi_project:" + projectId + ":algorithmId:" + algorithmId + ":docker-image";
    }


    public String getIsChoiceGpuByProjectId(String projectId) {
        return getProjectByProjectId(projectId).getIsChoiceGpu();
    }


    @SneakyThrows
    public void deleteYamlByProjectId(String projectId) {
        List<String> absolutePathList = FileUtil.listAbsolutePath(podYamlDirectory);
        for (String absolutePath : absolutePathList) {
            if (absolutePath.contains(projectId)) {
                boolean delete = new File(absolutePath).delete();
                log.debug("删除结果：" + delete);
            }
        }
    }


    public List<com.css.simulation.resource.scheduler.infra.entity.NodeEntity> getNodeListToCount(Map<String, Integer> nodeMap) {
        List<com.css.simulation.resource.scheduler.infra.entity.NodeEntity> result = new ArrayList<>();
        nodeMap.forEach((nodeName, parallelism) -> {
            for (int i = 0; i < parallelism; i++) {
                result.add(new com.css.simulation.resource.scheduler.infra.entity.NodeEntity(nodeName, 0));
            }
        });
        return result;
    }

    /**
     * 判断算法是否已经导入
     *
     * @return 算法是否已经导入
     */
    @SneakyThrows
    public boolean isImported(String dockerImageWithoutVersion) {
        String dockerImageListString = LinuxUtil.execute("docker images");
        return dockerImageListString.contains(dockerImageWithoutVersion);
    }

    public String getRandomPodName(String projectId) {
        return "project-" + projectId + "-" + StringUtil.getRandomEightBitUUID();
    }

    public String getPodYamlName(String nodeName, String podName) {
        return nodeName + "#" + podName + ".yaml";
    }

    public String getMultiRandomPodName(String projectId) {
        return "multi-project-" + projectId + "-" + StringUtil.getRandomEightBitUUID();
    }

    public String getMultiPodYamlName(String nodeName, String podName) {
        return nodeName + "#" + podName + ".yaml";
    }

    public void deletePod(String podName) {
        try {
            // 先删除 redis key
            KubernetesUtil.deletePod(apiClient, kubernetesConfiguration.getNamespace(), podName);
            log.info("等待 pod " + podName + " 的资源释放完成。");
            TimeUnit.SECONDS.sleep(3);
        } catch (ApiException apiException) {
            log.info("pod " + podName + " 已删除。");
        } catch (Exception e) {
            e.printStackTrace();
            log.error("删除 pod " + podName + " 报错。", e);
        }
    }

    public void deleteMultiProjectPod(String podName) {
        try {
            // 先删除 redis key
            KubernetesUtil.deletePod(apiClient, kubernetesConfiguration.getMultiNamespace(), podName);
            log.info("等待 pod " + podName + " 的资源释放完成。");
            TimeUnit.SECONDS.sleep(3);
        } catch (ApiException apiException) {
            log.info("pod " + podName + " 已删除。");
        } catch (Exception e) {
            e.printStackTrace();
            log.error("删除 pod " + podName + " 报错。", e);
        }
    }


    public String getNodeNameOfPod(String projectId, String podName) {
        String key = "project:" + projectId + ":pod:" + podName + ":node";
        final String s = stringRedisTemplate.opsForValue().get(key);
        if (StringUtil.isEmpty(s)) {
            throw new RuntimeException("无法获取 pod 运行所在节点：" + key);
        }
        return s;
    }

    public String getNodeNameOfMultiProjectPod(String projectId, String podName) {
        String key = "project:" + projectId + ":pod:" + podName + ":node";
        final String s = stringRedisTemplate.opsForValue().get(key);
        if (StringUtil.isEmpty(s)) {
            throw new RuntimeException("无法获取 pod 运行所在节点：" + key);
        }
        return s;
    }

    public String getNodeNameOfPod2(String projectId, String podName) {
        String key = "project:" + projectId + ":pod:" + podName + ":node";
        final String s = stringRedisTemplate.opsForValue().get(key);
        if (StringUtil.isEmpty(s)) {
            throw new RuntimeException("无法获取 pod 运行所在节点：" + key);
        }
        return s;
    }


    /**
     * 更改一个名字继续启动
     *
     * @param projectId   项目 id
     * @param nodeName    运行 pod 的节点名称
     * @param lastPodName 即将删除的 pod 名称
     */
    @SneakyThrows
    public void createNextPod(String projectUserId, String projectId, String projectType, String nodeName, String lastPodName) {
        final String isChoiceGpu = getIsChoiceGpuByProjectId(projectId);
        log.info("删除上一个 pod：projectId={}，nodeName={}，lastPodName={}", projectId, nodeName, lastPodName);
        String cpuOrderString = stringRedisTemplate.opsForValue().get("project:" + projectId + ":pod:" + lastPodName + ":cpu");
        deletePod(lastPodName);
        customRedisClient.delete("project:" + projectId + ":pod:" + lastPodName + ":cpu");
        //2 获取新的 yaml 信息
        final Set<String> yamlPathCacheKeySet = RedisUtil.getKeySetByPrefix(stringRedisTemplate, "project:" + projectId + ":node:" + nodeName + ":yaml");
        if (CollectionUtil.isEmpty(yamlPathCacheKeySet)) {
            // 如果当前节点没有下一个yaml，则返回一个并行度。
            log.info("节点 {} 已经没有项目 {} 的等待执行的 yaml。", nodeName, projectId);
            incrementOneParallelism(isChoiceGpu, nodeName);
            releaseLicense(getClusterUserIdByProjectUserId(projectUserId), getModelTypeByProjectIdAndProjectType(projectId, projectType), 1);
        } else {
            final String yamlPathCacheKey = new ArrayList<>(yamlPathCacheKeySet).get(0);
            final String absolutePath = stringRedisTemplate.opsForValue().get(yamlPathCacheKey);
            // 修改 cpu 编号
            Optional.ofNullable(cpuOrderString).orElseThrow(() -> new RuntimeException("pod " + lastPodName + " 缓存的 cpu 编号为空。"));
            final String read = FileUtil.read(absolutePath);
            final String replace = read.replace("cpu-order", "\"" + cpuOrderString + "\"");
            FileUtil.writeStringToLocalFile(replace, absolutePath);
            // 创建 pod
            createPod(projectId, yamlPathCacheKey, cpuOrderString);
            log.info("创建项目 " + projectId + " 在节点 " + nodeName + " 的下一个 pod，使用 cpu 编号为 " + cpuOrderString);
        }
    }

    /**
     * @param redisKey yaml 地址的缓存 key
     */
    public void createPodBegin(String projectId, String redisKey) {
        final String podYamlPath = stringRedisTemplate.opsForValue().get(redisKey);
        if (podYamlPath == null) {
            throw new RuntimeException("根据缓存 key 获取 yaml 地址为 null：" + redisKey);
        }
        stringRedisTemplate.delete(redisKey);
        String nodeName = new File(podYamlPath).getName().split("#")[0];
        String podName = podYamlPath.split("#")[1].split("\\.")[0];
        stringRedisTemplate.opsForValue().set("project:" + projectId + ":pod:" + podName + ":node", nodeName);    // 将 pod 运行在哪个 node 上记录到 redis
        new Thread(() -> KubernetesUtil.applyYaml(podYamlPath), "apply-" + podName).start();
    }

    public void createMultiPodBegin(MultiCreateYamlRet multiCreateYamlRet, String redisKey) {
        final String podYamlPath = stringRedisTemplate.opsForValue().get(redisKey);
        if (podYamlPath == null) {
            throw new RuntimeException("根据缓存 key 获取 yaml 地址为 null：" + redisKey);
        }
        stringRedisTemplate.delete(redisKey);
        String nodeName = new File(podYamlPath).getName().split("#")[0];
        String podName = podYamlPath.split("#")[1].split("\\.")[0];
        stringRedisTemplate.opsForValue().set("multi_project:" + multiCreateYamlRet.getProjectId() + ":sceneId:" + multiCreateYamlRet.getSceneId() + ":pod:" + podName + ":node", nodeName);    // 将 pod 运行在哪个 node 上记录到 redis
        new Thread(() -> KubernetesUtil.applyYaml(podYamlPath), "apply-" + podName).start();
    }


    /**
     * @param redisKey yaml 地址的缓存 key
     */
    public void createPod(String projectId, String redisKey, String cpuOrderString) {
        final String podYamlPath = stringRedisTemplate.opsForValue().get(redisKey);
        if (podYamlPath == null) {
            throw new RuntimeException("根据缓存 key 获取 yaml 地址为 null：" + redisKey);
        }
        stringRedisTemplate.delete(redisKey); // 在缓存中删除该yaml，因为已经开始运行。
        String nodeName = new File(podYamlPath).getName().split("#")[0];
        String podName = podYamlPath.split("#")[1].split("\\.")[0];
        stringRedisTemplate.opsForValue().set("project:" + projectId + ":pod:" + podName + ":node", nodeName);    // 将 pod 运行在哪个 node 上记录到 redis
        stringRedisTemplate.opsForValue().set("project:" + projectId + ":pod:" + podName + ":cpu", cpuOrderString);    // 将 pod 运行在哪个 node 上记录到 redis
        new Thread(() -> KubernetesUtil.applyYaml(podYamlPath), "create-" + podName).start();
    }


    public ProjectEntity getProjectByProjectId(String projectId) {
        ProjectEntity manualProjectEntity = simulationManualProjectMapper.selectById(projectId);
        ProjectEntity autoSubProjectEntity = simulationAutomaticSubprojectMapper.selectById(projectId);
        if (manualProjectEntity != null) {
            return manualProjectEntity;
        } else if (autoSubProjectEntity != null) {
            return autoSubProjectEntity;
        }
        throw new RuntimeException("不存在项目：" + projectId);
    }

    /**
     * 节点剩余可用并行度列表
     *
     * @return 节点映射（节点名，并行度）
     */
    public Map<String, Integer> getRemainderNodeMap(String isChoiceGpu) {
        Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            List<NodeEntity> initialNodeList = kubernetesConfiguration.getGpuNodeList();
            log.info("预设并行度的GPU节点列表为：" + initialNodeList);
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismKey = "gpu-node:" + nodeName + ":parallelism";
                String restParallelismString = stringRedisTemplate.opsForValue().get(restParallelismKey);
                int restParallelism;
                if (restParallelismString == null) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set(restParallelismKey, String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                }
                resultNodeMap.put(nodeName, restParallelism);
            }
            log.info("剩余并行度的GPU节点列表为：" + resultNodeMap);
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            List<NodeEntity> initialNodeList = kubernetesConfiguration.getCpuNodeList();
            log.info("预设并行度的CPU节点列表为：" + initialNodeList);
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismKey = "cpu-node:" + nodeName + ":parallelism";
                String restParallelismString = stringRedisTemplate.opsForValue().get(restParallelismKey);
                int restParallelism;
                if (restParallelismString == null) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set(restParallelismKey, String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                }
                resultNodeMap.put(nodeName, restParallelism);
            }
            log.info("剩余并行度的CPU节点列表为：" + resultNodeMap);
        }
        return resultNodeMap;
    }

    public Map<String, Integer> getMultiRemainderNodeMap(String isChoiceGpu) {
        Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            List<NodeEntity> initialNodeList = kubernetesConfiguration.getMultiGpuNodeList();
            log.info("多模式仿真预设并行度的GPU节点列表为：" + initialNodeList);
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismKey = "gpu-node:" + nodeName + ":parallelism";
                String restParallelismString = stringRedisTemplate.opsForValue().get(restParallelismKey);
                int restParallelism;
                if (restParallelismString == null) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set(restParallelismKey, String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                }
                resultNodeMap.put(nodeName, restParallelism);
            }
            log.info("剩余并行度的GPU节点列表为：" + resultNodeMap);
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            List<NodeEntity> initialNodeList = kubernetesConfiguration.getMultiCpuNodeList();
            log.info("多模式仿真预设并行度的CPU节点列表为：" + initialNodeList);
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismKey = "cpu-node:" + nodeName + ":parallelism";
                String restParallelismString = stringRedisTemplate.opsForValue().get(restParallelismKey);
                int restParallelism;
                if (restParallelismString == null) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set(restParallelismKey, String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                }
                resultNodeMap.put(nodeName, restParallelism);
            }
            log.info("剩余并行度的CPU节点列表为：" + resultNodeMap);
        }
        return resultNodeMap;
    }


    /**
     * 根据并行度获取用于执行的节点列表
     * 根据剩余可用并行度降序排序
     *
     * @return 节点映射（节点名，并行度）
     */
    public Map<String, Integer> getNodeMapToUse(String isChoiceGpu, int parallelism) {
        List<NodeEntity> initialNodeList; // 预设并行度的节点列表
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getGpuNodeList();
            log.info("预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("gpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("gpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.info("剩余并行度的节点列表为：" + restNodeList);
            Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
            if (!CollectionUtil.isEmpty(restNodeList)) {
                if (restNodeList.size() == 1) {
                    NodeEntity tempNode = restNodeList.get(0);
                    String tempNodeName = tempNode.getHostname();
                    int tempParallelism = tempNode.getParallelism();
                    resultNodeMap.put(tempNodeName, Math.min(tempParallelism, parallelism));
                }
                if (restNodeList.size() > 1) {
                    for (int i = 0; i < parallelism; i++) {
                        // 每次降序排序都取剩余并行度最大的一个。
                        restNodeList.sort((o1, o2) -> o2.getParallelism() - o1.getParallelism());
                        NodeEntity tempNode = restNodeList.get(0);
                        String tempNodeName = tempNode.getHostname();
                        int tempParallelism = tempNode.getParallelism();
                        if (tempParallelism > 0) {
                            tempNode.setParallelism(tempParallelism - 1);
                            CollectionUtil.addValueToMap(resultNodeMap, 1, tempNodeName);
                        }
                    }
                }
            }
            log.info("即将使用节点的并行度为：" + resultNodeMap);
            return resultNodeMap;
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getCpuNodeList();
            log.info("预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("cpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("cpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.info("剩余并行度的节点列表为：" + restNodeList);
            Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
            if (!CollectionUtil.isEmpty(restNodeList)) {
                if (restNodeList.size() == 1) {
                    NodeEntity tempNode = restNodeList.get(0);
                    String tempNodeName = tempNode.getHostname();
                    int tempParallelism = tempNode.getParallelism();
                    resultNodeMap.put(tempNodeName, Math.min(tempParallelism, parallelism));
                }
                if (restNodeList.size() > 1) {
                    for (int i = 0; i < parallelism; i++) {
                        // 每次降序排序都取剩余并行度最大的一个。
                        restNodeList.sort((o1, o2) -> o2.getParallelism() - o1.getParallelism());
                        NodeEntity tempNode = restNodeList.get(0);
                        String tempNodeName = tempNode.getHostname();
                        int tempParallelism = tempNode.getParallelism();
                        if (tempParallelism > 0) {
                            tempNode.setParallelism(tempParallelism - 1);
                            CollectionUtil.addValueToMap(resultNodeMap, 1, tempNodeName);
                        }
                    }
                }
            }
            log.info("即将使用节点的并行度为：" + resultNodeMap);
            return resultNodeMap;
        } else {
            throw new RuntimeException("未知是否使用 GPU：" + isChoiceGpu);
        }

    }

    public Map<String, Integer> getMultiNodeMapToUse(String isChoiceGpu, int parallelism) {
        List<NodeEntity> initialNodeList; // 预设并行度的节点列表
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiGpuNodeList();
            log.info("多模式仿真任务预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("gpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("gpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.info("多模式仿真剩余并行度的节点列表为：" + restNodeList);
            Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
            for (NodeEntity node: restNodeList) {
                resultNodeMap.put(node.getHostname(), node.getParallelism());
            }
            log.info("即将使用节点的并行度为：" + resultNodeMap);
            return resultNodeMap;
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiCpuNodeList();
            log.info("多模式仿真预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("cpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("cpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.info("多模式仿真剩余并行度的节点列表为：" + restNodeList);
            Map<String, Integer> resultNodeMap = new HashMap<>();    // 用于执行的节点映射（节点名，并行度）
            for (NodeEntity node: restNodeList) {
                resultNodeMap.put(node.getHostname(), node.getParallelism());
            }
            log.info("多模式仿真即将使用节点的并行度为：" + resultNodeMap);
            return resultNodeMap;
        } else {
            throw new RuntimeException("未知是否使用 GPU：" + isChoiceGpu);
        }

    }

    public void setMultiNodeMapUse(String isChoiceGpu, Map<String, Integer> map) {
        List<NodeEntity> initialNodeList; // 预设并行度的节点列表
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiGpuNodeList();
            log.info("多模式仿真任务预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                if (map.containsKey(nodeName)){
                    Integer integer = map.get(nodeName);
                    stringRedisTemplate.opsForValue().set("gpu-node:" + nodeName + ":parallelism", String.valueOf(integer));
                }
            }
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiCpuNodeList();
            log.info("多模式仿真预设并行度的节点列表为：" + initialNodeList);
            // 遍历所有节点，获取还有剩余并行度的节点
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                if (map.containsKey(nodeName)){
                    Integer integer = map.get(nodeName);
                    stringRedisTemplate.opsForValue().set("cpu-node:" + nodeName + ":parallelism", String.valueOf(integer));
                }
            }
        }else {
            throw new RuntimeException("未知是否使用 GPU：" + isChoiceGpu);
        }

    }

    /**
     * 获取集群剩余并行度
     *
     * @return 集群剩余并行度
     */
    public int getRemainderParallelism(String isChoiceGpu) {
        List<NodeEntity> initialNodeList; // 预设并行度的节点列表
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getGpuNodeList(); // 预设并行度的节点列表
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("gpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("gpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            return restNodeList.size() == 0 ? 0 : restNodeList.stream().mapToInt(NodeEntity::getParallelism).sum();
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getCpuNodeList(); // 预设并行度的节点列表
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("cpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("cpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.debug("集群剩余并行度为：" + restNodeList);
            return restNodeList.size() == 0 ? 0 : restNodeList.stream().mapToInt(NodeEntity::getParallelism).sum();
        } else {
            throw new RuntimeException("未知是否使用 GPU：" + isChoiceGpu);
        }

    }

    public int getRemainderMultiParallelism(String isChoiceGpu) {
        List<NodeEntity> initialNodeList; // 预设并行度的节点列表
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiGpuNodeList(); // 预设并行度的节点列表
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("gpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("gpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            return restNodeList.size() == 0 ? 0 : restNodeList.stream().mapToInt(NodeEntity::getParallelism).sum();
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            initialNodeList = kubernetesConfiguration.getMultiCpuNodeList(); // 预设并行度的节点列表
            // 遍历所有节点，获取还有剩余并行度的节点
            List<NodeEntity> restNodeList = new ArrayList<>();    // 剩余并行度的节点列表
            for (NodeEntity kubernetesNodeSource : initialNodeList) {
                NodeEntity kubernetesNodeCopy = kubernetesNodeSource.clone();
                String nodeName = kubernetesNodeCopy.getHostname();   // 节点名称
                int maxParallelism = kubernetesNodeCopy.getParallelism();
                String restParallelismString = stringRedisTemplate.opsForValue().get("cpu-node:" + nodeName + ":parallelism");// 获取节点剩余并行度的 key
                // -------------------------------- Comment --------------------------------
                int restParallelism;
                if (restParallelismString == null || Integer.parseInt(restParallelismString) > maxParallelism) {    // 如果剩余可用并行度没有值，说明是第一次查询，则重置成最大并行度的预设值
                    restParallelism = maxParallelism;
                    stringRedisTemplate.opsForValue().set("cpu-node:" + nodeName + ":parallelism", String.valueOf(restParallelism));
                } else {
                    restParallelism = Integer.parseInt(restParallelismString);
                    kubernetesNodeCopy.setParallelism(restParallelism);
                }
                if (restParallelism > 0) {
                    restNodeList.add(kubernetesNodeCopy);
                }
            }
            log.debug("集群剩余并行度为：" + restNodeList);
            return restNodeList.size() == 0 ? 0 : restNodeList.stream().mapToInt(NodeEntity::getParallelism).sum();
        } else {
            throw new RuntimeException("未知是否使用 GPU：" + isChoiceGpu);
        }

    }


    public PrefixEntity getRedisPrefixByUserIdAndProjectIdAndTaskId(String userId, String projectId, String taskId) {
        //3 获取用户类型（管理员账户、管理员子账户、普通账户、普通子账户）（独占、共享）
        UserEntity userEntity = userMapper.selectById(userId);
        String roleCode = userEntity.getRoleCode();
        String useType = userEntity.getUseType();
        String clusterId;
        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
            clusterId = DictConstants.SYSTEM_CLUSTER_ID;
        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
            clusterId = clusterMapper.selectByUserId(userId).getId();
        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
                clusterId = clusterMapper.selectByUserId(userId).getId();
            } else {    //3-4 共享子账户，根据父账户的共享节点排队
                String parentUserId = userEntity.getCreateUserId();
                clusterId = clusterMapper.selectByUserId(parentUserId).getId();
            }
        } else {
            throw new RuntimeException("未知账户类型，无法获取集群信息。");
        }
        String clusterPrefix = "cluster:" + clusterId;
        String clusterRunningPrefix = clusterPrefix + ":running";
        String projectRunningKey = clusterRunningPrefix + ":" + projectId;
        String taskTickKey = projectRunningKey + ":task:" + taskId + ":tick";
        String taskPodKey = projectRunningKey + ":task:" + taskId + ":pod";
        String taskRetryKey = projectRunningKey + ":task:" + taskId + ":retry";
        String taskMessageKey = projectRunningKey + ":task:" + taskId + ":message";

        return PrefixEntity.builder().clusterPrefix(clusterPrefix).clusterRunningPrefix(clusterRunningPrefix).projectRunningKey(projectRunningKey).taskTickKey(taskTickKey).taskPodKey(taskPodKey).taskRetryKey(taskRetryKey).taskMessageKey(taskMessageKey).build();

    }


    public PrefixEntity getRedisPrefixByClusterIdAndProjectId(String clusterId, String projectId) {
        String clusterPrefix = "cluster:" + clusterId;
        String clusterRunningPrefix = clusterPrefix + ":running";
        String clusterWaitingPrefix = clusterPrefix + ":waiting";
        String projectRunningKey = clusterRunningPrefix + ":" + projectId;
        String projectWaitingKey = clusterWaitingPrefix + ":" + projectId;
        String projectCheckKey = clusterWaitingPrefix + ":" + projectId + ":check";

        return PrefixEntity.builder().clusterPrefix(clusterPrefix).clusterRunningPrefix(clusterRunningPrefix).clusterWaitingPrefix(clusterWaitingPrefix).projectRunningKey(projectRunningKey).projectWaitingKey(projectWaitingKey).projectCheckKey(projectCheckKey).build();

    }


    public PrefixEntity getRedisPrefixByProjectIdAndProjectType(String projectId, String projectType) {
        String userId;
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            userId = simulationManualProjectMapper.selectCreateUserById(projectId);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            userId = simulationAutomaticSubprojectMapper.selectCreateUserById(projectId);
        } else {
            throw new RuntimeException("未知的项目类型。");
        }

        //3 获取用户类型（管理员账户、管理员子账户、普通账户、普通子账户）（独占、共享）
        UserEntity userEntity = userMapper.selectById(userId);
        String roleCode = userEntity.getRoleCode();
        String useType = userEntity.getUseType();
        String clusterId;
        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
            clusterId = DictConstants.SYSTEM_CLUSTER_ID;
        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
            clusterId = clusterMapper.selectByUserId(userId).getId();
        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
                clusterId = clusterMapper.selectByUserId(userId).getId();
            } else {    //3-4 共享子账户，根据父账户的共享节点排队
                String parentUserId = userEntity.getCreateUserId();
                clusterId = clusterMapper.selectByUserId(parentUserId).getId();
            }
        } else {
            throw new RuntimeException("未知账户类型，无法获取集群信息。");
        }
        String clusterPrefix = "cluster:" + clusterId;
        String clusterRunningPrefix = clusterPrefix + ":running";
        String clusterWaitingPrefix = clusterPrefix + ":waiting";
        String projectRunningKey = clusterRunningPrefix + ":" + projectId;
        String projectWaitingKey = clusterWaitingPrefix + ":" + projectId;
        String projectCheckKey = projectRunningKey + ":check";

        return PrefixEntity.builder().clusterPrefix(clusterPrefix).clusterRunningPrefix(clusterRunningPrefix).clusterWaitingPrefix(clusterWaitingPrefix).projectRunningKey(projectRunningKey).projectWaitingKey(projectWaitingKey).projectCheckKey(projectCheckKey).build();
    }


    public void incrementOneParallelism(String isChoiceGpu, String nodeName) {
        incrementParallelism(isChoiceGpu, nodeName, 1L);
    }

    public void incrementParallelism(String isChoiceGpu, String nodeName, long number) {
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            //1 先检查缓存中的并行度是否超过，超过了就不加缓存的并行度了，常用于测试
            String key = "gpu-node:" + nodeName + ":parallelism";
            final int currentRestParallelism = Integer.parseInt(customRedisClient.get(key));
            final List<NodeEntity> nodeList = kubernetesConfiguration.getGpuNodeList();
            for (NodeEntity node : nodeList) {
                if (nodeName.equals(node.getHostname())) {
                    if (currentRestParallelism + 1 <= node.getParallelism()) {
                        customRedisClient.increment(key, number);
                    }
                }
            }
            log.info("归还 GPU 节点 {} 的 {} 个并行度。", nodeName, number);
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            //1 先检查缓存中的并行度是否超过，超过了就不加缓存的并行度了，常用于测试
            String key = "cpu-node:" + nodeName + ":parallelism";
            final int currentRestParallelism = Integer.parseInt(customRedisClient.get(key));
            final List<NodeEntity> nodeList = kubernetesConfiguration.getCpuNodeList();
            for (NodeEntity node : nodeList) {
                if (nodeName.equals(node.getHostname())) {
                    if (currentRestParallelism + 1 <= node.getParallelism()) {
                        customRedisClient.increment(key, number);
                    }
                }
            }
            log.info("归还 CPU 节点 {} 的 {} 个并行度。", nodeName, number);
        }


    }
    public void incrementOneMultiParallelism(String isChoiceGpu, String nodeName) {
        incrementMultiParallelism(isChoiceGpu, nodeName, 1L);
    }
    public void incrementMultiParallelism(String isChoiceGpu, String nodeName, long number) {
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            String key = "gpu-node:" + nodeName + ":parallelism";
            final int currentRestParallelism = Integer.parseInt(customRedisClient.get(key));
            final List<NodeEntity> nodeList = kubernetesConfiguration.getMultiGpuNodeList();
            for (NodeEntity node : nodeList) {
                if (nodeName.equals(node.getHostname())) {
                    if (currentRestParallelism + 1 <= node.getParallelism()) {
                        customRedisClient.increment(key, number);
                    }
                }
            }
            log.info("归还 GPU 节点 {} 的 {} 个并行度。", nodeName, number);
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            String key = "cpu-node:" + nodeName + ":parallelism";
            final int currentRestParallelism = Integer.parseInt(customRedisClient.get(key));
            final List<NodeEntity> nodeList = kubernetesConfiguration.getMultiCpuNodeList();
            for (NodeEntity node : nodeList) {
                if (nodeName.equals(node.getHostname())) {
                    if (currentRestParallelism + 1 <= node.getParallelism()) {
                        customRedisClient.increment(key, number);
                    }
                }
            }
            log.info("归还 CPU 节点 {} 的 {} 个并行度。", nodeName, number);
        }


    }

    public void decrementParallelism(String isChoiceGpu, String nodeName, long number) {
        if (DictConstants.USE_GPU.equals(isChoiceGpu)) {
            String key = "gpu-node:" + nodeName + ":parallelism";
            customRedisClient.decrement(key, number);
            log.info("获取节点 {} 的 {} 个 GPU 并行度。", nodeName, number);
        } else if (DictConstants.USE_CPU.equals(isChoiceGpu)) {
            String key = "cpu-node:" + nodeName + ":parallelism";
            customRedisClient.decrement(key, number);
            log.info("获取节点 {} 的 {} 个 CPU 并行度。", nodeName, number);
        }

    }


    public void resetNodeParallelism() {
        //* -------------------------------- GPU节点 --------------------------------
        final List<NodeEntity> gpuNodeList = kubernetesConfiguration.getGpuNodeList();
        log.info("GPU 节点：" + gpuNodeList);
        gpuNodeList.forEach((node) -> customRedisClient.set("gpu-node:" + node.getHostname() + ":parallelism", String.valueOf(node.getParallelism())));
        //* -------------------------------- CPU节点 --------------------------------
        final List<NodeEntity> cpuNodeList = kubernetesConfiguration.getCpuNodeList();
        log.info("CPU 节点：" + cpuNodeList);
        cpuNodeList.forEach((node) -> customRedisClient.set("cpu-node:" + node.getHostname() + ":parallelism", String.valueOf(node.getParallelism())));
    }

    /**
     * 判断项目完成度
     */
    @Synchronized
    public boolean complete(PrefixEntity redisPrefix, String projectId) {
        boolean result = false;
        String projectMessageModelJson = customRedisClient.get(redisPrefix.getProjectRunningKey());
        ProjectStartMessageEntity projectStartMessageEntity = JsonUtil.jsonToBean(projectMessageModelJson, ProjectStartMessageEntity.class);
        int taskTotal = projectStartMessageEntity.getTaskTotal();
        int taskCompleted = projectStartMessageEntity.getTaskCompleted();
        log.info("项目 " + projectId + " 完成进度为：" + (taskCompleted + 1) + "/" + taskTotal);
        if (taskCompleted + 1 == taskTotal) {
            result = true;
        } else {    // 项目没有完成
            projectStartMessageEntity.setTaskCompleted(taskCompleted + 1);  // 增加已完成任务数
            stringRedisTemplate.opsForValue().set(redisPrefix.getProjectRunningKey(), JsonUtil.beanToJson(projectStartMessageEntity));
        }
        return result;
    }


    public Integer getUsingLicenseNumber(String userId, String licenseType) {
        String key;
        if (DictConstants.LICENSE_TYPE_SIMULATION.equals(licenseType)) {
            key = "user:" + userId + ":using-license:simulation";
        } else if (DictConstants.LICENSE_TYPE_DYNAMIC.equals(licenseType)) {
            key = "user:" + userId + ":using-license:dynamic";
        } else {
            throw new RuntimeException("未知证书类型：" + licenseType);
        }
        final String usingLicense = customRedisClient.get(key);
        if (StringUtil.isEmpty(usingLicense)) {
            customRedisClient.set(key, "0");
            return 0;
        } else {
            return Integer.parseInt(usingLicense);
        }
    }

    public void useLicense(String clusterUserId, String modelType, int parallelism) {
        if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
            useLicense("user:" + clusterUserId + ":using-license:simulation", parallelism);
        } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
            useLicense("user:" + clusterUserId + ":using-license:simulation", parallelism);
            useLicense("user:" + clusterUserId + ":using-license:dynamic", parallelism);
        } else {
            throw new RuntimeException("未知模型类型：" + modelType);
        }
    }

    public void useLicense(String key, int number) {
        final String usingLicense = customRedisClient.get(key);
        if (StringUtil.isEmpty(usingLicense)) {
            customRedisClient.set(key, String.valueOf(number));
        } else {
            customRedisClient.increment(key, number);
        }
    }


    public void releaseLicense(String clusterUserId, String modelType, int parallelism) {
        if (DictConstants.MODEL_TYPE_VTD.equals(modelType)) {
            releaseLicense("user:" + clusterUserId + ":using-license:simulation", parallelism);
        } else if (DictConstants.MODEL_TYPE_CARSIM.equals(modelType)) {
            releaseLicense("user:" + clusterUserId + ":using-license:simulation", parallelism);
            releaseLicense("user:" + clusterUserId + ":using-license:dynamic", parallelism);
        } else {
            throw new RuntimeException("未知模型类型：" + modelType);
        }
    }

    private void releaseLicense(String key, int number) {
        final String usingLicense = customRedisClient.get(key);
        if (StringUtil.isEmpty(usingLicense) || Integer.parseInt(usingLicense) <= number) {
            customRedisClient.set(key, "0");
        } else {
            customRedisClient.decrement(key, number);
        }
    }

    public String getClusterUserIdByProjectUserId(String projectUserId) {
        String clusterUserId;
        final UserEntity userEntity = userMapper.selectById(projectUserId);
        final String roleCode = userEntity.getRoleCode();
        final String useType = userEntity.getUseType();
        if (DictConstants.ROLE_CODE_SYSADMIN.equals(roleCode) || DictConstants.ROLE_CODE_ADMIN.equals(roleCode)) {  //3-1 管理员账户和管理员子账户直接执行
            clusterUserId = projectUserId;
        } else if (DictConstants.ROLE_CODE_UESR.equals(roleCode)) { //3-2 普通账户，不管是独占还是共享，都在自己的集群里排队，根据自己的独占节点排队
            clusterUserId = projectUserId;
        } else if (DictConstants.ROLE_CODE_SUBUESR.equals(roleCode)) {
            if (DictConstants.USER_TYPE_EXCLUSIVE.equals(useType)) {   //3-3 普通子账户，根据自己的独占节点排队
                clusterUserId = projectUserId;
            } else {    //3-4 共享子账户，根据父账户的共享节点排队
                clusterUserId = userEntity.getCreateUserId();
            }
        } else {
            throw new RuntimeException("未知角色类型：" + roleCode);
        }
        return clusterUserId;
    }


    public String getModelTypeByProjectIdAndProjectType(String projectId, String projectType) {
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            return simulationManualProjectMapper.selectModelTypeByProjectId(projectId);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            return simulationAutomaticSubprojectMapper.selectModelTypeByProjectId(projectId);
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
    }

    public String getUserIdByProjectIdAndProjectType(String projectId, String projectType) {
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            return simulationManualProjectMapper.selectCreateUserById(projectId);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            return simulationAutomaticSubprojectMapper.selectCreateUserById(projectId);
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
    }


    public UserEntity getUserEntityByProjectIdAndProjectType(String projectId, String projectType) {
        UserEntity userEntity;
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            userEntity = userMapper.selectByManualProjectId(projectId);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            userEntity = userMapper.selectByAutoSubProjectId(projectId);
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
        return userEntity;
    }

    public UserEntity getUserEntityByMultiProjectId(String projectId) {
        UserEntity userEntity = userMapper.selectByMultiManualProjectId(projectId);
        return userEntity;
    }


    @Async("pool1")
    @SneakyThrows
    public void selectProjectDetailsById(String projectType, String projectId) {
        String tokenUrl = customConfiguration.getTokenUri() + "?grant_type=client_credentials" + "&client_id=" + customConfiguration.getClientId() + "&client_secret=" + customConfiguration.getClientSecret();
        log.info("获取仿真云平台 token：" + tokenUrl);
        String response = HttpUtil.get(tokenUrl);
        ObjectMapper objectMapper = new ObjectMapper();
        JsonNode jsonNode = objectMapper.readTree(response);
        String accessToken = jsonNode.path("access_token").asText();
        log.info("仿真云平台 token 为：" + accessToken);
        Map<String, String> headers = new HashMap<>();
        headers.put("Authorization", "Bearer " + accessToken);
        Map<String, String> params = new HashMap<>();
        params.put("projectType", projectType);
        params.put("id", projectId);
        String result = HttpUtil.post(customConfiguration.getProjectDetailsUri(), headers, params);
        log.info("访问仿真云平台项目详情接口：" + customConfiguration.getProjectDetailsUri() + "，请求头为：" + headers + "，请求体为：" + params + "结果为：" + result);
        ResponseBodyVO<?> responseBodyVO = JsonUtil.jsonToBean(result, ResponseBodyVO.class);
        String projectDetailsVOJson = JsonUtil.beanToJson(responseBodyVO.getInfo());
        TimeUnit.SECONDS.sleep(3);
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            simulationManualProjectMapper.updateProjectDetails(projectId, projectDetailsVOJson);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            simulationAutomaticSubprojectMapper.updateProjectDetails(projectId, projectDetailsVOJson);
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
    }

    @Async("pool1")
    @SneakyThrows
    public void selectProjectReportById(String projectType, String projectId) {
//        String tokenUrl = customConfiguration.getTokenUri() + "?grant_type=client_credentials" + "&client_id=" + customConfiguration.getClientId() + "&client_secret=" + customConfiguration.getClientSecret();
        String tokenUrl = "http://10.14.85.241/simulation/oauth/client/sign/username_for_scheduler";
        Map<String, String> params1 = new HashMap<>();
        params1.put("username", "sysadmin");
        params1.put("password", "123456");
        String response = HttpUtil.post(tokenUrl, null, params1);
        log.info("获取仿真云平台 token：" + tokenUrl + "，结果为：" + response);
        String accessTokenString = JsonUtil.getString(response, "info.access_token");
        final String accessToken = accessTokenString.substring(1, accessTokenString.length() - 1);
        log.info("仿真云平台 token 为：" + accessToken);
        Map<String, String> headers = new HashMap<>();
        headers.put("Authorization", "Bearer " + accessToken);
        Map<String, String> params2 = new HashMap<>();
        params2.put("projectType", projectType);
        params2.put("id", projectId);
        String result = HttpUtil.post(customConfiguration.getProjectReportUri(), headers, params2);
        log.info("访问仿真云平台项目报告接口：" + customConfiguration.getProjectReportUri() + "，请求头为：" + headers + "，请求体为：" + params2 + "结果为：" + result);
        ResponseBodyVO<?> responseBodyVO = JsonUtil.jsonToBean(result, ResponseBodyVO.class);
        String projectReportVOJson = JsonUtil.beanToJson(responseBodyVO.getInfo());
        TimeUnit.SECONDS.sleep(3);
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            simulationManualProjectMapper.updateProjectReport(projectId, projectReportVOJson);
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            simulationAutomaticSubprojectMapper.updateProjectReport(projectId, projectReportVOJson);
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
    }

    public List<ProjectWaitQueueEntity> getWaitQueue() {
        final String waitQueueJson = customRedisClient.get(DictConstants.PROJECT_WAIT_QUEUE_KEY);
        if (StringUtil.isNotEmpty(waitQueueJson)) {
            return JsonUtil.jsonToList(waitQueueJson, ProjectWaitQueueEntity.class);
        } else {
            return null;
        }
    }

    public void setWaitQueue(List<ProjectWaitQueueEntity> projectWaitQueueEntities) {
        try {
            String projectWaitQueueEntitiesJson = JsonUtil.listToJson(projectWaitQueueEntities);
            customRedisClient.set(DictConstants.PROJECT_WAIT_QUEUE_KEY, projectWaitQueueEntitiesJson);
        } catch (JsonProcessingException e) {
            throw new RuntimeException(e);
        }
    }

    public void removeWaitQueue(String waitType, String projectId) {
        List<ProjectWaitQueueEntity> waitQueue = getWaitQueue();
        if (CollectionUtil.isNotEmpty(waitQueue)) {
            if (DictConstants.PROJECT_WAIT_TYPE_EXECUTE.equals(waitType)) {
                for (ProjectWaitQueueEntity projectWaitQueueEntity : waitQueue) {
                    if (projectWaitQueueEntity.getProjectStartMessageEntity().getProjectId().equals(projectId) && projectWaitQueueEntity.getWaitingType().equals(DictConstants.PROJECT_WAIT_TYPE_EXECUTE)) {
                        waitQueue.remove(projectWaitQueueEntity);
                        break;
                    }
                }
                setWaitQueue(waitQueue);
            } else if (DictConstants.PROJECT_WAIT_TYPE_EXPAND.equals(waitType)) {
                for (ProjectWaitQueueEntity projectWaitQueueEntity : waitQueue) {
                    if (projectWaitQueueEntity.getProjectStartMessageEntity().getProjectId().equals(projectId) && projectWaitQueueEntity.getWaitingType().equals(DictConstants.PROJECT_WAIT_TYPE_EXPAND)) {
                        waitQueue.remove(projectWaitQueueEntity);
                        break;
                    }
                }
            } else if (DictConstants.PROJECT_WAIT_TYPE_ALL.equals(waitType)) {
                for (ProjectWaitQueueEntity projectWaitQueueEntity : waitQueue) {
                    if (projectWaitQueueEntity.getProjectStartMessageEntity().getProjectId().equals(projectId)) {
                        waitQueue.remove(projectWaitQueueEntity);
                        break;
                    }
                }
            } else {
                throw new RuntimeException("未知等待类型：" + waitType);
            }
        } else {
            waitQueue = new ArrayList<>();
        }
        setWaitQueue(waitQueue);

    }


    /**
     * 如果是算法平台算法则修改状态
     *
     * @param projectType
     * @param projectId
     */
    public void checkAlgorithmIsExpand(String projectType, String projectId, String algorithmStatus) {
        String algorithmType;
        String algorithmId;
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            algorithmType = simulationManualProjectMapper.selectAlgorithmTypeById(SimulationManualProjectEntity.builder().id(projectId).build());
            algorithmId = simulationManualProjectMapper.selectAlgorithmById(SimulationManualProjectEntity.builder().id(projectId).build());
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            String automaticProjectId = simulationAutomaticSubprojectMapper.selectParentIdById(SimulationAutomaticSubprojectEntity.builder().id(projectId).build());
            algorithmType = simulationAutomaticProjectMapper.selectAlgorithmTypeById(SimulationAutomaticProjectEntity.builder().id(automaticProjectId).build());
            algorithmId = simulationAutomaticProjectMapper.selectAlgorithmById(SimulationAutomaticProjectEntity.builder().id(automaticProjectId).build());
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
        if (DictConstants.ALGORITHM_UPLOAD_MODE_PLATFORM.equals(algorithmType)) {
            algorithmExpandMapper.updateStatusByAlgorithmId(AlgorithmExpandEntity.builder().algorithmId(algorithmId).status(algorithmStatus).build());
            log.info("------- 修改算法平台算法" + algorithmId + "的状态为测试完成");
        }
    }


    public String getAlgorithmDockerImageByProjectId(String projectId) {
        return customRedisClient.get("project:" + projectId + ":docker-image");
    }

    public String getAlgorithmDockerImageByProjectTypeAndProjectId(String projectType, String projectId) {
        String algorithmType;
        String algorithmId;
        if (DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)) {
            algorithmType = simulationManualProjectMapper.selectAlgorithmTypeById(SimulationManualProjectEntity.builder().id(projectId).build());
            algorithmId = simulationManualProjectMapper.selectAlgorithmById(SimulationManualProjectEntity.builder().id(projectId).build());
        } else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)) {
            String automaticProjectId = simulationAutomaticSubprojectMapper.selectParentIdById(SimulationAutomaticSubprojectEntity.builder().id(projectId).build());
            algorithmType = simulationAutomaticProjectMapper.selectAlgorithmTypeById(SimulationAutomaticProjectEntity.builder().id(automaticProjectId).build());
            algorithmId = simulationAutomaticProjectMapper.selectAlgorithmById(SimulationAutomaticProjectEntity.builder().id(automaticProjectId).build());
        } else {
            throw new RuntimeException("未知项目类型：" + projectType);
        }
        if (DictConstants.ALGORITHM_UPLOAD_MODE_PLATFORM.equals(algorithmType)) {
            return algorithmExpandMapper.selectDockerImageByAlgorithmId(algorithmId);
        } else {
            return algorithmMapper.selectDockerImageById(algorithmId);
        }
    }
}
