mlx

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/ProjectScheduler.java

@@ -197,8 +197,12 @@ public class ProjectScheduler {
             if (StringUtil.isNotEmpty(lastNowString) && taskNumber == 0 && Long.parseLong(TimeUtil.getNowString()) - Long.parseLong(lastNowString) > (long) 120 * 1000) {
                 // 删除检查
                 redisTemplate.delete(redisPrefix.getProjectCheckKey());
-                // 删除 job
-                KubernetesUtil.deleteJob(apiClient, "default", "project-" + projectId);
+                try {
+                    // 删除 job
+                    KubernetesUtil.deleteJob(apiClient, "default", "project-" + projectId);
+                } catch (Exception e) {
+                    log.info("ProjectScheduler--projectCheck 删除项目 " + projectId + " 的 job 失败，可能是已经删除！");
+                }
                 log.info("ProjectScheduler--projectCheck 重新执行项目" + projectId);
                 KubernetesUtil.applyYaml(hostname, username, password, jobYaml + "project-" + projectId + ".yaml");
             }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ProjectService.java

@@ -320,12 +320,12 @@ public class ProjectService {
 
     /**
      * 运行
-     * @param jobTemplateYamlPathSource
-     * @param projectId
-     * @param algorithmDockerImage
-     * @param completions
-     * @param parallelism
-     * @param jobTemplateYamlPathTarget
+     * @param jobTemplateYamlPathSource 模板文件
+     * @param projectId 项目id
+     * @param algorithmDockerImage 算法镜像
+     * @param completions 完成度
+     * @param parallelism 并行度
+     * @param jobTemplateYamlPathTarget 执行文件
      */
     @SneakyThrows
     public void transferAndRunYaml(String jobTemplateYamlPathSource, String projectId, String algorithmDockerImage, long completions, long parallelism, String jobTemplateYamlPathTarget) {

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/util/KubernetesUtil.java

@@ -5,6 +5,7 @@ import io.kubernetes.client.openapi.ApiClient;
 import io.kubernetes.client.openapi.ApiException;
 import io.kubernetes.client.openapi.apis.BatchV1Api;
 import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.V1JobList;
 import io.kubernetes.client.openapi.models.V1Namespace;
 import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1PodList;
@@ -100,8 +101,59 @@ public class KubernetesUtil {
     }
 
 
-    public static void deleteJob(ApiClient apiClient, String namespaceName, String jobName) throws ApiException {
+    public static List<String> getJob(ApiClient apiClient, String namespaceName) throws ApiException {
         BatchV1Api batchV1Api = new BatchV1Api(apiClient);
-        batchV1Api.deleteNamespacedJob(jobName, namespaceName, null, null, null, null, null, null);
+        V1JobList v1JobList;
+        if (namespaceName == null || "".equals(namespaceName) || "default".equals(namespaceName)) {
+            v1JobList = batchV1Api.listNamespacedJob(
+                    "default",
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null);
+        } else if ("all".equals(namespaceName)) {
+            v1JobList = batchV1Api.listJobForAllNamespaces(
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null);
+        } else {
+            v1JobList = batchV1Api.listNamespacedJob(
+                    namespaceName,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null,
+                    null);
+        }
+        return v1JobList.getItems().stream().map(pod -> Objects.requireNonNull(pod.getMetadata()).getName()).collect(Collectors.toList());
+    }
+
+    public static void deleteJob(ApiClient apiClient, String namespaceName, String jobNameToDelete) throws ApiException {
+        List<String> runningJobNameList = getJob(apiClient, namespaceName);
+        for (String runningJobName : runningJobNameList) {
+            if (jobNameToDelete.equals(runningJobName)) {
+                BatchV1Api batchV1Api = new BatchV1Api(apiClient);
+                batchV1Api.deleteNamespacedJob(jobNameToDelete, namespaceName, null, null, null, null, null, null);
+                break;
+            }
+        }
     }
 }