mlx

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumer.java

@@ -452,6 +452,7 @@ public class ManualProjectConsumer {
         String projectId = jsonNode.path("projectId").asText();
         projectMapper.updateProjectState(projectId, DictConstants.PROJECT_TERMINATED, TimeUtil.getNowForMysql());
         LinuxUtil.execute("kubectl delete job project-" + projectId);
+        redisTemplate.delete(manualProjectTopic + ":" + projectId + ":check");
     }
 
 

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/TickScheduler.java

@@ -1,15 +1,14 @@
 package com.css.simulation.resource.scheduler.scheduler;
 
 import api.common.pojo.constants.DictConstants;
-import api.common.util.CollectionUtil;
-import api.common.util.LinuxUtil;
-import api.common.util.StringUtil;
-import api.common.util.TimeUtil;
+import api.common.util.*;
 import com.css.simulation.resource.scheduler.mapper.ProjectMapper;
 import com.css.simulation.resource.scheduler.mapper.TaskMapper;
 import com.css.simulation.resource.scheduler.pojo.po.TaskPO;
 import com.css.simulation.resource.scheduler.service.TaskService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.session.ClientSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
@@ -75,49 +74,51 @@ public class TickScheduler {
      */
     @Scheduled(fixedDelay = 30 * 1000)
     public void checkProject() throws IOException {
-//        SshClient client = SshUtil.getClient();
-//        ClientSession session = SshUtil.getSession(client, "182.92.203.182", "root", "CICV2022test");
+        SshClient client = SshUtil.getClient();
+        ClientSession session = SshUtil.getSession(client, "182.92.203.182", "root", "CICV2022test");
 
         //1 查询出正在运行中的 project
         List<String> projectIdList = projectMapper.selectIdByState("20");
         log.info("TickScheduler-------checkProject 查询出正在运行中的 project" + projectIdList);
         //2 根据 projectId 获取 pod
         projectIdList.forEach(projectId -> {
-
-            String key = manualProjectTopic + ":" + projectId + ":check";
-            String nowString = TimeUtil.getNowString();
-
             try {
+                String key = manualProjectTopic + ":" + projectId + ":check";
                 String lastNowString = redisTemplate.opsForValue().get(manualProjectTopic + ":" + projectId + ":check");
-//                String podList = SshUtil.execute(session, "kubectl get pod | grep project-" + projectId);
-                String podList = LinuxUtil.execute("kubectl get pod | grep project-" + projectId);
+                String podList = SshUtil.execute(session, "kubectl get pod | grep project-" + projectId);
+//                String podList = LinuxUtil.execute("kubectl get pod | grep project-" + projectId);
                 log.info("TickScheduler-------checkProject 项目 " + projectId + " 正在运行的 pod 为" + podList);
                 int taskNumber = StringUtil.countSubString(podList, "project");
                 if (StringUtil.isEmpty(lastNowString) && taskNumber == 0) {
-                    redisTemplate.opsForValue().set(key, nowString);
+                    redisTemplate.opsForValue().set(key, TimeUtil.getNowString());
                 }
 
                 if (StringUtil.isNotEmpty(lastNowString) && taskNumber == 0) {
                     // 判断两次是否超过2分钟
                     //3 如果 pod 为空，则重启 job
                     long lastNow = Long.parseLong(lastNowString);
-                    long now = Long.parseLong(nowString);
-                    if (now - lastNow > (long) 60 * 1000) {
-//                        SshUtil.execute(session,"kubectl delete job project-" + projectId);
-                        LinuxUtil.execute("kubectl delete job project-" + projectId);
+                    long now = Long.parseLong(TimeUtil.getNowString());
+
+                    if (now - lastNow > (long) 120 * 1000) {
+                        redisTemplate.opsForValue().set(key, TimeUtil.getNowString());
+                        SshUtil.execute(session, "kubectl delete job project-" + projectId);
+//                        LinuxUtil.execute("kubectl delete job project-" + projectId);
                         Thread.sleep(15000);
                         while (true) {
                             log.info("TickScheduler-------checkProject 准备重启项目 " + projectId);
-                            String podList2 = LinuxUtil.execute("kubectl get pod | grep project-" + projectId);
+//                            String podList2 = LinuxUtil.execute("kubectl get pod | grep project-" + projectId);
+                            String podList2 = SshUtil.execute(session, "kubectl get pod | grep project-" + projectId);
                             log.info("TickScheduler-------checkProject 项目 " + projectId + " 剩余的 pod 信息为：" + podList2);
                             int taskNumber2 = StringUtil.countSubString(podList2, "project");
                             if (taskNumber2 == 0) {
                                 break;
                             }
                         }
+                        Thread.sleep(15000);
                         log.info("TickScheduler-------checkProject 重新执行项目" + projectId);
                         String jobTemplateYamlPathTarget = jobYaml + "project-" + projectId + ".yaml";
-                        LinuxUtil.execute("kubectl apply -f " + jobTemplateYamlPathTarget);
+                        SshUtil.execute(session, "kubectl apply -f " + jobTemplateYamlPathTarget);
+//                        LinuxUtil.execute("kubectl apply -f " + jobTemplateYamlPathTarget);
                     }
                 }
             } catch (IOException | InterruptedException e) {
@@ -125,8 +126,8 @@ public class TickScheduler {
             }
         });
 
-//        session.close();
-//        client.stop();
+        session.close();
+        client.stop();
 
     }
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/TaskService.java

@@ -141,7 +141,7 @@ public class TaskService {
             log.info("TaskService--state 修改任务 " + taskId + "的状态为 Terminated，pod 名称为：" + podName
                     + "，并执行命令：" + command);
             String resultTerminated = LinuxUtil.execute1(command);
-            taskMapper.updateSuccessStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
+            taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
             redisTemplate.delete("podName:" + taskId);
         } else if ("PendingAnalysis".equals(state)) {
             LinuxUtil.execute("kubectl delete pod " + podName);