mlx 算法清理

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/docker/DockerConfiguration.java

@@ -0,0 +1,13 @@
+package com.css.simulation.resource.scheduler.configuration.docker;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties(prefix = "docker")
+@Data
+public class DockerConfiguration {
+    private Integer maxAlgorithmImage;
+    private Integer minAlgorithmImage;
+}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/AlgorithmMapper.java

@@ -5,6 +5,8 @@ import com.css.simulation.resource.scheduler.pojo.po.AlgorithmPO;
 import org.apache.ibatis.annotations.*;
 import org.apache.ibatis.type.JdbcType;
 
+import java.util.List;
+
 /**
  * 场景查询
  * scene_natural
@@ -14,7 +16,9 @@ import org.apache.ibatis.type.JdbcType;
 @Mapper
 public interface AlgorithmMapper {
 
-    @Results(id = "index", value = {
+    @Results(id = "algorithm", value = {
+            @Result(column = "id", property = "id", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "algorithm_code", property = "algorithmCode", jdbcType = JdbcType.VARCHAR),
             @Result(column = "minio_path", property = "minioPath", jdbcType = JdbcType.VARCHAR),
             @Result(column = "docker_import", property = "dockerImport", jdbcType = JdbcType.VARCHAR),
             @Result(column = "docker_image", property = "dockerImage", jdbcType = JdbcType.VARCHAR),
@@ -23,7 +27,9 @@ public interface AlgorithmMapper {
             @Result(column = "git_user_name", property = "gitUserName", jdbcType = JdbcType.VARCHAR),
             @Result(column = "git_password", property = "gitPassword", jdbcType = JdbcType.VARCHAR)
     })
-    @Select("select minio_path,\n" +
+    @Select("select id,\n" +
+            "       algorithm_code,\n" +
+            "       minio_path,\n" +
             "       docker_import,\n" +
             "       docker_image,\n" +
             "       upload_mode,\n" +
@@ -31,13 +37,40 @@ public interface AlgorithmMapper {
             "       git_user_name,\n" +
             "       git_password\n" +
             "from algorithm\n" +
-            "where is_deleted = '0'\n" +
-            "  and id = #{id}")
+            "where id = #{id}")
     AlgorithmPO selectById(@Param("id") String id);
 
+    @Select("select id,\n" +
+            "       algorithm_code,\n" +
+            "       minio_path,\n" +
+            "       docker_import,\n" +
+            "       docker_image,\n" +
+            "       upload_mode,\n" +
+            "       git_url,\n" +
+            "       git_user_name,\n" +
+            "       git_password\n" +
+            "from algorithm\n" +
+            "where docker_import = #{dockerImport}\n" +
+            "order by create_time desc")
+    List<AlgorithmPO> selectByDockerImport(@Param("dockerImport") String dockerImport);
+
     @Update("update algorithm\n" +
             "set docker_import = #{dockerImport},\n" +
             "    docker_image  = #{dockerImage}\n" +
             "where id = #{algorithmId}")
     void updateDockerImportAndDockerImageById(@Param("dockerImport") String dockerImport, @Param("dockerImage") String dockerImage, @Param("algorithmId") String algorithmId);
+
+    @Update("<script>" +
+            "update algorithm\n" +
+            "set docker_import = #{dockerImport},\n" +
+            "    docker_image  = #{dockerImage}\n" +
+            "where 1 = 1\n" +
+            "<if test='idList != null and idList.size() > 0'>\n" +
+            "   and id in\n" +
+            "       <foreach collection='idList' index='index' item='item' open='(' close=')' separator=','>\n" +
+            "           #{item}" +
+            "       </foreach>\n" +
+            "</if>\n" +
+            "</script>")
+    void updateDockerImportAndDockerImageByIdList(@Param("dockerImport") String dockerImport, @Param("dockerImage") String dockerImage, @Param("idList") List<String> idList);
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/AutoProjectMapper.java

@@ -1,19 +0,0 @@
-package com.css.simulation.resource.scheduler.mapper;
-
-
-import org.apache.ibatis.annotations.Mapper;
-import org.apache.ibatis.annotations.Param;
-import org.apache.ibatis.annotations.Update;
-
-@Mapper
-public interface AutoProjectMapper {
-
-
-    @Update("update simulation_automatic_project\n" +
-            "set now_run_state = #{nowRunState}\n" +
-            "where id = (select parent_id\n" +
-            "            from simulation_automatic_subproject\n" +
-            "            where id = #{subProjectId})")
-    void updateProjectStateBySubProjectId(@Param("subProjectId") String id, @Param("nowRunState") String state);
-
-}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/AlgorithmPO.java

@@ -8,6 +8,8 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 @AllArgsConstructor
 public class AlgorithmPO {
+    private String id;
+    private String algorithmCode;
     private String minioPath;
     private String dockerImport;
     private String dockerImage;

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/to/KubernetesNodeTO.java

@@ -12,4 +12,7 @@ import lombok.NoArgsConstructor;
 public class KubernetesNodeTO {
     private String name;
     private Long maxParallelism;
+    private String hostname;
+    private String username;
+    private String password;
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/scheduler/AlgorithmScheduler.java

@@ -0,0 +1,69 @@
+package com.css.simulation.resource.scheduler.scheduler;
+
+import api.common.util.SshUtil;
+import com.css.simulation.resource.scheduler.configuration.docker.DockerConfiguration;
+import com.css.simulation.resource.scheduler.configuration.kubernetes.KubernetesConfiguration;
+import com.css.simulation.resource.scheduler.mapper.AlgorithmMapper;
+import com.css.simulation.resource.scheduler.pojo.po.AlgorithmPO;
+import com.css.simulation.resource.scheduler.pojo.to.KubernetesNodeTO;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.session.ClientSession;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.List;
+
+@Component
+@Slf4j
+public class AlgorithmScheduler {
+
+    @Resource
+    KubernetesConfiguration kubernetesConfiguration;
+    @Resource
+    DockerConfiguration dockerConfiguration;
+    @Resource
+    AlgorithmMapper algorithmMapper;
+
+
+    /**
+     * 定时删除算法镜像
+     */
+    @Scheduled(fixedDelay = 60 * 1000)
+    @SneakyThrows
+    public void removeAlgorithmImage() {
+        int maxAlgorithmImage = dockerConfiguration.getMaxAlgorithmImage();
+        int minAlgorithmImage = dockerConfiguration.getMinAlgorithmImage();
+        //1 查询已经导入的算法，根据创建时间排序
+        List<AlgorithmPO> algorithmImportedList = algorithmMapper.selectByDockerImport("1");
+        int algorithmImportNumber = algorithmImportedList.size();
+        log.info("AlgorithmScheduler--removeAlgorithmImage kubernetes 各个节点已经导入 " + algorithmImportNumber + " 个算法。");
+        if (algorithmImportNumber < maxAlgorithmImage) {
+            return;
+        }
+        //2 如果已经导入到 docker 中的算法镜像已经达到 maxAlgorithmImage 个，则清理算法镜像到 minAlgorithmImage 个；
+        List<String> algorithmIdToUpdateList = new ArrayList<>();
+        StringBuilder dockerRmiCommand = new StringBuilder("docker rmi ");
+        for (int i = minAlgorithmImage; i < algorithmImportedList.size(); i++) {
+            AlgorithmPO tempAlgorithm = algorithmImportedList.get(i);
+            algorithmIdToUpdateList.add(tempAlgorithm.getId());
+            dockerRmiCommand.append(tempAlgorithm.getDockerImage());
+        }
+        //3 修改数据库为未导入
+        algorithmMapper.updateDockerImportAndDockerImageByIdList("0", "", algorithmIdToUpdateList);
+        //4 删除镜像
+        List<KubernetesNodeTO> nodeList = kubernetesConfiguration.getNodeList();
+        for (KubernetesNodeTO kubernetesNodeTO : nodeList) {
+            String hostname = kubernetesNodeTO.getHostname();
+            String username = kubernetesNodeTO.getUsername();
+            String password = kubernetesNodeTO.getPassword();
+            SshClient client = SshUtil.getClient();
+            ClientSession session = SshUtil.getSession(client, hostname, username, password);
+            SshUtil.execute(session, dockerRmiCommand.toString());
+            SshUtil.stop(client, session);
+        }
+    }
+}