mlx

api-common/src/main/java/api/common/pojo/constants/DictConstants.java

@@ -33,7 +33,11 @@ public class DictConstants {
 
 
 
-    public static final String ALGORITHM_FILE = "algorithmFile";//算法文件上传type
+    public static final String ALGORITHM_FILE = "algorithmFile";    //算法文件上传type
+
+    // 算法镜像上传类型
+    public static final String ALGORITHM_UPLOAD_MODE_FILE = "1";    // 文件上传
+    public static final String ALGORITHM_UPLOAD_MODE_GIT = "2";     // git 仓库
 
 
 

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ProjectConsumer.java

@@ -216,7 +216,6 @@ public class ProjectConsumer {
         projectService.transferAndRunYaml(
                 jobTemplate + "job-template.yaml",
                 projectId,
-                projectType,
                 algorithmDockerImage,
                 scenePOList.size(),
                 parallelism,
@@ -242,8 +241,7 @@ public class ProjectConsumer {
         JsonNode jsonNode = objectMapper.readTree(json);
         String projectId = jsonNode.path("projectId").asText();
         String type = jsonNode.path("type").asText();
-        projectService.stopProject(projectId,type);
-
+        projectService.stopProject(projectId, type);
 
 
     }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/mapper/AlgorithmMapper.java

@@ -17,12 +17,19 @@ public interface AlgorithmMapper {
     @Results(id = "index", value = {
             @Result(column = "minio_path", property = "minioPath", jdbcType = JdbcType.VARCHAR),
             @Result(column = "docker_import", property = "dockerImport", jdbcType = JdbcType.VARCHAR),
-            @Result(column = "docker_image", property = "dockerImage", jdbcType = JdbcType.VARCHAR)
+            @Result(column = "docker_image", property = "dockerImage", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "upload_mode", property = "uploadMode", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "git_url", property = "gitUrl", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "git_user_name", property = "gitUserName", jdbcType = JdbcType.VARCHAR),
+            @Result(column = "git_password", property = "gitPassword", jdbcType = JdbcType.VARCHAR)
     })
     @Select("select minio_path,\n" +
-            "   docker_import,\n" +
-            "   docker_image,\n" +
-            "   upload_mode\n" +
+            "       docker_import,\n" +
+            "       docker_image,\n" +
+            "       upload_mode,\n" +
+            "       git_url,\n" +
+            "       git_user_name,\n" +
+            "       git_password\n" +
             "from algorithm\n" +
             "where is_deleted = '0'\n" +
             "  and id = #{id}")

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/pojo/po/AlgorithmPO.java

@@ -11,4 +11,8 @@ public class AlgorithmPO {
     private String minioPath;
     private String dockerImport;
     private String dockerImage;
+    private String uploadMode;
+    private String gitUrl;
+    private String gitUserName;
+    private String gitPassword;
 }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ProjectService.java

@@ -5,6 +5,7 @@ import api.common.util.*;
 import com.css.simulation.resource.scheduler.mapper.*;
 import com.css.simulation.resource.scheduler.pojo.po.*;
 import com.css.simulation.resource.scheduler.pojo.to.*;
+import com.css.simulation.resource.scheduler.util.GitUtil;
 import com.css.simulation.resource.scheduler.util.KubernetesUtil;
 import com.css.simulation.resource.scheduler.util.MinioUtil;
 import com.css.simulation.resource.scheduler.util.ProjectUtil;
@@ -311,20 +312,28 @@ public class ProjectService {
             FileUtil.writeInputStreamToLocalFile(inputStream, algorithmTarLinuxTempPath);
             LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
         } else {
-            String minioPath = algorithmPO.getMinioPath();
-            if ("0".equals(algorithmPO.getDockerImport()) || algorithmPO.getDockerImport() == null) {
-                // 下载算法文件到本地（ 2 到仓库服务器）
-                MinioUtil.downloadToFile(minioClient, bucketName, minioPath, algorithmTarLinuxTempPath);
-                //4-2 本地执行 docker load 算法文件成镜像（集群版可改成用 docker-java 操作仓库）
+            if ("1".equals(algorithmPO.getDockerImport()) && StringUtil.isNotEmpty(algorithmPO.getDockerImage())) {
+                dockerImage = algorithmPO.getDockerImage();
+            } else if ("0".equals(algorithmPO.getDockerImport()) || algorithmPO.getDockerImport() == null) {
+                String uploadMode = algorithmPO.getUploadMode();
+                if (DictConstants.ALGORITHM_UPLOAD_MODE_FILE.equals(uploadMode)) {
+                    String minioPath = algorithmPO.getMinioPath();
+                    MinioUtil.downloadToFile(minioClient, bucketName, minioPath, algorithmTarLinuxTempPath);    // 下载算法文件到本地
+                } else if (DictConstants.ALGORITHM_UPLOAD_MODE_GIT.equals(uploadMode)) {
+                    String gitUrl = algorithmPO.getGitUrl();
+                    String gitUserName = algorithmPO.getGitUserName();
+                    String gitPassword = algorithmPO.getGitPassword();
+                    GitUtil.clone(gitUrl, gitUserName, gitPassword, algorithmTarLinuxTempPath); // 下载算法文件到本地
+                } else {
+                    throw new RuntimeException("算法 " + algorithmId + " 的 upload_mode 字段数据有误！");
+                }
+                // 本地执行 docker load 算法文件成镜像（可改成用 docker-java 操作仓库）
                 LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
                 algorithmMapper.updateDockerImportAndDockerImageById("1", dockerImage, algorithmId);
-            } else if ("1".equals(algorithmPO.getDockerImport()) && StringUtil.isNotEmpty(algorithmPO.getDockerImage())) {
-                dockerImage = algorithmPO.getDockerImage();
             } else {
                 throw new RuntimeException("算法 " + algorithmId + " 的 mysql 数据有误！");
             }
         }
-
         log.info("ProjectService--handleAlgorithm 项目 " + projectId + " 使用的算法镜像为：" + dockerImage);
         return dockerImage;
     }
@@ -340,7 +349,7 @@ public class ProjectService {
      * @param jobTemplateYamlPathTarget 执行文件
      */
     @SneakyThrows
-    public void transferAndRunYaml(String jobTemplateYamlPathSource, String projectId, String projectType, String algorithmDockerImage, long completions, long parallelism, String jobTemplateYamlPathTarget) {
+    public void transferAndRunYaml(String jobTemplateYamlPathSource, String projectId, String algorithmDockerImage, long completions, long parallelism, String jobTemplateYamlPathTarget) {
         log.info("ProjectConsumer--transferYaml 项目 " + projectId + " 的完成度为：" + completions);
         log.info("ProjectConsumer--transferYaml 项目 " + projectId + " 的并行度为：" + parallelism);
         String yamlSource = FileUtil.read(jobTemplateYamlPathSource);
@@ -353,16 +362,9 @@ public class ProjectService {
         String replace5 = replace4.replace("completions-number", completions + "");
         String replace6 = replace5.replace("parallelism-number", parallelism + "");
         String replace7 = replace6.replace("apiVers1on", "apiVersion");
-        String replace8 = replace7.replace("1atch/v1", "batch/v1");
-        String finalYaml;
-        if(DictConstants.PROJECT_TYPE_MANUAL.equals(projectType)){
-            finalYaml = replace8;
-        }else if (DictConstants.PROJECT_TYPE_AUTO_SUB.equals(projectType)){
-            //1 获取 git 仓库地址
-            
-        }
-        log.info("ProjectConsumer--parseManualProject 开始执行 yaml 文件" + replace8);
-        FileUtil.writeStringToLocalFile(replace8, jobTemplateYamlPathTarget);
+        String finalYaml = replace7.replace("1atch/v1", "batch/v1");
+        log.info("ProjectConsumer--parseManualProject 开始执行 yaml 文件" + finalYaml);
+        FileUtil.writeStringToLocalFile(finalYaml, jobTemplateYamlPathTarget);
         //  启动
         KubernetesUtil.applyYaml(hostname, username, password, jobTemplateYamlPathTarget);
     }

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/util/GitUtil.java

@@ -0,0 +1,32 @@
+package com.css.simulation.resource.scheduler.util;
+
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+
+import java.io.File;
+import java.util.Objects;
+
+/**
+ * <!-- git 操作类库 -->
+ * <dependency>
+ * <groupId>org.eclipse.jgit</groupId>
+ * <artifactId>org.eclipse.jgit</artifactId>
+ * <version>${org.eclipse.jgit.version}</version>
+ * </dependency>
+ */
+public class GitUtil {
+
+    public static boolean clone(String url, String username, String password, String localDirectory) throws GitAPIException {
+        File localDirectoryObject = new File(localDirectory);
+        if (Objects.requireNonNull(localDirectoryObject.listFiles()).length > 0) {
+            return false;
+        }
+        Git git = Git.cloneRepository()
+                .setURI(url)
+                .setCredentialsProvider(new UsernamePasswordCredentialsProvider(username, password))
+                .setDirectory(localDirectoryObject)
+                .call();
+        return true;
+    }
+}