mlx

simulation-resource-common/src/main/java/com/css/simulation/resource/common/controller/AlgPlatformCtrl.java

@@ -7,6 +7,7 @@ import api.common.util.JsonUtil;
 import api.common.util.ObjectUtil;
 import com.css.simulation.resource.common.configuration.algPlatform.ClientDetail;
 import com.fasterxml.jackson.databind.JsonNode;
+import lombok.SneakyThrows;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -29,7 +30,8 @@ public class AlgPlatformCtrl {
 
     @RequestMapping("/getAlgorithmList")
     @ResponseBody
-    public ResponseBodyVO getAlgorithmList(@RequestBody String param) throws Exception {
+    @SneakyThrows
+    public ResponseBodyVO getAlgorithmList(@RequestBody String param){
         //取得token
         String token = getToken();
         if(ObjectUtil.isNull(token)){

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/algorithm/AlgorithmPlatform.java

@@ -0,0 +1,63 @@
+package com.css.simulation.resource.scheduler.configuration.algorithm;
+
+import api.common.util.HttpUtil;
+import api.common.util.JsonUtil;
+import api.common.util.ObjectUtil;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.SneakyThrows;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AlgorithmPlatform {
+
+    private static final String TOKEN_KEY = "ALGPLATFORM:TOKEN";
+
+    @Autowired
+    OauthParameter oauthParameter;
+
+    @Autowired
+    StringRedisTemplate redisTemplate;
+
+    @Autowired
+    CloseableHttpClient closeableHttpClient;
+    @Autowired
+    RequestConfig requestConfig;
+
+//    @SneakyThrows
+//    public InputStream getAlgorithmFile(String algorithmId) {
+//        //取得token
+//        String token = getToken();
+//        String algorithmAddressUrl = oauthParameter.getAlgorithmListUri()
+//                + "?access_token=" + token
+//                + "&id=" + algorithmId;
+//        return HttpUtil.get(closeableHttpClient, requestConfig, algorithmAddressUrl);
+//    }
+
+
+    @SneakyThrows
+    private String getToken() {
+
+        String token = redisTemplate.opsForValue().get(TOKEN_KEY);
+        if (ObjectUtil.isNull(token)) {//token失效重新获取
+            String tokenUri = oauthParameter.getTokenUri();
+            String appid = oauthParameter.getAppid();
+            String secret = oauthParameter.getSecret();
+            String URI = tokenUri + "&appid=" + appid + "&secret=" + secret;
+            String result = HttpUtil.get(HttpUtil.getHttpClient(), HttpUtil.getRequestConfig(), URI);
+            JsonNode jsonNode = JsonUtil.readTree(result);
+            JsonNode dataNode = jsonNode.path("data");
+            if (ObjectUtil.isNotNull(dataNode)) {
+                token = dataNode.path("access_token").asText();
+                if (ObjectUtil.isNotNull(token)) {
+                    int expiresSec = dataNode.path("expires_in_sec").asInt();
+                    redisTemplate.opsForValue().set(TOKEN_KEY, token, expiresSec / 60);
+                }
+            }
+        }
+        return token;
+    }
+}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/algorithm/OauthParameter.java

@@ -0,0 +1,18 @@
+package com.css.simulation.resource.scheduler.configuration.algorithm;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "algorithm-platform")
+public class OauthParameter {
+
+    private String appid;//客户端id
+    private String secret;//秘钥
+    private String tokenUri;//获取token接口
+    private String algorithmListUri;//获取列表查询接口
+    private String algorithmAddrUri;//获取算法地址接口
+
+}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/configuration/http/HttpConfiguration.java

@@ -0,0 +1,62 @@
+package com.css.simulation.resource.scheduler.configuration.http;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+
+@Configuration
+public class HttpConfiguration {
+    @Bean
+    public CloseableHttpClient closeableHttpClient() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+
+        SSLContext sslContext = SSLContext.getInstance("TLS");
+        X509TrustManager x509TrustManager = new X509TrustManager() {
+            @Override
+            public X509Certificate[] getAcceptedIssuers() {
+                return null;
+            }
+
+            @Override
+            public void checkClientTrusted(X509Certificate[] arg0, String arg1) {
+            }
+
+            @Override
+            public void checkServerTrusted(X509Certificate[] arg0, String arg1) {
+            }
+        };
+        sslContext.init(null, new TrustManager[]{x509TrustManager}, null);
+        SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
+        PoolingHttpClientConnectionManager pool = new PoolingHttpClientConnectionManager();
+        pool.setMaxTotal(300);
+        pool.setDefaultMaxPerRoute(300);
+        return HttpClients.custom()
+                .setConnectionManager(pool)
+                .setSSLSocketFactory(sslConnectionSocketFactory)
+                .build();
+    }
+
+
+    @Bean
+    public RequestConfig requestConfig() {
+        return RequestConfig.custom()
+                .setSocketTimeout(5000)
+                .setConnectTimeout(5000)
+                .setConnectionRequestTimeout(5000)
+                .setRedirectsEnabled(false)
+                .setExpectContinueEnabled(false)
+                .build();
+    }
+}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/consumer/ManualProjectConsumer.java

@@ -3,13 +3,11 @@ package com.css.simulation.resource.scheduler.consumer;
 
 import api.common.pojo.constants.DictConstants;
 import api.common.pojo.dto.ProjectMessageDTO;
-import api.common.util.CollectionUtil;
-import api.common.util.JsonUtil;
-import api.common.util.StringUtil;
-import api.common.util.TimeUtil;
+import api.common.util.*;
 import com.css.simulation.resource.scheduler.mapper.*;
 import com.css.simulation.resource.scheduler.pojo.po.*;
 import com.css.simulation.resource.scheduler.pojo.to.*;
+import com.css.simulation.resource.scheduler.util.MinioUtil;
 import io.kubernetes.client.openapi.ApiClient;
 import io.kubernetes.client.openapi.apis.BatchV1Api;
 import io.kubernetes.client.openapi.models.V1Job;
@@ -228,24 +226,27 @@ public class ManualProjectConsumer {
         }
 
         // -------------------------------- 4 算法导入（一期按单机版做） --------------------------------
-//        // 私有仓库导入算法镜像
-//        String algorithmId = projectMessageDTO.getAlgorithmId();    // 算法 id
-//        //4-1 根据算法 id 获取算法文件地址、是否已导入成镜像。
-//        AlgorithmPO algorithmPO = algorithmMapper.selectById(algorithmId);
-//        String minioPath = algorithmPO.getMinioPath();
-//        String dockerImage;
-//        if ("0".equals(algorithmPO.getDockerImport())) {
-//            dockerImage = "algorithm_" + algorithmId + ":latest";
-//            String algorithmTarLinuxTempPath = linuxTempPath + minioPath;
-//            // 下载算法文件到本地（ 2 到仓库服务器）
-//            MinioUtil.downloadToFile(minioClient, bucketName, minioPath, algorithmTarLinuxTempPath);
-//            //4-2 本地执行 docker load 算法文件成镜像（集群版可改成用 docker-java 操作仓库）
-//            LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
-//        } else if ("1".equals(algorithmPO.getDockerImport()) && StringUtil.isNotEmpty(algorithmPO.getDockerImage())) {
-//            dockerImage = algorithmPO.getDockerImage();
-//        } else {
-//            throw new RuntimeException("算法 " + algorithmId + "的 mysql 数据有误！");
-//        }
+        // 私有仓库导入算法镜像
+        String algorithmId = projectMessageDTO.getAlgorithmId();    // 算法 id
+        //4-1 根据算法 id 获取算法文件地址、是否已导入成镜像。
+        AlgorithmPO algorithmPO = algorithmMapper.selectById(algorithmId);
+        if (algorithmPO == null){
+            // 访问索为远程接口
+        }
+        String minioPath = algorithmPO.getMinioPath();
+        String dockerImage;
+        if ("0".equals(algorithmPO.getDockerImport())) {
+            dockerImage = "algorithm_" + algorithmId + ":latest";
+            String algorithmTarLinuxTempPath = linuxTempPath + minioPath;
+            // 下载算法文件到本地（ 2 到仓库服务器）
+            MinioUtil.downloadToFile(minioClient, bucketName, minioPath, algorithmTarLinuxTempPath);
+            //4-2 本地执行 docker load 算法文件成镜像（集群版可改成用 docker-java 操作仓库）
+            LinuxUtil.execute("docker import " + algorithmTarLinuxTempPath + " " + dockerImage);
+        } else if ("1".equals(algorithmPO.getDockerImport()) && StringUtil.isNotEmpty(algorithmPO.getDockerImage())) {
+            dockerImage = algorithmPO.getDockerImage();
+        } else {
+            throw new RuntimeException("算法 " + algorithmId + "的 mysql 数据有误！");
+        }
         // -------------------------------- 5 创建 pod 开始执行 --------------------------------
         int completions = sceneList.size();     // 结束标
         int parallelism = projectMessageDTO.getParallelism();    // 并行度

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/controller/MinioController.java

@@ -1,17 +1,24 @@
 package com.css.simulation.resource.scheduler.controller;
 
+import api.common.pojo.param.MinioParameter;
+import api.common.util.FileUtil;
 import com.css.simulation.resource.scheduler.util.MinioUtil;
 import io.minio.MinioClient;
+import io.minio.errors.*;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestPart;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
 @RequestMapping("/minio")
 @RestController
 @Slf4j
@@ -20,10 +27,26 @@ public class MinioController {
     @Autowired
     MinioClient minioClient;
 
+    private static final String BUCKET_NAME = "test";
+
 
     @PostMapping(value = "/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
     @SneakyThrows
     public void upload(@RequestPart("file") MultipartFile file, String path) {
-        MinioUtil.uploadFromMultipartFile(minioClient, file, "test", path);
+        MinioUtil.uploadFromMultipartFile(minioClient, file, BUCKET_NAME, path);
+    }
+
+    @PostMapping("/download")
+    public void download(
+            @RequestBody @Validated MinioParameter minioParameter,
+            HttpServletResponse response
+    ) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException {
+        InputStream inputStream = MinioUtil.downloadToStream(
+                minioClient,
+                BUCKET_NAME,
+                minioParameter.getObjectName()
+        );
+        String fileName = FileUtil.getFileName(minioParameter.getObjectName());
+        FileUtil.downloadForHttp(fileName, inputStream, response, 1024);
     }
 }