mlx

simulation-resource-monitor/pom.xml

@@ -17,6 +17,16 @@
     </properties>
     <dependencies>
 
+        <!-- minio - 开始 -->
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.minio</groupId>
+            <artifactId>minio</artifactId>
+        </dependency>
+        <!-- minio - 结束 -->
         <!-- 数据库 - 开始 -->
         <dependency>
             <groupId>com.github.pagehelper</groupId>

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/configuration/minio/MinioConfiguration.java

@@ -0,0 +1,26 @@
+package com.css.simulation.resource.monitor.configuration.minio;
+
+import io.minio.MinioClient;
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "minio")
+public class MinioConfiguration {
+    private String endpoint;
+    private String endpointWithoutHttp;
+    private String accessKey;
+    private String secretKey;
+    private String bucketName;
+
+    @Bean
+    public MinioClient minioClient() {
+        return MinioClient.builder()
+                .endpoint(endpoint)
+                .credentials(accessKey, secretKey)
+                .build();
+    }
+}

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/mappper/ScenePackageSublistMapper.java

@@ -1,6 +1,8 @@
 package com.css.simulation.resource.monitor.mappper;
 
+import com.css.simulation.resource.monitor.pojo.GeneralPO;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
 
 import java.util.List;
@@ -15,4 +17,14 @@ public interface ScenePackageSublistMapper {
             "where scene_generalization_ids is not null\n" +
             "  and scene_generalization_ids != ''")
     List<String> selectSceneGeneralizationIds();
+
+    @Select("<script>" +
+            "select id, osgb_address, xodr_address, xosc_address\n" +
+            "from scene_general_data\n" +
+            "where id in \n" +
+            "    <foreach item='item' collection='sceneGeneralizationIdList' separator=',' open='(' close=')'>\n" +
+            "      #{item}\n" +
+            "    </foreach>" +
+            "</script>")
+    List<GeneralPO> selectGeneralDataNotIn(@Param("sceneGeneralizationIdList") List<String> sceneGeneralizationIdList);
 }

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/pojo/GeneralPO.java

@@ -0,0 +1,16 @@
+package com.css.simulation.resource.monitor.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class GeneralPO {
+    private String id;
+    private String osgbAddress;
+    private String xodrAddress;
+    private String xoscAddress;
+
+}

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/scheduler/MyScheduler.java

@@ -5,8 +5,12 @@ import api.common.pojo.po.home.SystemServerPO;
 import api.common.util.SshUtil;
 import api.common.util.StringUtil;
 import api.common.util.TimeUtil;
+import com.css.simulation.resource.monitor.configuration.minio.MinioConfiguration;
 import com.css.simulation.resource.monitor.manager.SystemServerManager;
 import com.css.simulation.resource.monitor.mappper.ScenePackageSublistMapper;
+import com.css.simulation.resource.monitor.pojo.GeneralPO;
+import com.css.simulation.resource.monitor.util.MinioUtil;
+import io.minio.MinioClient;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.sshd.client.SshClient;
@@ -36,6 +40,10 @@ public class MyScheduler {
     private SystemServerManager systemServerManager;
     @Resource
     private ScenePackageSublistMapper scenePackageSublistMapper;
+    @Resource
+    private MinioClient minioClient;
+    @Resource
+    private MinioConfiguration minioConfiguration;
 
 
     @Scheduled(fixedDelay = 60 * 1000)
@@ -108,7 +116,14 @@ public class MyScheduler {
                 sceneGeneralizationIdList.addAll(Arrays.asList(split));
             });
             //3 查询不在id列表中的所有泛化数据
-//            scenePackageSublistMapper.selectGeneralDataNotIn(sceneGeneralizationIdList);
+            List<GeneralPO> generalPOS = scenePackageSublistMapper.selectGeneralDataNotIn(sceneGeneralizationIdList);
+            generalPOS.forEach(
+                    generalPO -> {
+                        MinioUtil.rm(minioClient, minioConfiguration.getBucketName(), generalPO.getOsgbAddress());
+                        MinioUtil.rm(minioClient, minioConfiguration.getBucketName(), generalPO.getXodrAddress());
+                        MinioUtil.rm(minioClient, minioConfiguration.getBucketName(), generalPO.getXoscAddress());
+                    }
+            );
         } catch (Exception e) {
             e.printStackTrace();
             throw new RuntimeException(e);

simulation-resource-monitor/src/main/java/com/css/simulation/resource/monitor/util/MinioUtil.java

@@ -0,0 +1,212 @@
+package com.css.simulation.resource.monitor.util;
+
+import io.minio.*;
+import io.minio.errors.*;
+import io.minio.http.Method;
+import io.minio.messages.Item;
+import jdk.nashorn.internal.runtime.regexp.joni.exception.InternalException;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+@Slf4j
+public class MinioUtil {
+
+    /**
+     * 删除文件夹
+     *
+     * @param bucketName bucket名称
+     * @param prefix     文件夹全路径
+     * @since tarzan LIU
+     */
+    @SneakyThrows
+    public static void rmR(MinioClient minioClient, String bucketName, String prefix) {
+        Iterable<Result<Item>> list = minioClient.listObjects(ListObjectsArgs.builder().bucket(bucketName)
+                .prefix(prefix).recursive(true).build());
+        if (list == null || !list.iterator().hasNext()) {
+            log.info(prefix + " 不存在。");
+            return;
+        }
+        for (Result<Item> object : list) {
+            rm(minioClient, bucketName, object.get().objectName());
+        }
+        log.info("rmR() 删除 minio 目录 " + bucketName + "/" + prefix + "完成。");
+    }
+
+    /**
+     * 删除文件
+     *
+     * @param minioClient minio 客户端对象
+     * @param bucket      桶
+     * @param object      文件对象
+     */
+    @SneakyThrows
+    public static void rm(MinioClient minioClient, String bucket, String object) {
+        minioClient.removeObject(RemoveObjectArgs.builder()
+                .bucket(bucket)
+                .object(object)
+                .build());
+    }
+
+    /**
+     * 判断 bucket 是否存在
+     */
+    public static boolean isBucketExist(MinioClient minioClient, String bucketName) throws InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
+    }
+
+    /**
+     * 判断 object 是否存在
+     */
+    public static boolean isObjectExist(MinioClient minioClient, String bucket, String object) {
+        try {
+            StatObjectResponse objectStat = minioClient.statObject(StatObjectArgs.builder().bucket(bucket).object(object).build());
+            return objectStat != null;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    /**
+     * 创建 bucket
+     */
+    public static void createBucket(MinioClient minioClient, String bucketName) throws InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        if (!isBucketExist(minioClient, bucketName)) {
+            minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
+        }
+    }
+
+    /**
+     * 获取预览路径
+     *
+     * @return 预览路径
+     */
+    public static String getPreviewUrl(MinioClient minioClient, Method method, String bucket, String object) throws ServerException, InsufficientDataException, ErrorResponseException, IOException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, io.minio.errors.InternalException {
+        return minioClient.getPresignedObjectUrl(GetPresignedObjectUrlArgs.builder()
+                .method(method)
+                .bucket(bucket)
+                .object(object)
+                .build());
+    }
+
+    /**
+     * 通过文件路径上传文件上传文件
+     */
+    public static void uploadFromFile(
+            MinioClient minioClient,
+            String sourceFilePath,
+            String bucketName,
+            String objectName
+    ) throws IOException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        minioClient.uploadObject(UploadObjectArgs.builder()
+                .filename(sourceFilePath)
+                .bucket(bucketName)
+                .object(objectName)
+                .build());
+    }
+
+    /**
+     * 通过文件路径上传文件上传文件
+     *
+     * @param partSize 分片最小 5MB
+     */
+    public static void uploadFromStream(
+            MinioClient minioClient,
+            InputStream inputStream,
+            long objectSize,
+            long partSize,
+            String bucketName,
+            String objectName
+    ) throws IOException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        minioClient.putObject(PutObjectArgs.builder()
+                .stream(inputStream, objectSize, partSize)
+                .bucket(bucketName)
+                .object(objectName)
+                .build());
+    }
+
+    /**
+     * 通过文件路径上传文件上传文件
+     */
+    public static void uploadFromMultipartFile(
+            MinioClient minioClient,
+            MultipartFile multipartFile,
+            String bucketName,
+            String objectName
+    ) throws IOException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        InputStream inputStream = multipartFile.getInputStream();
+        long objectSize = multipartFile.getSize();
+//        long partSize = 5 * 1024 * 1024L; // 分片最小 5M
+        long partSize = -1; // 不分片
+        minioClient.putObject(PutObjectArgs.builder()
+                .stream(inputStream, objectSize, partSize)
+                .bucket(bucketName)
+                .object(objectName)
+                .build());
+    }
+
+
+    /**
+     * 下载文件
+     */
+    @SneakyThrows
+    public static void downloadToFile(MinioClient minioClient, String bucket, String object, String targetFilePath) {
+        File file = new File(targetFilePath);
+        if (!file.getParentFile().exists()) {
+            boolean mkdir = file.getParentFile().mkdirs();
+        }
+        boolean objectExist = isObjectExist(minioClient, bucket, object);
+        if (objectExist) {
+            minioClient.downloadObject(DownloadObjectArgs.builder()
+                    .bucket(bucket)
+                    .object(object)
+                    .filename(targetFilePath)
+                    .build());
+        } else {
+            throw new RuntimeException("downloadToFile() minio 文件" + object + "不存在。");
+        }
+    }
+
+    /**
+     * 下载文件
+     */
+    public static InputStream downloadToStream(
+            MinioClient minioClient,
+            String bucketName,
+            String objectName
+    ) throws IOException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, InternalException, ServerException, io.minio.errors.InternalException {
+        return minioClient.getObject(GetObjectArgs.builder()
+                .bucket(bucketName)
+                .object(objectName)
+                .build());
+    }
+
+    /**
+     * 读取文件内容
+     */
+    public static String downloadToString(
+            MinioClient minioClient,
+            String bucketName,
+            String objectName
+    ) throws IOException, ServerException, InsufficientDataException, ErrorResponseException, NoSuchAlgorithmException, InvalidKeyException, InvalidResponseException, XmlParserException, io.minio.errors.InternalException {
+        GetObjectResponse response = minioClient.getObject(GetObjectArgs.builder()
+                .bucket(bucketName)
+                .object(objectName)
+                .build());
+        StringBuilder result = new StringBuilder();
+        byte[] buf = new byte[4096];//创建字节数组，存储临时读取的数据
+        int len;//记录数据读取的长度
+        //循环读取数据
+        while ((len = response.read(buf)) != -1) { //长度为-1则读取完毕
+            result.append(new String(buf, 0, len)).append("\n");
+        }
+        response.close();
+        return result.toString();
+    }
+}

simulation-resource-scheduler/src/main/java/com/css/simulation/resource/scheduler/service/ProjectService.java

@@ -22,7 +22,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.kafka.clients.admin.Admin;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.redis.core.StringRedisTemplate;
-import org.springframework.kafka.core.KafkaTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -75,8 +74,6 @@ public class ProjectService {
     @Resource(name = "myKafkaAdmin")
     Admin kafkaAdminClient;
     @Resource
-    KafkaTemplate<String, String> kafkaTemplate;
-    @Resource
     CloseableHttpClient closeableHttpClient;
     @Resource
     RequestConfig requestConfig;
@@ -87,8 +84,6 @@ public class ProjectService {
     @Resource
     IndexTemplateMapper indexTemplateMapper;
     @Resource
-    IndexMapper indexMapper;
-    @Resource
     SceneMapper sceneMapper;
     @Resource
     AlgorithmMapper algorithmMapper;