|
@@ -99,34 +99,38 @@ public class TaskManager {
|
|
|
} else { // 出错和结束的 pod 都直接删除
|
|
|
// -------------------------------- 处理状态 --------------------------------
|
|
|
//TODO 暂时不用重试操作
|
|
|
- KubernetesUtil.deletePod(apiClient, kubernetesNamespace, podName);
|
|
|
- log.info("TaskManager--state 修改任务 " + taskId + "的状态为 " + state + ",pod 名称为:" + podName + ",并删除 pod。");
|
|
|
- if ("Aborted".equals(state)) {
|
|
|
- String minioPathOfErrorLog = resultPathMinio + projectId + "/" + taskId + "error.log";
|
|
|
- boolean objectExist = MinioUtil.isObjectExist(minioClient, bucketName, minioPathOfErrorLog);
|
|
|
- String targetEvaluate;
|
|
|
- if (objectExist) {
|
|
|
- String errorString = MinioUtil.downloadToString(minioClient, bucketName, minioPathOfErrorLog);
|
|
|
- String[] lines = errorString.split("\n");
|
|
|
- StringBuilder errorMessage = new StringBuilder();
|
|
|
- for (String line : lines) {
|
|
|
- if (line.startsWith("Original Error")) {
|
|
|
- errorMessage.append(line).append("\n");
|
|
|
- }
|
|
|
- if (line.startsWith("Possible Cause")) {
|
|
|
- errorMessage.append(line);
|
|
|
- break;
|
|
|
+ try {
|
|
|
+ KubernetesUtil.deletePod(apiClient, kubernetesNamespace, podName);
|
|
|
+ log.info("TaskManager--state 修改任务 " + taskId + "的状态为 " + state + ",pod 名称为:" + podName + ",并删除 pod。");
|
|
|
+ if ("Aborted".equals(state)) {
|
|
|
+ String minioPathOfErrorLog = resultPathMinio + projectId + "/" + taskId + "error.log";
|
|
|
+ boolean objectExist = MinioUtil.isObjectExist(minioClient, bucketName, minioPathOfErrorLog);
|
|
|
+ String targetEvaluate;
|
|
|
+ if (objectExist) {
|
|
|
+ String errorString = MinioUtil.downloadToString(minioClient, bucketName, minioPathOfErrorLog);
|
|
|
+ String[] lines = errorString.split("\n");
|
|
|
+ StringBuilder errorMessage = new StringBuilder();
|
|
|
+ for (String line : lines) {
|
|
|
+ if (line.startsWith("Original Error")) {
|
|
|
+ errorMessage.append(line).append("\n");
|
|
|
+ }
|
|
|
+ if (line.startsWith("Possible Cause")) {
|
|
|
+ errorMessage.append(line);
|
|
|
+ break;
|
|
|
+ }
|
|
|
}
|
|
|
+ targetEvaluate = errorMessage.toString();
|
|
|
+ } else {
|
|
|
+ targetEvaluate = DictConstants.TASK_ERROR_REASON_2;
|
|
|
}
|
|
|
- targetEvaluate = errorMessage.toString();
|
|
|
- } else {
|
|
|
- targetEvaluate = DictConstants.TASK_ERROR_REASON_2;
|
|
|
+ taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), targetEvaluate);
|
|
|
+ } else if ("Terminated".equals(state)) {
|
|
|
+ taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_3);
|
|
|
+ } else if ("PendingAnalysis".equals(state)) {
|
|
|
+ taskMapper.updateSuccessStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
|
|
|
}
|
|
|
- taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), targetEvaluate);
|
|
|
- } else if ("Terminated".equals(state)) {
|
|
|
- taskMapper.updateFailStateWithStopTime(taskId, state, TimeUtil.getNowForMysql(), DictConstants.TASK_ERROR_REASON_3);
|
|
|
- } else if ("PendingAnalysis".equals(state)) {
|
|
|
- taskMapper.updateSuccessStateWithStopTime(taskId, state, TimeUtil.getNowForMysql());
|
|
|
+ } catch (io.kubernetes.client.openapi.ApiException apiException) {
|
|
|
+ log.error("TaskManager--isCompleted pod " + podName + " 已经被手动删除,该项目可能已经失败或删除。");
|
|
|
}
|
|
|
// -------------------------------- 判断项目是否结束 --------------------------------
|
|
|
ProjectMessageDTO projectMessageDTO = JsonUtil.jsonToBean(stringRedisTemplate.opsForValue().get(redisPrefix.getProjectRunningKey()), ProjectMessageDTO.class);
|