commit

src/python2/pjisuv/csv-pjisuv.py

@@ -9,7 +9,7 @@ import logging
 
 path1 = '/mnt/disk001/dcl_data_process/src/python2/pjisuv/'
 
-logging.basicConfig(filename=path1 + 'log/csv.log', level=logging.INFO,
+logging.basicConfig(filename=path1 + 'log/csv-pjisuv.log', level=logging.INFO,
                     format='%(asctime)s - %(levelname)s - %(message)s')
 
 key1 = 'pjisuv/'
@@ -42,25 +42,15 @@ def parse_to_csv(merged_bag_file_path, parse_prefix, local_parse_dir, local_dele
         logging.exception("生成csv报错: %s" % str(e))
 
 
-'''
-cname：http://open-bucket.oss.icvdc.com
-内网endpoint: oss-cn-beijing-gqzl-d01-a.ops.gqzl-cloud.com
-oss桶名： open-bucket
-keyid：n8glvFGS25MrLY7j
-secret：xZ2Fozoarpfw0z28FUhtg8cu0yDc5d
-'''
 # ------- 获取合并之后的bag包，解析出csv -------
 if __name__ == '__main__':
     # 1 创建阿里云对象
     auth = oss2.Auth('n8glvFGS25MrLY7j', 'xZ2Fozoarpfw0z28FUhtg8cu0yDc5d')
-    # cname = 'http://open-bucket.oss.icvdc.com'
-    # bucket = oss2.Bucket(auth, cname, 'open-bucket', is_cname=True)
     endpoint = 'oss-cn-beijing-gqzl-d01-a.ops.gqzl-cloud.com'
     bucket = oss2.Bucket(auth, endpoint, 'open-bucket')
     while True:
         try:
             logging.info("开始新一轮扫描")
-            prefix_list = []
             # 2 获取已经上传完成的所有目录并分组
             for obj1 in oss2.ObjectIterator(bucket, prefix=key1):
                 local_delete_list = []
@@ -68,13 +58,12 @@ if __name__ == '__main__':
                 try:
                     # 获取合并后的包
                     merged_bag_object_key = str(obj1.key)
-                    # print(f'判断1{merged_bag_object_key}')
                     if 'data_merge' in str(obj1.key) and str(obj1.key).endswith('.bag'):
-                        # print(f'需要解析{merged_bag_object_key}')
                         merged_bag_object_key_split = merged_bag_object_key.split('/')
                         merged_prefix = '/'.join(merged_bag_object_key_split[:-1])
                         parse_prefix = merged_prefix.replace('data_merge', 'data_parse')
-                        parse_prefix_full = merged_bag_object_key.replace('data_merge', 'data_parse')[:-4] + '/'
+                        parse_prefix_full = (
+                            merged_bag_object_key.replace('data_merge', 'data_parse').replace('.bag', '/'))
                         csv1_done = False
                         # csv2_done = False
                         csv3_done = False