Merge remote-tracking branch 'origin/master'

src/python2/pjibot_delivery/callback.py

@@ -7,12 +7,12 @@ from datetime import datetime, timedelta
 
 import logging
 
-path1 = '/mnt/disk001/cicv-data-closedloop/python2-pjibot-double-camera-module-pji/'
+path1 = '/mnt/disk001/dcl_data_process/src/python2/pjibot_delivery/'
 
-logging.basicConfig(filename=path1 + 'log/pjibot_callback.log', level=logging.INFO,
+logging.basicConfig(filename=path1 + 'log/callback.log', level=logging.INFO,
                     format='%(asctime)s - %(levelname)s - %(message)s')
 
-key1 = 'pjibot/'
+key1 = 'pjibot_delivery/'
 key2 = 'data/'
 key3 = 'data_merge/'
 key4 = 'data_parse/'
@@ -37,21 +37,12 @@ keyid：n8glvFGS25MrLY7j
 secret：xZ2Fozoarpfw0z28FUhtg8cu0yDc5d
 '''
 if __name__ == '__main__':
-    # 1 登录验证 。
     auth = oss2.Auth('n8glvFGS25MrLY7j', 'xZ2Fozoarpfw0z28FUhtg8cu0yDc5d')
-
-    # 2 填写自定义域名，例如example.com。获取桶。
-    # 3 填写Bucket名称，并设置is_cname=True来开启CNAME。CNAME是指将自定义域名绑定到存储空间。
-    # cname = 'http://open-bucket.oss.icvdc.com'
-    # bucket = oss2.Bucket(auth, cname, 'open-bucket', is_cname=True)
     endpoint = 'oss-cn-beijing-gqzl-d01-a.ops.gqzl-cloud.com'
     bucket_name = 'pji-bucket1'
     bucket = oss2.Bucket(auth, endpoint, bucket_name)
-
-    turn = 1
     while True:
-        logging.info("当前轮次: %s" % str(turn))
-        turn += 1
+        logging.info("开始新一轮扫描")
         try:
             local_delete_list = []
             oss_delete_list = []
@@ -62,30 +53,35 @@ if __name__ == '__main__':
                     if 'callback.json' in str(obj1.key):
                         prefix = '/'.join(str(obj1.key).split('/')[:-1])
 
-                        done1 = False
-                        done2 = False
-                        done3 = False
-                        done4 = False
-                        done5 = False
-                        done6 = False
-                        done7 = False
-                        # print '检测是否文件完整：', prefix
+                        file1 = False
+                        file2 = False
+                        file3 = False
+                        file4 = False
+                        file5 = False
+                        file6 = False
+                        file7 = False
+                        file8 = False
+                        file9 = False
                         for obj2 in oss2.ObjectIterator(bucket, prefix=prefix):
+                            if '/callback.json' in str(obj2.key):
+                                file1 = True
                             if '/ego_pji.csv' in str(obj2.key):
-                                done1 = True
+                                file2 = True
                             if '/objects_pji.csv' in str(obj2.key):
-                                done2 = True
+                                file3 = True
                             if '/pcd_overlook.mp4' in str(obj2.key):
-                                done3 = True
+                                file4 = True
                             if '/pos_pji.csv' in str(obj2.key):
-                                done4 = True
+                                file5 = True
+                            if '/report.pdf' in str(obj2.key):
+                                file6 = True
+                            if '/scenario_orig.mp4' in str(obj2.key):
+                                file7 = True
                             if '/simulation.xosc' in str(obj2.key):
-                                done5 = True
+                                file8 = True
                             if '/trajectory_pji.csv' in str(obj2.key):
-                                done6 = True
-                            if '/report.pdf' in str(obj2.key):
-                                done7 = True
-                        if not done1 or not done2 or not done3 or not done4 or not done5 or not done6 or not done7:
+                                file9 = True
+                        if not file1 or not file2 or not file3 or not file4 or not file5 or not file6 or not file7 or not file8 or not file9:
                             continue
                         time.sleep(1)
                         logging.info("发送： %s", str(obj1.key))
@@ -121,7 +117,7 @@ if __name__ == '__main__':
                             logging.exception("callback报错：%s", str(e))
                             continue
 
-                        # 将时区统一（室内机器人需要加8，室外不需要了）
+                        # 将时区统一（室外不需要需要加8，根据机器人终端的时区判断）
                         # new_date = add_hour(old_date, 8)
                         new_date = old_date
                         old_delete_list = []
@@ -131,12 +127,13 @@ if __name__ == '__main__':
                                 bucket.copy_object(bucket_name, str(obj_old.key),
                                                    str(obj_old.key).replace(old_date, new_date).replace('callback.json',
                                                                                                         'callback_done.json'))
-                            else:
-                                bucket.copy_object(bucket_name, str(obj_old.key),
-                                                   str(obj_old.key).replace(old_date, new_date))
-                        bucket.copy_object(bucket_name, old_ros_bag_path, old_ros_bag_path.replace(old_date, new_date))
-                        bucket.delete_object(old_ros_bag_path)
-                        bucket.batch_delete_objects(old_delete_list)
+                        # todo 时区不变也就不需要移动文件了
+                        #     else:
+                        #         bucket.copy_object(bucket_name, str(obj_old.key),
+                        #                            str(obj_old.key).replace(old_date, new_date))
+                        # bucket.copy_object(bucket_name, old_ros_bag_path, old_ros_bag_path.replace(old_date, new_date))
+                        # bucket.delete_object(old_ros_bag_path)
+                        # bucket.batch_delete_objects(old_delete_list)
 
                         data2 = {
                             "dataName": new_date,

src/python2/pjibot_delivery/simulation-nohup.sh

@@ -1,2 +1,8 @@
 #!/bin/bash
+if [ ! -d "./log" ]; then
+    mkdir "./log"
+    echo "Directory './log' created."
+else
+    echo "Directory './log' already exists."
+fi
 nohup python2 simulation.py > log/simulation.out 2>&1 &

src/python2/pjibot_delivery/simulation.py

@@ -11,6 +11,7 @@ key1 = 'pjibot_delivery/'
 path1 = '/scenarios3/'
 path2 = '/mnt/disk001/dcl_data_process/src/python2/pjibot_delivery/'
 path3 = '/mnt/disk001/dcl_data_process/src/python2/pjibot_delivery/resource/'
+vehicle_name = 'PuJin_distribution'  # 配送 PuJin_distribution 巡检 PuJin_patrol_robot
 
 logging.basicConfig(filename=path2 + 'log/simulation.log', level=logging.INFO,
                     format='%(asctime)s - %(levelname)s - %(message)s')
@@ -62,7 +63,7 @@ def move_xosc_before_simulation(root_path):
                     if node1.get("name") == 'Ego':
                         for node2 in node1:
                             if node2.tag == 'Vehicle':
-                                node2.set('name', 'Pji_Utility_Vehicle')
+                                node2.set('name', vehicle_name)
         tree1.write(root_path + 'openx0.xosc')
 
     except Exception as e:
@@ -73,7 +74,7 @@ def upload_simulation(parse_prefix, mp41):
     try:
         bucket.put_object_from_file(parse_prefix + 'scenario_orig.mp4', mp41)
         logging.info('上传仿真视频到 %s' % parse_prefix + 'scenario_orig.mp4')
-        shutil.rmtree(path1) # 仿真完就删除
+        shutil.rmtree(path1)  # 仿真完就删除
     except Exception as e:
         logging.exception("上传视频报错 %s" % str(e))
 
@@ -124,49 +125,56 @@ if __name__ == '__main__':
     # 1 创建阿里云对象
     auth = oss2.Auth('n8glvFGS25MrLY7j', 'xZ2Fozoarpfw0z28FUhtg8cu0yDc5d')
     endpoint = 'oss-cn-beijing-gqzl-d01-a.ops.gqzl-cloud.com'
-    bucket = oss2.Bucket(auth, endpoint, 'open-bucket')
+    bucket = oss2.Bucket(auth, endpoint, 'pji-bucket1')
     while True:
-        logging.info("开始新一轮扫描")
-        local_delete_list = []
-        oss_delete_list = []
-        prefix_list = []
-        # 2 获取已经上传完成的所有目录并分组
-        all_prefix_set = set()
-        for obj1 in oss2.ObjectIterator(bucket, prefix=key1):
-            # 获取csv
-            if 'data_parse' in str(obj1.key) and str(obj1.key).count('/') == 4:
-                all_prefix_set.add('/'.join(str(obj1.key).split('/')[:-1]) + '/')
-        for obj2_key in all_prefix_set:
-            xosc_done = False
-            mp4_done = False
-            for obj3 in oss2.ObjectIterator(bucket, prefix=obj2_key):
-                if '/scenario.xosc' in str(obj3.key):
-                    xosc_done = True
-                if '/scenario_orig.mp4' in str(obj3.key):
-                    mp4_done = True
-            if not xosc_done or mp4_done:
-                continue
-
-            logging.info("需要生成仿真视频: %s" % obj2_key)
-            parse_prefix_full = obj2_key
-            local_dir_full = path1 + parse_prefix_full
-            if not os.path.exists(local_dir_full):
-                os.makedirs(local_dir_full)
-            # 下载两个csv
-            root_path1 = local_dir_full + 'orig/'
-            if not os.path.exists(root_path1):
-                os.makedirs(root_path1)
-
-            bucket.get_object_to_file(parse_prefix_full + 'simulation.xosc', root_path1 + 'scenario_orig.xosc')
-            local_delete_list.append(root_path1 + 'scenario_orig.xosc')
-            move_xosc_before_simulation(root_path1)
-            simulation(parse_prefix_full, root_path1 + 'simulation.mp4')
-
-        # 删除本地临时文件
-        if len(local_delete_list) > 0:
-            for local_delete in local_delete_list:
+        try:
+            logging.info("开始新一轮扫描")
+            local_delete_list = []
+            oss_delete_list = []
+            prefix_list = []
+            # 2 获取已经上传完成的所有目录并分组
+            for obj1 in oss2.ObjectIterator(bucket, prefix=key1):
                 try:
-                    os.remove(local_delete)
+                    # 获取合并后的包
+                    merged_bag_object_key = str(obj1.key)
+                    if 'data_merge' in str(obj1.key) and str(obj1.key).endswith('.bag'):
+                        merged_bag_object_key_split = merged_bag_object_key.split('/')
+                        merged_prefix = '/'.join(merged_bag_object_key_split[:-1])
+                        parse_prefix = merged_prefix.replace('data_merge', 'data_parse')
+                        parse_prefix_full = merged_bag_object_key.replace('data_merge', 'data_parse')[:-4] + '/'
+                        xosc_done = False
+                        mp4_done = False
+                        for obj3 in oss2.ObjectIterator(bucket, prefix=str(parse_prefix_full)):
+                            if '/simulation.xosc' in str(obj3.key):
+                                xosc_done = True
+                            if '/scenario_orig.mp4' in str(obj3.key):
+                                mp4_done = True
+                        if not xosc_done:
+                            continue
+                        if mp4_done:
+                            continue
+                        logging.info("需要生成仿真视频: %s" % str(parse_prefix_full))
+                        parse_prefix_full = str(parse_prefix_full)
+                        local_dir_full = path1 + parse_prefix_full
+                        if not os.path.exists(local_dir_full):
+                            os.makedirs(local_dir_full)
+                        # 下载两个csv
+                        root_path1 = local_dir_full + 'orig/'
+                        if not os.path.exists(root_path1):
+                            os.makedirs(root_path1)
+                        bucket.get_object_to_file(parse_prefix_full + 'simulation.xosc', root_path1 + 'scenario_orig.xosc')
+                        local_delete_list.append(root_path1 + 'scenario_orig.xosc')
+                        move_xosc_before_simulation(root_path1)
+                        simulation(parse_prefix_full, root_path1 + 'simulation.mp4')
                 except Exception as e:
-                    logging.exception("删除本地临时文件报错: %s" % str(e))
-        time.sleep(sleep_time)
+                    logging.exception("局部异常处理: %s", str(e))
+            # 删除本地临时文件
+            if len(local_delete_list) > 0:
+                for local_delete in local_delete_list:
+                    try:
+                        os.remove(local_delete)
+                    except Exception as e:
+                        pass
+            time.sleep(sleep_time)
+        except Exception as e:
+            logging.exception("全局异常处理: %s", str(e))