utils/command_runner.py

@@ -8,20 +8,19 @@ from utils.odm_monitor import ODMProcessMonitor
 class CommandRunner:
     """执行网格处理命令的类"""
 
-    def __init__(self, output_dir: str, mode: str = "快拼模式"):
+    def __init__(self, output_dir: str, max_retries: int = 3, mode: str = "快拼模式"):
         self.output_dir = output_dir
+        self.max_retries = max_retries
         self.logger = logging.getLogger('UAV_Preprocess.CommandRunner')
-        self.monitor = ODMProcessMonitor(mode=mode)
+        self.monitor = ODMProcessMonitor(max_retries=max_retries, mode=mode)
         self.mode = mode
 
-    def run_grid_commands(self, grid_points: Dict[int, pd.DataFrame]):
-        """处理所有网格"""
-        self.logger.info("开始执行网格处理")
-        for grid_idx in grid_points.keys():
-            grid_dir = os.path.abspath(os.path.join(
-                self.output_dir, f'grid_{grid_idx + 1}'
-            ))
-
+    def _run_command(self, grid_idx: int):
+        """执行单个网格的处理"""
+        try:
+            grid_dir = os.path.join(self.output_dir, f'grid_{grid_idx + 1}')
+            grid_dir = os.path.abspath(grid_dir)  # 确保使用绝对路径
+            
             success, error_msg = self.monitor.run_odm_with_monitor(
                 grid_dir=grid_dir,
                 grid_idx=grid_idx,
@@ -29,4 +28,19 @@ class CommandRunner:
             )
 
             if not success:
-                raise Exception(f"网格 {grid_idx + 1} 处理失败: {error_msg}")
+                raise Exception(error_msg)
+
+        except Exception as e:
+            self.logger.error(f"网格 {grid_idx + 1} 处理失败: {str(e)}")
+            raise
+
+    def run_grid_commands(self, grid_points: Dict[int, pd.DataFrame]):
+        """处理所有网格"""
+        self.logger.info("开始执行网格处理")
+
+        for grid_idx in grid_points.keys():
+            try:
+                self._run_command(grid_idx)
+            except Exception as e:
+                self.logger.error(f"网格 {grid_idx + 1} 处理失败，停止后续执行: {str(e)}")
+                raise

utils/odm_monitor.py

@@ -1,4 +1,5 @@
 import os
+import time
 import logging
 import subprocess
 from typing import Tuple
@@ -21,54 +22,57 @@ class ODMProcessMonitor:
 
     def run_odm_with_monitor(self, grid_dir: str, grid_idx: int, fast_mode: bool = True) -> Tuple[bool, str]:
         """运行ODM命令"""
-        try:
-            self.logger.info(f"开始处理网格 {grid_idx + 1}")
+        attempt = 0
+        while attempt < self.max_retries:
+            try:
+                self.logger.info(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试")
                 
-            # 构建基础命令
-            cmd = [
-                "docker", "run", "--rm",
-                "-v", f"{grid_dir}:/datasets",
-            "opendronemap/odm",
-                "--project-path", "/datasets", "project",
-                "--max-concurrency", "10",
-                "--force-gps",
-                "--rerun-all"
-            ]
+                # 构建基础命令
+                cmd = [
+                    "docker", "run", "--rm",
+                    "-v", f"{grid_dir}:/datasets",
+                    "opendronemap/odm",
+                    "--project-path", "/datasets", "project",
+                    "--max-concurrency", "10",
+                    "--force-gps",
+                    "--rerun-all"
+                ]
 
+                # 快速模式额外参数
                 if fast_mode:
-                    command.extend([
+                    cmd.extend([
                         "--feature-quality", "lowest",
                         "--orthophoto-resolution", "8",
                         "--fast-orthophoto",
                         "--skip-3dmodel"
                     ])
 
-                # 运行容器
-                container = self.client.containers.run(
-                    "opendronemap/odm",
-                    command=command,
-                    volumes=volumes,
-                    detach=True,
-                    remove=True
+                # 执行命令，不捕获输出
+                result = subprocess.run(
+                    cmd,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.PIPE,
+                    text=True
                 )
 
-                # 等待容器完成
-                result = container.wait()
-
-                # 只在失败时获取日志
-                if result['StatusCode'] != 0:
-                    logs = container.logs().decode('utf-8')
-                    self.logger.error("容器执行失败，最后10行日志：")
-                    self.logger.error(''.join(logs.split('\n')[-10:]))
-
-                # 检查是否成功完成
-                if result['StatusCode'] == 0 and self._check_success(grid_dir):
+                # 检查执行结果
+                if result.returncode == 0 and self._check_success(grid_dir):
                     self.logger.info(f"网格 {grid_idx + 1} ODM处理成功")
                     return True, ""
+                
+                # 只在失败时记录错误信息
+                if result.returncode != 0:
+                    self.logger.error(f"网格 {grid_idx + 1} 执行失败，错误信息：{result.stderr[-500:]}")
 
-                self.logger.warning(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试失败")
+            except Exception as e:
+                error_msg = f"执行异常: {str(e)}"
+                self.logger.error(error_msg)
+                return False, error_msg
 
-        except Exception as e:
-            error_msg = f"执行异常: {str(e)}"
-            self.logger.error(error_msg)
-            return False, error_msg
+            attempt += 1
+            if attempt < self.max_retries:
+                time.sleep(30)
+
+        error_msg = f"网格 {grid_idx + 1} 在 {self.max_retries} 次尝试后仍然失败"
+        self.logger.error(error_msg)
+        return False, error_msg