From 1393839ac836a02889f0ca9a79613deaa5cea2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BE=99=E6=BE=B3?= Date: Wed, 25 Dec 2024 14:28:01 +0800 Subject: [PATCH] docker sdk --- requirements.txt | 1 + utils/command_runner.py | 50 ++++------------------- utils/odm_monitor.py | 89 +++++++++++++++++++++-------------------- 3 files changed, 55 insertions(+), 85 deletions(-) diff --git a/requirements.txt b/requirements.txt index 12f3f05..ff696df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ matplotlib piexif geopy psutil +docker>=6.1.3 diff --git a/utils/command_runner.py b/utils/command_runner.py index c4cc39c..b098ca1 100644 --- a/utils/command_runner.py +++ b/utils/command_runner.py @@ -1,7 +1,5 @@ import os import logging -import subprocess -import time from typing import Dict import pandas as pd from utils.odm_monitor import ODMProcessMonitor @@ -11,13 +9,6 @@ class CommandRunner: """执行网格处理命令的类""" def __init__(self, output_dir: str, max_retries: int = 3, mode: str = "快拼模式"): - """ - 初始化命令执行器 -i - Args: - output_dir: 输出目录路径 - max_retries: 最大重试次数 - """ self.output_dir = output_dir self.max_retries = max_retries self.logger = logging.getLogger('UAV_Preprocess.CommandRunner') @@ -25,35 +16,16 @@ i self.mode = mode def _run_command(self, grid_idx: int): - """执行单个网格的命令""" + """执行单个网格的处理""" try: grid_dir = os.path.join(self.output_dir, f'grid_{grid_idx + 1}') - grid_dir = grid_dir[0].lower() + grid_dir[1:].replace('\\', '/') + grid_dir = os.path.abspath(grid_dir) # 确保使用绝对路径 - # 简化命令构建 - base_command = ( - f"docker run --rm " # 移除 -i 参数 - f"-v {grid_dir}:/datasets " - f"opendronemap/odm " - f"--project-path /datasets project " - f"--max-concurrency 10 " - f"--force-gps " - ) - - if self.mode == "快拼模式": - command = base_command + ( - f"--feature-quality lowest " - f"--orthophoto-resolution 8 " - f"--fast-orthophoto " - f"--skip-3dmodel " - f"--rerun-all" - ) - else: - command = base_command + "--rerun-all" - - self.logger.info(f"运行命令: {command}") success, error_msg = self.monitor.run_odm_with_monitor( - command, grid_dir, grid_idx) + grid_dir=grid_dir, + grid_idx=grid_idx, + fast_mode=(self.mode == "快拼模式") + ) if not success: raise Exception(error_msg) @@ -63,14 +35,8 @@ i raise def run_grid_commands(self, grid_points: Dict[int, pd.DataFrame]): - """ - 为每个网格顺序运行指定命令 - - Args: - grid_points: 网格点数据字典，键为网格索引，值为该网格的点数据 - """ - - self.logger.info("开始执行网格处理命令") + """处理所有网格""" + self.logger.info("开始执行网格处理") for grid_idx in grid_points.keys(): try: diff --git a/utils/odm_monitor.py b/utils/odm_monitor.py index 94f6d39..0a5fa3a 100644 --- a/utils/odm_monitor.py +++ b/utils/odm_monitor.py @@ -1,30 +1,18 @@ import os import time -import psutil import logging -import subprocess -from typing import Optional, Tuple -from collections import deque +import docker +from typing import Tuple class ODMProcessMonitor: """ODM进程监控器""" - def __init__(self, max_retries: int = 3, check_interval: int = 10, mode: str = "快拼模式"): - """ - 初始化监控器 - - Args: - max_retries: 最大重试次数 - check_interval: 检查间隔（秒） - mode: 模式 - """ + def __init__(self, max_retries: int = 3, mode: str = "快拼模式"): self.max_retries = max_retries - self.check_interval = check_interval self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor') self.mode = mode - # 用于存储最后N行输出 - self.last_outputs = deque(maxlen=50) + self.client = docker.from_env() def _check_success(self, grid_dir: str) -> bool: """检查ODM是否执行成功""" @@ -33,51 +21,66 @@ class ODMProcessMonitor: success_markers.append('odm_texturing') return all(os.path.exists(os.path.join(grid_dir, 'project', marker)) for marker in success_markers) - def run_odm_with_monitor(self, command: str, grid_dir: str, grid_idx: int) -> Tuple[bool, str]: - """运行ODM命令并监控进程""" + def run_odm_with_monitor(self, grid_dir: str, grid_idx: int, fast_mode: bool = True) -> Tuple[bool, str]: + """运行ODM容器""" attempt = 0 while attempt < self.max_retries: try: - self.logger.info(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试执行ODM") + self.logger.info(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试") - # 使用 subprocess.Popen 启动进程 - process = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=-1, # 使用系统默认缓冲 - text=True + # 准备容器配置 + volumes = { + grid_dir: {'bind': '/datasets', 'mode': 'rw'} + } + + # 准备命令参数 + command = [ + "--project-path", "/datasets", "project", + "--max-concurrency", "10", + "--force-gps", + "--rerun-all" + ] + + if fast_mode: + command.extend([ + "--feature-quality", "lowest", + "--orthophoto-resolution", "8", + "--fast-orthophoto", + "--skip-3dmodel" + ]) + + # 运行容器 + container = self.client.containers.run( + "opendronemap/odm", + command=command, + volumes=volumes, + detach=True, + remove=True ) - # 等待进程完成并获取所有输出 - stdout, stderr = process.communicate() - return_code = process.returncode - - # 只保存最后N行输出 - last_lines = stdout.strip().split('\n')[-50:] - if last_lines: - self.logger.info(f"网格 {grid_idx + 1} 最后的输出:") - for line in last_lines: - self.logger.info(line) - + # 等待容器完成 + result = container.wait() + + # 只在失败时获取日志 + if result['StatusCode'] != 0: + logs = container.logs().decode('utf-8') + self.logger.error(f"容器执行失败，最后100行日志：\n{''.join(logs.split('\n')[-100:])}") + # 检查是否成功完成 - if return_code == 0 and self._check_success(grid_dir): + if result['StatusCode'] == 0 and self._check_success(grid_dir): self.logger.info(f"网格 {grid_idx + 1} ODM处理成功") return True, "" self.logger.warning(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试失败") - if stderr: - self.logger.error(f"错误输出: {stderr}") except Exception as e: - error_msg = f"监控进程发生异常: {str(e)}" + error_msg = f"执行异常: {str(e)}" self.logger.error(error_msg) return False, error_msg attempt += 1 if attempt < self.max_retries: - time.sleep(30) # 固定等待时间 + time.sleep(30) error_msg = f"网格 {grid_idx + 1} 在 {self.max_retries} 次尝试后仍然失败" self.logger.error(error_msg)