docker sdk
This commit is contained in:
龙澳
parent
392638b10a
commit
1393839ac8
@ -5,3 +5,4 @@ matplotlib
|
||||
piexif
|
||||
geopy
|
||||
psutil
|
||||
docker>=6.1.3
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
import os
|
||||
import logging
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Dict
|
||||
import pandas as pd
|
||||
from utils.odm_monitor import ODMProcessMonitor
|
||||
@ -11,13 +9,6 @@ class CommandRunner:
|
||||
"""执行网格处理命令的类"""
|
||||
|
||||
def __init__(self, output_dir: str, max_retries: int = 3, mode: str = "快拼模式"):
|
||||
"""
|
||||
初始化命令执行器
|
||||
i
|
||||
Args:
|
||||
output_dir: 输出目录路径
|
||||
max_retries: 最大重试次数
|
||||
"""
|
||||
self.output_dir = output_dir
|
||||
self.max_retries = max_retries
|
||||
self.logger = logging.getLogger('UAV_Preprocess.CommandRunner')
|
||||
@ -25,35 +16,16 @@ i
|
||||
self.mode = mode
|
||||
|
||||
def _run_command(self, grid_idx: int):
|
||||
"""执行单个网格的命令"""
|
||||
"""执行单个网格的处理"""
|
||||
try:
|
||||
grid_dir = os.path.join(self.output_dir, f'grid_{grid_idx + 1}')
|
||||
grid_dir = grid_dir[0].lower() + grid_dir[1:].replace('\\', '/')
|
||||
grid_dir = os.path.abspath(grid_dir) # 确保使用绝对路径
|
||||
|
||||
# 简化命令构建
|
||||
base_command = (
|
||||
f"docker run --rm " # 移除 -i 参数
|
||||
f"-v {grid_dir}:/datasets "
|
||||
f"opendronemap/odm "
|
||||
f"--project-path /datasets project "
|
||||
f"--max-concurrency 10 "
|
||||
f"--force-gps "
|
||||
)
|
||||
|
||||
if self.mode == "快拼模式":
|
||||
command = base_command + (
|
||||
f"--feature-quality lowest "
|
||||
f"--orthophoto-resolution 8 "
|
||||
f"--fast-orthophoto "
|
||||
f"--skip-3dmodel "
|
||||
f"--rerun-all"
|
||||
)
|
||||
else:
|
||||
command = base_command + "--rerun-all"
|
||||
|
||||
self.logger.info(f"运行命令: {command}")
|
||||
success, error_msg = self.monitor.run_odm_with_monitor(
|
||||
command, grid_dir, grid_idx)
|
||||
grid_dir=grid_dir,
|
||||
grid_idx=grid_idx,
|
||||
fast_mode=(self.mode == "快拼模式")
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise Exception(error_msg)
|
||||
@ -63,14 +35,8 @@ i
|
||||
raise
|
||||
|
||||
def run_grid_commands(self, grid_points: Dict[int, pd.DataFrame]):
|
||||
"""
|
||||
为每个网格顺序运行指定命令
|
||||
|
||||
Args:
|
||||
grid_points: 网格点数据字典，键为网格索引，值为该网格的点数据
|
||||
"""
|
||||
|
||||
self.logger.info("开始执行网格处理命令")
|
||||
"""处理所有网格"""
|
||||
self.logger.info("开始执行网格处理")
|
||||
|
||||
for grid_idx in grid_points.keys():
|
||||
try:
|
||||
|
||||
@ -1,30 +1,18 @@
|
||||
import os
|
||||
import time
|
||||
import psutil
|
||||
import logging
|
||||
import subprocess
|
||||
from typing import Optional, Tuple
|
||||
from collections import deque
|
||||
import docker
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
class ODMProcessMonitor:
|
||||
"""ODM进程监控器"""
|
||||
|
||||
def __init__(self, max_retries: int = 3, check_interval: int = 10, mode: str = "快拼模式"):
|
||||
"""
|
||||
初始化监控器
|
||||
|
||||
Args:
|
||||
max_retries: 最大重试次数
|
||||
check_interval: 检查间隔（秒）
|
||||
mode: 模式
|
||||
"""
|
||||
def __init__(self, max_retries: int = 3, mode: str = "快拼模式"):
|
||||
self.max_retries = max_retries
|
||||
self.check_interval = check_interval
|
||||
self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor')
|
||||
self.mode = mode
|
||||
# 用于存储最后N行输出
|
||||
self.last_outputs = deque(maxlen=50)
|
||||
self.client = docker.from_env()
|
||||
|
||||
def _check_success(self, grid_dir: str) -> bool:
|
||||
"""检查ODM是否执行成功"""
|
||||
@ -33,51 +21,66 @@ class ODMProcessMonitor:
|
||||
success_markers.append('odm_texturing')
|
||||
return all(os.path.exists(os.path.join(grid_dir, 'project', marker)) for marker in success_markers)
|
||||
|
||||
def run_odm_with_monitor(self, command: str, grid_dir: str, grid_idx: int) -> Tuple[bool, str]:
|
||||
"""运行ODM命令并监控进程"""
|
||||
def run_odm_with_monitor(self, grid_dir: str, grid_idx: int, fast_mode: bool = True) -> Tuple[bool, str]:
|
||||
"""运行ODM容器"""
|
||||
attempt = 0
|
||||
while attempt < self.max_retries:
|
||||
try:
|
||||
self.logger.info(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试执行ODM")
|
||||
self.logger.info(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试")
|
||||
|
||||
# 使用 subprocess.Popen 启动进程
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
shell=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
bufsize=-1, # 使用系统默认缓冲
|
||||
text=True
|
||||
# 准备容器配置
|
||||
volumes = {
|
||||
grid_dir: {'bind': '/datasets', 'mode': 'rw'}
|
||||
}
|
||||
|
||||
# 准备命令参数
|
||||
command = [
|
||||
"--project-path", "/datasets", "project",
|
||||
"--max-concurrency", "10",
|
||||
"--force-gps",
|
||||
"--rerun-all"
|
||||
]
|
||||
|
||||
if fast_mode:
|
||||
command.extend([
|
||||
"--feature-quality", "lowest",
|
||||
"--orthophoto-resolution", "8",
|
||||
"--fast-orthophoto",
|
||||
"--skip-3dmodel"
|
||||
])
|
||||
|
||||
# 运行容器
|
||||
container = self.client.containers.run(
|
||||
"opendronemap/odm",
|
||||
command=command,
|
||||
volumes=volumes,
|
||||
detach=True,
|
||||
remove=True
|
||||
)
|
||||
|
||||
# 等待进程完成并获取所有输出
|
||||
stdout, stderr = process.communicate()
|
||||
return_code = process.returncode
|
||||
# 等待容器完成
|
||||
result = container.wait()
|
||||
|
||||
# 只保存最后N行输出
|
||||
last_lines = stdout.strip().split('\n')[-50:]
|
||||
if last_lines:
|
||||
self.logger.info(f"网格 {grid_idx + 1} 最后的输出:")
|
||||
for line in last_lines:
|
||||
self.logger.info(line)
|
||||
# 只在失败时获取日志
|
||||
if result['StatusCode'] != 0:
|
||||
logs = container.logs().decode('utf-8')
|
||||
self.logger.error(f"容器执行失败，最后100行日志：\n{''.join(logs.split('\n')[-100:])}")
|
||||
|
||||
# 检查是否成功完成
|
||||
if return_code == 0 and self._check_success(grid_dir):
|
||||
if result['StatusCode'] == 0 and self._check_success(grid_dir):
|
||||
self.logger.info(f"网格 {grid_idx + 1} ODM处理成功")
|
||||
return True, ""
|
||||
|
||||
self.logger.warning(f"网格 {grid_idx + 1} 第 {attempt + 1} 次尝试失败")
|
||||
if stderr:
|
||||
self.logger.error(f"错误输出: {stderr}")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"监控进程发生异常: {str(e)}"
|
||||
error_msg = f"执行异常: {str(e)}"
|
||||
self.logger.error(error_msg)
|
||||
return False, error_msg
|
||||
|
||||
attempt += 1
|
||||
if attempt < self.max_retries:
|
||||
time.sleep(30) # 固定等待时间
|
||||
time.sleep(30)
|
||||
|
||||
error_msg = f"网格 {grid_idx + 1} 在 {self.max_retries} 次尝试后仍然失败"
|
||||
self.logger.error(error_msg)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user