精简代码

This commit is contained in:
weixin_46229132 2025-04-11 17:22:11 +08:00
parent e86fe196f8
commit 5c382e1810
9 changed files with 281 additions and 494 deletions

View File

@ -1,12 +1,12 @@
import os import os
import shutil import shutil
from datetime import timedelta
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict, Tuple from typing import Dict, Tuple
import psutil import psutil
import pandas as pd import pandas as pd
from filter.cluster_filter import GPSCluster from filter.cluster_filter import GPSCluster
from utils.directory_manager import DirectoryManager
from utils.odm_monitor import ODMProcessMonitor from utils.odm_monitor import ODMProcessMonitor
from utils.gps_extractor import GPSExtractor from utils.gps_extractor import GPSExtractor
from utils.grid_divider import GridDivider from utils.grid_divider import GridDivider
@ -19,115 +19,38 @@ from post_pro.conv_obj import ConvertOBJ
@dataclass @dataclass
class ProcessConfig: class ProcessConfig:
"""预处理配置类""" """预处理配置类"""
image_dir: str image_dir: str
output_dir: str output_dir: str
# 聚类过滤参数 # 聚类过滤参数
cluster_eps: float = 0.01 cluster_eps: float = 0.01
cluster_min_samples: int = 5 cluster_min_samples: int = 5
# 时间组重叠过滤参数
time_group_overlap_threshold: float = 0.7
time_group_interval: timedelta = timedelta(minutes=5)
# 孤立点过滤参数
filter_distance_threshold: float = 0.001 # 经纬度距离
filter_min_neighbors: int = 6
# 密集点过滤参数
filter_grid_size: float = 0.001
filter_dense_distance_threshold: float = 10 # 普通距离,单位:米
filter_time_threshold: timedelta = timedelta(minutes=5)
# 网格划分参数 # 网格划分参数
grid_overlap: float = 0.05 grid_overlap: float = 0.05
grid_size: float = 500 grid_size: float = 500
# 几个pipline过程是否开启
mode: str = "快拼模式" mode: str = "三维模式"
accuracy: str = "medium"
produce_dem: bool = False
class ODM_Plugin: class ODM_Plugin:
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
# 检查磁盘空间 # 初始化目录管理器
self._check_disk_space() self.dir_manager = DirectoryManager(config)
# 清理并重建输出目录 # 清理并重建输出目录
if os.path.exists(config.output_dir): self.dir_manager.clean_output_dir()
self._clean_output_dir() self.dir_manager.setup_output_dirs()
self._setup_output_dirs() # 检查磁盘空间
self.dir_manager.check_disk_space()
# 初始化其他组件 # 初始化其他组件
self.logger = setup_logger(config.output_dir) self.logger = setup_logger(config.output_dir)
self.gps_points = None self.gps_points = pd.DataFrame(columns=["file", "lat", "lon"])
self.odm_monitor = ODMProcessMonitor( self.odm_monitor = ODMProcessMonitor(
config.output_dir, mode=config.mode) config.output_dir, mode=config.mode)
self.visualizer = FilterVisualizer(config.output_dir) self.visualizer = FilterVisualizer(config.output_dir)
def _clean_output_dir(self):
"""清理输出目录"""
try:
shutil.rmtree(self.config.output_dir)
print(f"已清理输出目录: {self.config.output_dir}")
except Exception as e:
print(f"清理输出目录时发生错误: {str(e)}")
raise
def _setup_output_dirs(self):
"""创建必要的输出目录结构"""
try:
# 创建主输出目录
os.makedirs(self.config.output_dir)
# 创建过滤图像保存目录
os.makedirs(os.path.join(self.config.output_dir, 'filter_imgs'))
# 创建日志目录
os.makedirs(os.path.join(self.config.output_dir, 'logs'))
print(f"已创建输出目录结构: {self.config.output_dir}")
except Exception as e:
print(f"创建输出目录时发生错误: {str(e)}")
raise
def _get_directory_size(self, path):
"""获取目录的总大小(字节)"""
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
try:
total_size += os.path.getsize(file_path)
except (OSError, FileNotFoundError):
continue
return total_size
def _check_disk_space(self):
"""检查磁盘空间是否足够"""
# 获取输入目录大小
input_size = self._get_directory_size(self.config.image_dir)
# 获取输出目录所在磁盘的剩余空间
output_drive = os.path.splitdrive(
os.path.abspath(self.config.output_dir))[0]
if not output_drive: # 处理Linux/Unix路径
output_drive = '/home'
disk_usage = psutil.disk_usage(output_drive)
free_space = disk_usage.free
# 计算所需空间输入大小的1.5倍)
required_space = input_size * 12
if free_space < required_space:
error_msg = (
f"磁盘空间不足!\n"
f"输入目录大小: {input_size / (1024**3):.2f} GB\n"
f"所需空间: {required_space / (1024**3):.2f} GB\n"
f"可用空间: {free_space / (1024**3):.2f} GB\n"
f"在驱动器 {output_drive}"
)
raise RuntimeError(error_msg)
def extract_gps(self) -> pd.DataFrame: def extract_gps(self) -> pd.DataFrame:
"""提取GPS数据""" """提取GPS数据"""
self.logger.info("开始提取GPS数据") self.logger.info("开始提取GPS数据")
@ -150,10 +73,9 @@ class ODM_Plugin:
self.visualizer.visualize_filter_step( self.visualizer.visualize_filter_step(
self.gps_points, previous_points, "1-Clustering") self.gps_points, previous_points, "1-Clustering")
def divide_grids(self) -> Tuple[Dict[tuple, pd.DataFrame], Dict[tuple, tuple]]: def divide_grids(self) -> Dict[tuple, pd.DataFrame]:
"""划分网格 """划分网格
Returns: Returns:
tuple: (grid_points, translations)
- grid_points: 网格点数据字典 - grid_points: 网格点数据字典
- translations: 网格平移量字典 - translations: 网格平移量字典
""" """
@ -162,14 +84,14 @@ class ODM_Plugin:
grid_size=self.config.grid_size, grid_size=self.config.grid_size,
output_dir=self.config.output_dir output_dir=self.config.output_dir
) )
grids, translations, grid_points = grid_divider.adjust_grid_size_and_overlap( grids, grid_points = grid_divider.adjust_grid_size_and_overlap(
self.gps_points self.gps_points
) )
grid_divider.visualize_grids(self.gps_points, grids) grid_divider.visualize_grids(self.gps_points, grids)
if len(grids) >= 20: if len(grids) >= 20:
self.logger.warning("网格数量已超过20, 需要人工调整分区") self.logger.warning("网格数量已超过20, 需要人工调整分区")
return grid_points, translations return grid_points
def copy_images(self, grid_points: Dict[tuple, pd.DataFrame]): def copy_images(self, grid_points: Dict[tuple, pd.DataFrame]):
"""复制图像到目标文件夹""" """复制图像到目标文件夹"""
@ -192,46 +114,45 @@ class ODM_Plugin:
self.logger.info( self.logger.info(
f"网格 ({grid_id[0]},{grid_id[1]}) 包含 {len(points)} 张图像") f"网格 ({grid_id[0]},{grid_id[1]}) 包含 {len(points)} 张图像")
def merge_tif(self, grid_points: Dict[tuple, pd.DataFrame], mode: str, produce_dem: bool): def merge_tif(self, grid_lt):
"""合并所有网格的影像产品""" """合并所有网格的影像产品"""
self.logger.info("开始合并所有影像产品") self.logger.info("开始合并所有影像产品")
merger = MergeTif(self.config.output_dir) merger = MergeTif(self.config.output_dir)
merger.merge_orthophoto(grid_points) merger.merge_orthophoto(grid_lt)
def convert_obj(self, grid_points: Dict[tuple, pd.DataFrame]): def convert_obj(self, grid_lt):
"""转换OBJ模型""" """转换OBJ模型"""
self.logger.info("开始转换OBJ模型") self.logger.info("开始转换OBJ模型")
converter = ConvertOBJ(self.config.output_dir) converter = ConvertOBJ(self.config.output_dir)
converter.convert_grid_obj(grid_points) converter.convert_grid_obj(grid_lt)
def post_process(self, successful_grid_points: Dict[tuple, pd.DataFrame], grid_points: Dict[tuple, pd.DataFrame], translations: Dict[tuple, tuple]): def post_process(self, successful_grid_lt: list, grid_points: Dict[tuple, pd.DataFrame]):
"""后处理:合并或复制处理结果""" """后处理:合并或复制处理结果"""
if len(successful_grid_points) < len(grid_points): if len(successful_grid_lt) < len(grid_points):
self.logger.warning( self.logger.warning(
f"{len(grid_points) - len(successful_grid_points)} 个网格处理失败," f"{len(grid_points) - len(successful_grid_lt)} 个网格处理失败,"
f"将只合并成功处理的 {len(successful_grid_points)} 个网格" f"将只合并成功处理的 {len(successful_grid_lt)} 个网格"
) )
self.merge_tif(successful_grid_points, self.config.mode, self.merge_tif(successful_grid_lt)
self.config.produce_dem)
if self.config.mode == "三维模式": if self.config.mode == "三维模式":
self.convert_obj(successful_grid_lt)
self.convert_obj(successful_grid_points) else:
pass
def process(self): def process(self):
"""执行完整的预处理流程""" """执行完整的预处理流程"""
try: try:
self.extract_gps() self.extract_gps()
self.cluster() self.cluster()
grid_points, translations = self.divide_grids() grid_points = self.divide_grids()
self.copy_images(grid_points) self.copy_images(grid_points)
self.logger.info("预处理任务完成") self.logger.info("预处理任务完成")
successful_grid_points = self.odm_monitor.process_all_grids( successful_grid_lt = self.odm_monitor.process_all_grids(
grid_points, self.config.produce_dem, self.config.accuracy) grid_points)
self.post_process(successful_grid_points, self.post_process(successful_grid_lt, grid_points)
grid_points, translations)
self.logger.info("重建任务完成") self.logger.info("重建任务完成")
except Exception as e: except Exception as e:

17
main.py
View File

@ -1,5 +1,4 @@
import argparse import argparse
from datetime import timedelta
from app_plugin import ProcessConfig, ODM_Plugin from app_plugin import ProcessConfig, ODM_Plugin
@ -12,13 +11,10 @@ def parse_args():
# 可选参数 # 可选参数
parser.add_argument('--mode', default='三维模式', parser.add_argument('--mode', default='三维模式',
choices=['快拼模式', '三维模式', '重建模式'], help='处理模式') choices=['快拼模式', '三维模式'], help='处理模式')
parser.add_argument('--accuracy', default='medium',
choices=['high', 'medium', 'low'], help='精度')
parser.add_argument('--grid_size', type=float, default=800, help='网格大小(米)') parser.add_argument('--grid_size', type=float, default=800, help='网格大小(米)')
parser.add_argument('--grid_overlap', type=float, parser.add_argument('--grid_overlap', type=float,
default=0.05, help='网格重叠率') default=0.05, help='网格重叠率')
# parser.add_argument('--produce_dem', action='store_true', help='是否生成DEM')
args = parser.parse_args() args = parser.parse_args()
return args return args
@ -32,21 +28,12 @@ def main():
image_dir=args.image_dir, image_dir=args.image_dir,
output_dir=args.output_dir, output_dir=args.output_dir,
mode=args.mode, mode=args.mode,
accuracy=args.accuracy,
grid_size=args.grid_size, grid_size=args.grid_size,
grid_overlap=args.grid_overlap,
produce_dem=True,
# 其他参数使用默认值 # 其他参数使用默认值
grid_overlap=0.05,
cluster_eps=0.01, cluster_eps=0.01,
cluster_min_samples=5, cluster_min_samples=5,
time_group_overlap_threshold=0.7,
time_group_interval=timedelta(minutes=5),
filter_distance_threshold=0.001,
filter_min_neighbors=6,
filter_grid_size=0.001,
filter_dense_distance_threshold=10,
filter_time_threshold=timedelta(minutes=5),
) )
# 创建处理器并执行 # 创建处理器并执行

View File

@ -18,13 +18,13 @@ class ConvertOBJ:
"EPSG:32649", "EPSG:4326", always_xy=True) "EPSG:32649", "EPSG:4326", always_xy=True)
self.logger = logging.getLogger('UAV_Preprocess.ConvertOBJ') self.logger = logging.getLogger('UAV_Preprocess.ConvertOBJ')
def convert_grid_obj(self, grid_points): def convert_grid_obj(self, grid_lt):
"""转换每个网格的OBJ文件为OSGB格式""" """转换每个网格的OBJ文件为OSGB格式"""
os.makedirs(os.path.join(self.output_dir, os.makedirs(os.path.join(self.output_dir,
"osgb", "Data"), exist_ok=True) "osgb", "Data"), exist_ok=True)
# 以第一个grid的UTM坐标作为参照系 # 以第一个grid的UTM坐标作为参照系
first_grid_id = list(grid_points.keys())[0] first_grid_id = grid_lt[0]
first_grid_dir = os.path.join( first_grid_dir = os.path.join(
self.output_dir, self.output_dir,
f"grid_{first_grid_id[0]}_{first_grid_id[1]}", f"grid_{first_grid_id[0]}_{first_grid_id[1]}",
@ -34,15 +34,15 @@ class ConvertOBJ:
first_grid_dir, "odm_orthophoto", "odm_orthophoto_log.txt") first_grid_dir, "odm_orthophoto", "odm_orthophoto_log.txt")
self.ref_east, self.ref_north = self.read_utm_offset(log_file) self.ref_east, self.ref_north = self.read_utm_offset(log_file)
for grid_id in grid_points.keys(): for grid_id in grid_lt:
try: try:
self._convert_single_grid(grid_id, grid_points) self._convert_single_grid(grid_id)
except Exception as e: except Exception as e:
self.logger.error(f"网格 {grid_id} 转换失败: {str(e)}") self.logger.error(f"网格 {grid_id} 转换失败: {str(e)}")
self._create_merged_metadata() self._create_merged_metadata()
def _convert_single_grid(self, grid_id, grid_points): def _convert_single_grid(self, grid_id):
"""转换单个网格的OBJ文件""" """转换单个网格的OBJ文件"""
# 构建相关路径 # 构建相关路径
grid_name = f"grid_{grid_id[0]}_{grid_id[1]}" grid_name = f"grid_{grid_id[0]}_{grid_id[1]}"
@ -50,7 +50,6 @@ class ConvertOBJ:
texturing_dir = os.path.join(project_dir, "odm_texturing") texturing_dir = os.path.join(project_dir, "odm_texturing")
texturing_dst_dir = os.path.join(project_dir, "odm_texturing_dst") texturing_dst_dir = os.path.join(project_dir, "odm_texturing_dst")
split_obj_dir = os.path.join(texturing_dst_dir, "split_obj") split_obj_dir = os.path.join(texturing_dst_dir, "split_obj")
opensfm_dir = os.path.join(project_dir, "opensfm")
log_file = os.path.join( log_file = os.path.join(
project_dir, "odm_orthophoto", "odm_orthophoto_log.txt") project_dir, "odm_orthophoto", "odm_orthophoto_log.txt")
os.makedirs(texturing_dst_dir, exist_ok=True) os.makedirs(texturing_dst_dir, exist_ok=True)

View File

@ -19,11 +19,11 @@ class MergeTif:
self.output_dir = output_dir self.output_dir = output_dir
self.logger = logging.getLogger('UAV_Preprocess.MergeTif') self.logger = logging.getLogger('UAV_Preprocess.MergeTif')
def merge_orthophoto(self, grid_points: Dict[tuple, pd.DataFrame]): def merge_orthophoto(self, grid_lt):
"""合并网格的正射影像""" """合并网格的正射影像"""
try: try:
all_orthos_and_ortho_cuts = [] all_orthos_and_ortho_cuts = []
for grid_id, points in grid_points.items(): for grid_id in grid_lt:
grid_ortho_dir = os.path.join( grid_ortho_dir = os.path.join(
self.output_dir, self.output_dir,
f"grid_{grid_id[0]}_{grid_id[1]}", f"grid_{grid_id[0]}_{grid_id[1]}",

View File

@ -0,0 +1,78 @@
import os
import shutil
import psutil
class DirectoryManager:
def __init__(self, config):
"""
初始化目录管理器
Args:
config: 配置对象包含输入和输出目录等信息
"""
self.config = config
def clean_output_dir(self):
"""清理输出目录"""
try:
shutil.rmtree(self.config.output_dir)
print(f"已清理输出目录: {self.config.output_dir}")
except Exception as e:
print(f"清理输出目录时发生错误: {str(e)}")
raise
def setup_output_dirs(self):
"""创建必要的输出目录结构"""
try:
# 创建主输出目录
os.makedirs(self.config.output_dir)
# 创建过滤图像保存目录
os.makedirs(os.path.join(self.config.output_dir, 'filter_imgs'))
# 创建日志目录
os.makedirs(os.path.join(self.config.output_dir, 'logs'))
print(f"已创建输出目录结构: {self.config.output_dir}")
except Exception as e:
print(f"创建输出目录时发生错误: {str(e)}")
raise
def _get_directory_size(self, path):
"""获取目录的总大小(字节)"""
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
try:
total_size += os.path.getsize(file_path)
except (OSError, FileNotFoundError):
continue
return total_size
def check_disk_space(self):
"""检查磁盘空间是否足够"""
# 获取输入目录大小
input_size = self._get_directory_size(self.config.image_dir)
# 获取输出目录所在磁盘的剩余空间
output_drive = os.path.splitdrive(
os.path.abspath(self.config.output_dir))[0]
if not output_drive: # 处理Linux/Unix路径
output_drive = '/home'
disk_usage = psutil.disk_usage(output_drive)
free_space = disk_usage.free
# 计算所需空间输入大小的10倍
required_space = input_size * 10
if free_space < required_space:
error_msg = (
f"磁盘空间不足!\n"
f"输入目录大小: {input_size / (1024**3):.2f} GB\n"
f"所需空间: {required_space / (1024**3):.2f} GB\n"
f"可用空间: {free_space / (1024**3):.2f} GB\n"
f"在驱动器 {output_drive}"
)
raise RuntimeError(error_msg)

View File

@ -3,11 +3,10 @@ from PIL import Image
import piexif import piexif
import logging import logging
import pandas as pd import pandas as pd
from datetime import datetime
class GPSExtractor: class GPSExtractor:
"""从图像文件提取GPS坐标和拍摄日期""" """从图像文件提取GPS坐标"""
def __init__(self, image_dir): def __init__(self, image_dir):
self.image_dir = image_dir self.image_dir = image_dir
@ -18,17 +17,8 @@ class GPSExtractor:
"""将DMS格式转换为十进制度""" """将DMS格式转换为十进制度"""
return dms[0][0] / dms[0][1] + (dms[1][0] / dms[1][1]) / 60 + (dms[2][0] / dms[2][1]) / 3600 return dms[0][0] / dms[0][1] + (dms[1][0] / dms[1][1]) / 60 + (dms[2][0] / dms[2][1]) / 3600
@staticmethod def get_gps(self, image_path):
def _parse_datetime(datetime_str): """提取单张图片的GPS坐标"""
"""解析EXIF中的日期时间字符串"""
try:
# EXIF日期格式通常为 'YYYY:MM:DD HH:MM:SS'
return datetime.strptime(datetime_str.decode(), '%Y:%m:%d %H:%M:%S')
except Exception:
return None
def get_gps_and_date(self, image_path):
"""提取单张图片的GPS坐标和拍摄日期"""
try: try:
image = Image.open(image_path) image = Image.open(image_path)
exif_data = piexif.load(image.info['exif']) exif_data = piexif.load(image.info['exif'])
@ -39,38 +29,21 @@ class GPSExtractor:
if gps_info: if gps_info:
lat = self._dms_to_decimal(gps_info.get(2, [])) lat = self._dms_to_decimal(gps_info.get(2, []))
lon = self._dms_to_decimal(gps_info.get(4, [])) lon = self._dms_to_decimal(gps_info.get(4, []))
self.logger.debug(f"成功提取图片GPS坐标: {image_path} - 纬度: {lat}, 经度: {lon}") self.logger.debug(
f"成功提取图片GPS坐标: {image_path} - 纬度: {lat}, 经度: {lon}")
# 提取拍摄日期
date_info = None
if "Exif" in exif_data:
# 优先使用DateTimeOriginal
date_str = exif_data["Exif"].get(36867) # DateTimeOriginal
if not date_str:
# 备选DateTime
date_str = exif_data["Exif"].get(36868) # DateTimeDigitized
if not date_str:
# 最后使用基本DateTime
date_str = exif_data["0th"].get(306) # DateTime
if date_str:
date_info = self._parse_datetime(date_str)
self.logger.debug(f"成功提取图片拍摄日期: {image_path} - {date_info}")
if not gps_info: if not gps_info:
self.logger.warning(f"图片无GPS信息: {image_path}") self.logger.warning(f"图片无GPS信息: {image_path}")
if not date_info:
self.logger.warning(f"图片无拍摄日期信息: {image_path}")
return lat, lon, date_info return lat, lon
except Exception as e: except Exception as e:
self.logger.error(f"提取图片信息时发生错误: {image_path} - {str(e)}") self.logger.error(f"提取图片信息时发生错误: {image_path} - {str(e)}")
return None, None, None return None, None, None
def extract_all_gps(self): def extract_all_gps(self):
"""提取所有图片的GPS坐标和拍摄日期""" """提取所有图片的GPS坐标"""
self.logger.info(f"开始从目录提取GPS坐标和拍摄日期: {self.image_dir}") self.logger.info(f"开始从目录提取GPS坐标: {self.image_dir}")
gps_data = [] gps_data = []
total_images = 0 total_images = 0
successful_extractions = 0 successful_extractions = 0
@ -78,15 +51,15 @@ class GPSExtractor:
for image_file in os.listdir(self.image_dir): for image_file in os.listdir(self.image_dir):
total_images += 1 total_images += 1
image_path = os.path.join(self.image_dir, image_file) image_path = os.path.join(self.image_dir, image_file)
lat, lon, date = self.get_gps_and_date(image_path) lat, lon = self.get_gps(image_path)
if lat and lon: # 仍然以GPS信息作为主要判断依据 if lat and lon: # 仍然以GPS信息作为主要判断依据
successful_extractions += 1 successful_extractions += 1
gps_data.append({ gps_data.append({
'file': image_file, 'file': image_file,
'lat': lat, 'lat': lat,
'lon': lon, 'lon': lon,
'date': date
}) })
self.logger.info(f"GPS坐标和拍摄日期提取完成 - 总图片数: {total_images}, 成功提取: {successful_extractions}, 失败: {total_images - successful_extractions}") self.logger.info(
f"GPS坐标提取完成 - 总图片数: {total_images}, 成功提取: {successful_extractions}, 失败: {total_images - successful_extractions}")
return pd.DataFrame(gps_data) return pd.DataFrame(gps_data)

View File

@ -16,50 +16,15 @@ class GridDivider:
self.num_grids_width = 0 # 添加网格数量属性 self.num_grids_width = 0 # 添加网格数量属性
self.num_grids_height = 0 self.num_grids_height = 0
def adjust_grid_size(self, points_df):
"""动态调整网格大小
Args:
points_df: 包含GPS点的DataFrame
Returns:
tuple: (grids, translations, grid_points, final_grid_size)
"""
self.logger.info(f"开始动态调整网格大小,初始大小: {self.grid_size}")
while True:
# 使用当前grid_size划分网格
grids, translations = self.divide_grids(points_df)
grid_points, multiple_grid_points = self.assign_to_grids(points_df, grids)
# 检查每个网格中的点数
max_points = 0
for grid_id, points in grid_points.items():
max_points = max(max_points, len(points))
self.logger.info(f"当前网格大小: {self.grid_size}米, 单个网格最大点数: {max_points}")
# 如果最大点数超过1600减小网格大小
if max_points > 1600:
self.grid_size -= 100
self.logger.info(f"点数超过1500减小网格大小至: {self.grid_size}")
if self.grid_size < 500: # 设置一个最小网格大小限制
self.logger.warning("网格大小已达到最小值500米停止调整")
break
else:
self.logger.info(f"找到合适的网格大小: {self.grid_size}")
break
return grids
def adjust_grid_size_and_overlap(self, points_df): def adjust_grid_size_and_overlap(self, points_df):
"""动态调整网格重叠率""" """动态调整网格重叠率"""
grids = self.adjust_grid_size(points_df) grids = self.adjust_grid_size(points_df)
self.logger.info(f"开始动态调整网格重叠率,初始重叠率: {self.overlap}") self.logger.info(f"开始动态调整网格重叠率,初始重叠率: {self.overlap}")
while True: while True:
# 使用调整好的网格大小划分网格 # 使用调整好的网格大小划分网格
grids, translations = self.divide_grids(points_df) grids = self.divide_grids(points_df)
grid_points, multiple_grid_points = self.assign_to_grids(points_df, grids) grid_points, multiple_grid_points = self.assign_to_grids(
points_df, grids)
if len(grids) == 1: if len(grids) == 1:
self.logger.info(f"网格数量为1跳过重叠率调整") self.logger.info(f"网格数量为1跳过重叠率调整")
@ -68,16 +33,52 @@ class GridDivider:
self.overlap += 0.02 self.overlap += 0.02
self.logger.info(f"重叠率增加到: {self.overlap}") self.logger.info(f"重叠率增加到: {self.overlap}")
else: else:
self.logger.info(f"找到合适的重叠率: {self.overlap}, 有{multiple_grid_points}个点被分配到多个网格") self.logger.info(
f"找到合适的重叠率: {self.overlap}, 有{multiple_grid_points}个点被分配到多个网格")
break break
return grids, translations, grid_points return grids, grid_points
def adjust_grid_size(self, points_df):
"""动态调整网格大小
Args:
points_df: 包含GPS点的DataFrame
Returns:
tuple: grids
"""
self.logger.info(f"开始动态调整网格大小,初始大小: {self.grid_size}")
while True:
# 使用当前grid_size划分网格
grids = self.divide_grids(points_df)
grid_points, multiple_grid_points = self.assign_to_grids(
points_df, grids)
# 检查每个网格中的点数
max_points = 0
for grid_id, points in grid_points.items():
max_points = max(max_points, len(points))
self.logger.info(
f"当前网格大小: {self.grid_size}米, 单个网格最大点数: {max_points}")
# 如果最大点数超过2000减小网格大小
if max_points > 2000:
self.grid_size -= 100
self.logger.info(f"点数超过2000减小网格大小至: {self.grid_size}")
if self.grid_size < 500: # 设置一个最小网格大小限制
self.logger.warning("网格大小已达到最小值500米停止调整")
break
else:
self.logger.info(f"找到合适的网格大小: {self.grid_size}")
break
return grids
def divide_grids(self, points_df): def divide_grids(self, points_df):
"""计算边界框并划分网格 """计算边界框并划分网格
Returns: Returns:
tuple: (grids, translations) tuple: grids 网格边界列表
- grids: 网格边界列表
- translations: 网格平移量字典
""" """
self.logger.info("开始划分网格") self.logger.info("开始划分网格")
@ -91,12 +92,15 @@ class GridDivider:
self.logger.info(f"区域宽度: {width:.2f}米, 高度: {height:.2f}") self.logger.info(f"区域宽度: {width:.2f}米, 高度: {height:.2f}")
# 精细调整网格的长宽避免出现2*grid_size-1的情况的影响 # 精细调整网格的长宽避免出现2*grid_size-1的情况的影响
grid_size_lt = [self.grid_size -200, self.grid_size -100, self.grid_size , self.grid_size +100, self.grid_size +200] grid_size_lt = [self.grid_size - 200, self.grid_size - 100,
self.grid_size, self.grid_size + 100, self.grid_size + 200]
width_modulus_lt = [width % grid_size for grid_size in grid_size_lt] width_modulus_lt = [width % grid_size for grid_size in grid_size_lt]
grid_width = grid_size_lt[width_modulus_lt.index(min(width_modulus_lt))] grid_width = grid_size_lt[width_modulus_lt.index(
min(width_modulus_lt))]
height_modulus_lt = [height % grid_size for grid_size in grid_size_lt] height_modulus_lt = [height % grid_size for grid_size in grid_size_lt]
grid_height = grid_size_lt[height_modulus_lt.index(min(height_modulus_lt))] grid_height = grid_size_lt[height_modulus_lt.index(
min(height_modulus_lt))]
self.logger.info(f"网格宽度: {grid_width:.2f}米, 网格高度: {grid_height:.2f}") self.logger.info(f"网格宽度: {grid_width:.2f}米, 网格高度: {grid_height:.2f}")
# 计算需要划分的网格数量 # 计算需要划分的网格数量
@ -108,18 +112,19 @@ class GridDivider:
lon_step = (max_lon - min_lon) / self.num_grids_width lon_step = (max_lon - min_lon) / self.num_grids_width
grids = [] grids = []
grid_translations = {} # 存储每个网格相对于第一个网格的平移量
# 先创建所有网格 # 先创建所有网格
for i in range(self.num_grids_height): for i in range(self.num_grids_height):
for j in range(self.num_grids_width): for j in range(self.num_grids_width):
grid_min_lat = min_lat + i * lat_step - self.overlap * lat_step grid_min_lat = min_lat + i * lat_step - self.overlap * lat_step
grid_max_lat = min_lat + (i + 1) * lat_step + self.overlap * lat_step grid_max_lat = min_lat + \
(i + 1) * lat_step + self.overlap * lat_step
grid_min_lon = min_lon + j * lon_step - self.overlap * lon_step grid_min_lon = min_lon + j * lon_step - self.overlap * lon_step
grid_max_lon = min_lon + (j + 1) * lon_step + self.overlap * lon_step grid_max_lon = min_lon + \
(j + 1) * lon_step + self.overlap * lon_step
grid_id = (i, j) # 使用(i,j)作为网格标识i代表行j代表列 grid_bounds = (grid_min_lat, grid_max_lat,
grid_bounds = (grid_min_lat, grid_max_lat, grid_min_lon, grid_max_lon) grid_min_lon, grid_max_lon)
grids.append(grid_bounds) grids.append(grid_bounds)
self.logger.debug( self.logger.debug(
@ -127,26 +132,10 @@ class GridDivider:
f"经度[{grid_min_lon:.6f}, {grid_max_lon:.6f}]" f"经度[{grid_min_lon:.6f}, {grid_max_lon:.6f}]"
) )
# 计算每个网格相对于第一个网格的平移量
reference_grid = grids[0]
for i in range(self.num_grids_height):
for j in range(self.num_grids_width):
grid_id = (i, j)
grid_idx = i * self.num_grids_width + j
if grid_idx == 0: # 参考网格
grid_translations[grid_id] = (0, 0)
else:
translation = self.calculate_grid_translation(reference_grid, grids[grid_idx])
grid_translations[grid_id] = translation
self.logger.debug(
f"网格[{i},{j}]相对于参考网格的平移量: x={translation[0]:.2f}m, y={translation[1]:.2f}m"
)
self.logger.info( self.logger.info(
f"成功划分为 {len(grids)} 个网格 ({self.num_grids_width}x{self.num_grids_height})") f"成功划分为 {len(grids)} 个网格 ({self.num_grids_width}x{self.num_grids_height})")
return grids, grid_translations return grids
def assign_to_grids(self, points_df, grids): def assign_to_grids(self, points_df, grids):
"""将点分配到对应网格""" """将点分配到对应网格"""
@ -205,7 +194,8 @@ class GridDivider:
# 计算网格的实际长度和宽度(米) # 计算网格的实际长度和宽度(米)
width = geodesic((min_lat, min_lon), (min_lat, max_lon)).meters width = geodesic((min_lat, min_lon), (min_lat, max_lon)).meters
height = geodesic((min_lat, min_lon), (max_lat, min_lon)).meters height = geodesic((min_lat, min_lon),
(max_lat, min_lon)).meters
plt.plot([min_lon, max_lon, max_lon, min_lon, min_lon], plt.plot([min_lon, max_lon, max_lon, min_lon, min_lon],
[min_lat, min_lat, max_lat, max_lat, min_lat], [min_lat, min_lat, max_lat, max_lat, min_lat],

View File

@ -1,134 +1,22 @@
import os import os
import time import time
import logging import logging
import subprocess
from typing import Dict, Tuple from typing import Dict, Tuple
import pandas as pd import pandas as pd
import numpy as np import numpy as np
from osgeo import gdal from osgeo import gdal
import docker
class NotOverlapError(Exception):
"""图像重叠度不足异常"""
pass
class ODMProcessMonitor: class ODMProcessMonitor:
"""ODM处理监控器""" """ODM处理监控器"""
def __init__(self, output_dir: str, mode: str = "快拼模式"): def __init__(self, output_dir: str, mode: str = "三维模式"):
self.output_dir = output_dir self.output_dir = output_dir
self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor') self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor')
self.mode = mode self.mode = mode
def _check_success(self, grid_dir: str) -> bool: def run_odm_with_monitor(self, grid_dir: str, grid_id: tuple) -> Tuple[bool, str]:
"""检查ODM是否执行成功
检查项目:
1. 必要的文件夹和文件是否存在
2. 产品文件是否有效
"""
project_dir = os.path.join(grid_dir, 'project')
# 根据不同模式检查不同的产品
if self.mode == "快拼模式":
# 只检查正射影像
# if not self._check_orthophoto(project_dir):
# return False
pass
elif self.mode == "三维模式":
# 检查点云和实景三维
if not all([
os.path.exists(os.path.join(
project_dir, 'odm_georeferencing', 'odm_georeferenced_model.laz')),
os.path.exists(os.path.join(
project_dir, 'odm_texturing', 'odm_textured_model_geo.obj'))
]):
self.logger.error("点云或实景三维文件夹未生成")
return False
# TODO: 添加点云和实景三维的质量检查
elif self.mode == "重建模式":
# 检查所有产品
if not all([
os.path.exists(os.path.join(
project_dir, 'odm_georeferencing', 'odm_georeferenced_model.laz')),
os.path.exists(os.path.join(
project_dir, 'odm_texturing', 'odm_textured_model_geo.obj'))
]):
self.logger.error("部分必要的文件夹未生成")
return False
# 检查正射影像
# if not self._check_orthophoto(project_dir):
# return False
# TODO: 添加点云和实景三维的质量检查
return True
# TODO 正射影像怎么检查最好
def _check_orthophoto(self, project_dir: str) -> bool:
"""检查正射影像的质量"""
ortho_path = os.path.join(
project_dir, 'odm_orthophoto', 'odm_orthophoto.original.tif')
if not os.path.exists(ortho_path):
self.logger.error("正射影像文件未生成")
return False
# 检查文件大小
file_size_mb = os.path.getsize(ortho_path) / (1024 * 1024) # 转换为MB
if file_size_mb < 1:
self.logger.error(f"正射影像文件过小: {file_size_mb:.2f}MB")
return False
try:
# 打开影像文件
ds = gdal.Open(ortho_path)
if ds is None:
self.logger.error("无法打开正射影像文件")
return False
# 读取第一个波段
band = ds.GetRasterBand(1)
# 获取统计信息
stats = band.GetStatistics(False, True)
if stats is None:
self.logger.error("无法获取影像统计信息")
return False
min_val, max_val, mean, std = stats
# 计算空值比例
no_data_value = band.GetNoDataValue()
array = band.ReadAsArray()
if no_data_value is not None:
no_data_ratio = np.sum(array == no_data_value) / array.size
else:
no_data_ratio = 0
# 检查空值比例是否过高超过50%
if no_data_ratio > 0.5:
self.logger.error(f"正射影像空值比例过高: {no_data_ratio:.2%}")
return False
# 检查影像是否全黑或全白
if max_val - min_val < 1:
self.logger.error("正射影像可能无效:像素值范围过小")
return False
ds = None # 关闭数据集
return True
except Exception as e:
self.logger.error(f"检查正射影像时发生错误: {str(e)}")
return False
def run_odm_with_monitor(self, grid_dir: str, grid_id: tuple, produce_dem: bool = False, accuracy='medium') -> Tuple[bool, str]:
"""运行ODM命令""" """运行ODM命令"""
self.logger.info(f"开始处理网格 ({grid_id[0]},{grid_id[1]})") self.logger.info(f"开始处理网格 ({grid_id[0]},{grid_id[1]})")
success = False success = False
@ -136,21 +24,21 @@ class ODMProcessMonitor:
max_retries = 3 max_retries = 3
current_try = 0 current_try = 0
# 根据模式设置是否使用lowest quality # 初始化 Docker 客户端
use_lowest_quality = self.mode == "快拼模式" client = docker.from_env()
while current_try < max_retries: while current_try < max_retries:
current_try += 1 current_try += 1
self.logger.info( self.logger.info(
f"{current_try} 次尝试处理网格 ({grid_id[0]},{grid_id[1]})") f"{current_try} 次尝试处理网格 ({grid_id[0]},{grid_id[1]})")
try: # 构建 Docker 容器运行参数
# 构建Docker命令 grid_dir = grid_dir[0].lower(
grid_dir = grid_dir[0].lower()+grid_dir[1:].replace('\\', '/') ) + grid_dir[1:].replace('\\', '/')
docker_command = ( volumes = {
f"docker run --gpus all -ti --rm " grid_dir: {'bind': '/datasets', 'mode': 'rw'}
f"-v {grid_dir}:/datasets " }
f"opendronemap/odm:gpu " command = (
f"--project-path /datasets project " f"--project-path /datasets project "
f"--max-concurrency 15 " f"--max-concurrency 15 "
f"--force-gps " f"--force-gps "
@ -159,117 +47,70 @@ class ODMProcessMonitor:
f"--optimize-disk-space " f"--optimize-disk-space "
f"--orthophoto-cutline " f"--orthophoto-cutline "
f"--feature-type sift " f"--feature-type sift "
# f"--orthophoto-resolution 8 " f"--orthophoto-resolution 8 "
)
if accuracy == "high":
docker_command += (
f"--feature-quality ultra "
f"--pc-quality ultra "
f"--mesh-size 3000000 "
f"--mesh-octree-depth 12 "
f"--orthophoto-resolution 2 "
) )
if produce_dem and self.mode != "快拼模式": if self.mode == "快拼模式":
docker_command += ( command += (
f"--fast-orthophoto "
f"--skip-3dmodel "
)
else: # 三维模式参数
command += (
f"--dsm " f"--dsm "
f"--dtm " f"--dtm "
) )
# 根据是否使用lowest quality添加参数 command += "--rerun-all"
if use_lowest_quality:
# docker_command += f"--feature-quality lowest "
pass
if self.mode == "快拼模式": self.logger.info(f"Docker 命令: {command}")
docker_command += (
f"--fast-orthophoto " # 运行 Docker 容器
f"--skip-3dmodel " container = client.containers.run(
image="opendronemap/odm:gpu",
command=command,
volumes=volumes,
detach=True,
remove=False,
runtime="nvidia", # 使用 GPU
) )
# elif self.mode == "三维模式": # 等待容器运行完成
# docker_command += ( exit_status = container.wait()
# f"--skip-orthophoto " if exit_status["StatusCode"] != 0:
# ) self.logger.error(
f"容器运行失败,退出状态码: {exit_status['StatusCode']}")
docker_command += "--rerun-all" # 获取容器的错误日志
self.logger.info(docker_command) error_msg = container.logs(
stderr=True).decode("utf-8").splitlines()
process = subprocess.Popen( self.logger.error("容器运行失败的详细错误日志:")
docker_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in error_msg:
logging.info(f"进程{process.pid}开始执行") self.logger.error(line)
stdout, stderr = process.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
# 关闭process防止内存堆积
process.terminate()
return_code = process.poll() # 获取进程返回码,如果返回 None 说明进程仍在运行
if return_code is None:
logging.info(f"进程{process.pid}仍在运行")
else: else:
logging.info(f"进程{process.pid}已被关闭,返回码:{return_code}.") # 获取所有日志
logs = container.logs().decode("utf-8").splitlines()
stdout_lines = stdout.strip().split('\n') # 输出最后 50 行日志
last_lines = '\n'.join( self.logger.info("容器运行完成,以下是最后 50 行日志:")
stdout_lines[-50:] if len(stdout_lines) > 10 else stdout_lines) for line in logs[-50:]:
self.logger.info(f"==========stdout==========: {last_lines}") self.logger.info(line)
if stderr:
self.logger.error(f"docker run指令执行失败")
self.logger.error(f"==========stderr==========: {stderr}")
if "error during connect" in stderr or "The system cannot find the file specified" in stderr:
error_msg = "Docker没有启动请启动Docker"
elif "user declined directory sharing" in stderr:
error_msg = "Docker无法访问目录请检查目录权限和共享设置"
else:
error_msg = "Docker运行失败需要人工排查错误"
break
else:
self.logger.info("docker run指令执行成功")
if "ODM app finished" in last_lines:
self.logger.info("ODM处理完成")
if self._check_success(grid_dir):
self.logger.info(
f"网格 ({grid_id[0]},{grid_id[1]}) 处理成功")
success = True success = True
error_msg = "" error_msg = ""
break break
else:
self.logger.error(
f"虽然ODM处理完成但是生产产品质量可能不合格需要人工检查")
raise NotOverlapError
# TODO 先写成这样,后面这三种情况可能处理不一样
elif "enough overlap" in last_lines:
raise NotOverlapError
elif "out of memory" in last_lines:
raise NotOverlapError
elif "strange values" in last_lines:
raise NotOverlapError
else:
raise NotOverlapError
except NotOverlapError: # 删除容器
if use_lowest_quality and self.mode == "快拼模式": container.remove()
self.logger.warning( time.sleep(5)
"检测到not overlap错误移除lowest quality参数后重试")
use_lowest_quality = False
time.sleep(10)
continue
else:
self.logger.error(
"即使移除lowest quality参数后仍然出现错误")
error_msg = "图像重叠度不足,需要人工检查数据集的采样间隔情况"
break
return success, error_msg return success, error_msg
def process_all_grids(self, grid_points: Dict[tuple, pd.DataFrame], produce_dem: bool, accuracy: str) -> Dict[tuple, pd.DataFrame]: def process_all_grids(self, grid_points: Dict[tuple, pd.DataFrame]) -> list:
"""处理所有网格 """处理所有网格
Returns: Returns:
Dict[tuple, pd.DataFrame]: 成功处理的网格点数据字典 Dict[tuple, pd.DataFrame]: 成功处理的网格点数据字典
""" """
self.logger.info("开始执行网格处理") self.logger.info("开始执行网格处理")
successful_grid_points = {} successful_grid_lt = []
failed_grids = [] failed_grids = []
for grid_id, points in grid_points.items(): for grid_id, points in grid_points.items():
@ -281,12 +122,10 @@ class ODMProcessMonitor:
success, error_msg = self.run_odm_with_monitor( success, error_msg = self.run_odm_with_monitor(
grid_dir=grid_dir, grid_dir=grid_dir,
grid_id=grid_id, grid_id=grid_id,
produce_dem=produce_dem,
accuracy=accuracy
) )
if success: if success:
successful_grid_points[grid_id] = points successful_grid_lt.append(grid_id)
else: else:
self.logger.error( self.logger.error(
f"网格 ({grid_id[0]},{grid_id[1]}) 处理失败: {error_msg}") f"网格 ({grid_id[0]},{grid_id[1]}) 处理失败: {error_msg}")
@ -301,7 +140,7 @@ class ODMProcessMonitor:
# 汇总处理结果 # 汇总处理结果
total_grids = len(grid_points) total_grids = len(grid_points)
failed_count = len(failed_grids) failed_count = len(failed_grids)
success_count = len(successful_grid_points) success_count = len(successful_grid_lt)
self.logger.info( self.logger.info(
f"网格处理完成。总计: {total_grids}, 成功: {success_count}, 失败: {failed_count}") f"网格处理完成。总计: {total_grids}, 成功: {success_count}, 失败: {failed_count}")
@ -311,7 +150,7 @@ class ODMProcessMonitor:
self.logger.error( self.logger.error(
f"网格 ({grid_id[0]},{grid_id[1]}): {error_msg}") f"网格 ({grid_id[0]},{grid_id[1]}): {error_msg}")
if len(successful_grid_points) == 0: if len(successful_grid_lt) == 0:
raise Exception("所有网格处理都失败,无法继续处理") raise Exception("所有网格处理都失败,无法继续处理")
return successful_grid_points return successful_grid_lt

View File

@ -65,7 +65,7 @@ class FilterVisualizer:
# 创建图形 # 创建图形
plt.rcParams['font.sans-serif']=['SimHei']#黑体 plt.rcParams['font.sans-serif']=['SimHei']#黑体
plt.rcParams['axes.unicode_minus'] = False plt.rcParams['axes.unicode_minus'] = False
plt.figure(figsize=(20, 16)) plt.figure()
# 绘制保留的点 # 绘制保留的点
plt.scatter(current_x, current_y, plt.scatter(current_x, current_y,