along/UAV_odm_merge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

大改：使用odm的merge

Browse Source
This commit is contained in:
龙澳 2024-12-30 20:41:22 +08:00
parent f6a5068350
commit 54ed939dc7
5 changed files with 160 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
filter/cluster_filter.py
View File

@ -63,7 +63,7 @@ class GPSCluster:
clustered_points: 带有聚类标签的DataFrame clustered_points: 带有聚类标签的DataFrame
返回: 返回:
聚类统计信息的字典 聚类统计信息的字典, 主类, 噪声点
""" """
main_cluster_points = sum(clustered_points["cluster"] == 1) main_cluster_points = sum(clustered_points["cluster"] == 1)
stats = { stats = {
@ -72,11 +72,4 @@ class GPSCluster:
"noise_points": sum(clustered_points["cluster"] == -1), "noise_points": sum(clustered_points["cluster"] == -1),
} }
noise_cluster = self.get_noise_cluster(clustered_points) return stats, clustered_points[clustered_points["cluster"] == 1], clustered_points[clustered_points["cluster"] == -1]
return stats
def get_main_cluster(self, clustered_points):
return clustered_points[clustered_points["cluster"] == 1]
def get_noise_cluster(self, clustered_points):
return clustered_points[clustered_points["cluster"] == -1]

3
filter/gps_filter.py
View File

@ -243,6 +243,7 @@ class GPSFilter:
f"删除孤立点: {row['file']} (邻居数: {neighbors_count[i]})") f"删除孤立点: {row['file']} (邻居数: {neighbors_count[i]})")
filtered_df = points_df[~points_df['file'].isin(isolated_points)] filtered_df = points_df[~points_df['file'].isin(isolated_points)]
removed_df = points_df[points_df['file'].isin(isolated_points)]
self.logger.info( self.logger.info(
f"孤立点过滤完成，共删除 {len(isolated_points)} 个点，剩余 {len(filtered_df)} 个点") f"孤立点过滤完成，共删除 {len(isolated_points)} 个点，剩余 {len(filtered_df)} 个点")
return filtered_df return filtered_df, removed_df

171
odm_preprocess.py
View File

@ -2,7 +2,7 @@ import os
import shutil import shutil
from datetime import timedelta from datetime import timedelta
from dataclasses import dataclass from dataclasses import dataclass
from typing import Dict from typing import Dict, Optional
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import pandas as pd import pandas as pd
@ -27,7 +27,7 @@ class PreprocessConfig:
"""预处理配置类""" """预处理配置类"""
image_dir: str image_dir: str
output_dir: str output_dir: Optional[str] = None
# 聚类过滤参数 # 聚类过滤参数
cluster_eps: float = 0.01 cluster_eps: float = 0.01
cluster_min_samples: int = 5 cluster_min_samples: int = 5
@ -45,13 +45,15 @@ class PreprocessConfig:
grid_overlap: float = 0.05 grid_overlap: float = 0.05
grid_size: float = 500 grid_size: float = 500
# 几个pipline过程是否开启 # 几个pipline过程是否开启
mode: str = "快拼模式" fast_mode: bool = False
class ImagePreprocessor: class ImagePreprocessor:
def __init__(self, config: PreprocessConfig): def __init__(self, config: PreprocessConfig):
self.config = config self.config = config
self.grandpa_dir = os.path.dirname(
os.path.dirname(self.config.image_dir))
self.config.output_dir = os.path.join(self.grandpa_dir, 'preprocess')
# 清理并重建输出目录 # 清理并重建输出目录
if os.path.exists(config.output_dir): if os.path.exists(config.output_dir):
self._clean_output_dir() self._clean_output_dir()
@ -61,7 +63,7 @@ class ImagePreprocessor:
self.logger = setup_logger(config.output_dir) self.logger = setup_logger(config.output_dir)
self.gps_points = None self.gps_points = None
self.odm_monitor = ODMProcessMonitor( self.odm_monitor = ODMProcessMonitor(
config.output_dir, mode=config.mode) config.output_dir, fast_mode=config.fast_mode)
self.visualizer = FilterVisualizer(config.output_dir) self.visualizer = FilterVisualizer(config.output_dir)
def _clean_output_dir(self): def _clean_output_dir(self):
@ -98,22 +100,21 @@ class ImagePreprocessor:
self.logger.info(f"成功提取 {len(self.gps_points)} 个GPS点") self.logger.info(f"成功提取 {len(self.gps_points)} 个GPS点")
return self.gps_points return self.gps_points
def cluster(self) -> pd.DataFrame: def cluster(self, previous_points) -> pd.DataFrame:
"""使用DBSCAN对GPS点进行聚类，只保留最大的类""" """使用DBSCAN对GPS点进行聚类，只保留最大的类"""
self.logger.info("开始聚类") self.logger.info("开始聚类")
previous_points = self.gps_points.copy()
# 创建聚类器并执行聚类 # 创建聚类器并执行聚类
clusterer = GPSCluster( clusterer = GPSCluster(
self.gps_points, output_dir=self.config.output_dir, previous_points, output_dir=self.config.output_dir,
eps=self.config.cluster_eps, min_samples=self.config.cluster_min_samples) eps=self.config.cluster_eps, min_samples=self.config.cluster_min_samples)
# 获取主要类别的点 # 获取主要类别的点
self.clustered_points = clusterer.fit() self.clustered_points = clusterer.fit()
self.gps_points = clusterer.get_main_cluster(self.clustered_points)
# 获取统计信息并记录 # 获取统计信息并记录
stats = clusterer.get_cluster_stats(self.clustered_points) stats, retained_points, removed_points = clusterer.get_cluster_stats(
self.clustered_points)
self.logger.info( self.logger.info(
f"聚类完成：主要类别包含 {stats['main_cluster_points']} 个点，" f"聚类完成：主要类别包含 {stats['main_cluster_points']} 个点，"
f"噪声点 {stats['noise_points']} 个" f"噪声点 {stats['noise_points']} 个"
@ -121,16 +122,16 @@ class ImagePreprocessor:
# 可视化聚类结果 # 可视化聚类结果
self.visualizer.visualize_filter_step( self.visualizer.visualize_filter_step(
self.gps_points, previous_points, "1-Clustering") retained_points, removed_points, "1-Clustering")
# 移动被过滤的图片
self.move_images(removed_points, "cluster")
return retained_points
return self.gps_points def filter_time_group_overlap(self, previous_points) -> pd.DataFrame:
def filter_time_group_overlap(self) -> pd.DataFrame:
"""过滤重叠的时间组""" """过滤重叠的时间组"""
self.logger.info("开始过滤重叠时间组") self.logger.info("开始过滤重叠时间组")
self.logger.info("开始过滤重叠时间组") self.logger.info("开始过滤重叠时间组")
previous_points = self.gps_points.copy()
filter = TimeGroupOverlapFilter( filter = TimeGroupOverlapFilter(
self.config.image_dir, self.config.image_dir,
@ -143,39 +144,43 @@ class ImagePreprocessor:
) )
# 更新GPS点数据，移除被删除的图像 # 更新GPS点数据，移除被删除的图像
self.gps_points = self.gps_points[~self.gps_points['file'].isin( retained_points = previous_points[~previous_points['file'].isin(
deleted_files)] deleted_files)]
self.logger.info(f"重叠时间组过滤后剩余 {len(self.gps_points)} 个GPS点") removed_points = previous_points[previous_points['file'].isin(
deleted_files)]
self.logger.info(f"重叠时间组过滤后剩余 {len(retained_points)} 个GPS点")
# 可视化过滤结果 # 可视化过滤结果
self.visualizer.visualize_filter_step( self.visualizer.visualize_filter_step(
self.gps_points, previous_points, "2-Time Group Overlap") retained_points, removed_points, "2-Time Group Overlap")
# 移动被过滤的图片
return self.gps_points self.move_images(removed_points, "time_group_overlap")
return retained_points
# TODO 过滤算法还需要更新 # TODO 过滤算法还需要更新
def filter_points(self) -> pd.DataFrame: def filter_points(self, previous_points) -> pd.DataFrame:
"""过滤GPS点""" """过滤GPS点"""
self.logger.info("开始过滤GPS点") self.logger.info("开始过滤GPS点")
filter = GPSFilter(self.config.output_dir) filter = GPSFilter(self.config.output_dir)
# 过滤孤立点 # 过滤孤立点
previous_points = self.gps_points.copy()
self.logger.info( self.logger.info(
f"开始过滤孤立点(距离阈值: {self.config.filter_distance_threshold}, " f"开始过滤孤立点(距离阈值: {self.config.filter_distance_threshold}, "
f"最小邻居数: {self.config.filter_min_neighbors})" f"最小邻居数: {self.config.filter_min_neighbors})"
) )
self.gps_points = filter.filter_isolated_points( retained_points, removed_points = filter.filter_isolated_points(
self.gps_points, previous_points,
self.config.filter_distance_threshold, self.config.filter_distance_threshold,
self.config.filter_min_neighbors, self.config.filter_min_neighbors,
) )
self.logger.info(f"孤立点过滤后剩余 {len(self.gps_points)} 个GPS点") self.logger.info(f"孤立点过滤后剩余 {len(retained_points)} 个GPS点")
# 可视化孤立点过滤结果 # 可视化孤立点过滤结果
self.visualizer.visualize_filter_step( self.visualizer.visualize_filter_step(
self.gps_points, previous_points, "3-Isolated Points") retained_points, removed_points, "3-Isolated Points")
# 移动被过滤的图片
self.move_images(removed_points, "isolated_points")
# # 过滤密集点 # # 过滤密集点
# previous_points = self.gps_points.copy() # previous_points = self.gps_points.copy()
@ -195,7 +200,7 @@ class ImagePreprocessor:
# self.visualizer.visualize_filter_step( # self.visualizer.visualize_filter_step(
# self.gps_points, previous_points, "4-Dense Points") # self.gps_points, previous_points, "4-Dense Points")
return self.gps_points return retained_points
def divide_grids(self) -> Dict[int, pd.DataFrame]: def divide_grids(self) -> Dict[int, pd.DataFrame]:
"""划分网格""" """划分网格"""
@ -212,7 +217,9 @@ class ImagePreprocessor:
# 生成image_groups.txt文件 # 生成image_groups.txt文件
try: try:
groups_file = os.path.join(self.config.output_dir, "image_groups.txt") groups_file = os.path.join(
os.path.dirname(self.config.image_dir), "image_groups.txt"
)
self.logger.info(f"开始生成分组文件: {groups_file}") self.logger.info(f"开始生成分组文件: {groups_file}")
with open(groups_file, 'w') as f: with open(groups_file, 'w') as f:
@ -229,59 +236,80 @@ class ImagePreprocessor:
self.logger.error(f"生成分组文件时发生错误: {str(e)}", exc_info=True) self.logger.error(f"生成分组文件时发生错误: {str(e)}", exc_info=True)
raise raise
return grid_points def move_images(self, removed_points: pd.DataFrame, step_name: str):
"""
将被过滤掉的图片移动到ret文件夹中
def copy_images(self, grid_points: Dict[int, pd.DataFrame]): Args:
"""复制图像到目标文件夹""" removed_points: 被过滤掉的GPS点对应的数据
self.logger.info("开始复制图像文件") step_name: 过滤步骤名称，用于创建子文件夹
self.logger.info("开始复制图像文件") """
if removed_points.empty:
return
for grid_idx, points in grid_points.items(): # 创建ret目录和对应步骤的子目录
output_dir = os.path.join( ret_dir = os.path.join(self.grandpa_dir, 'ret')
self.config.output_dir, f"grid_{grid_idx + 1}", "project", "images" os.makedirs(ret_dir, exist_ok=True)
)
os.makedirs(output_dir, exist_ok=True) self.logger.info(f"开始移动{step_name}步骤中被过滤的图片")
for point in tqdm(points, desc=f"复制网格 {grid_idx + 1} 的图像"): # 移动每张被过滤的图片
src = os.path.join(self.config.image_dir, point["file"]) for _, point in removed_points.iterrows():
dst = os.path.join(output_dir, point["file"]) src_path = os.path.join(self.config.image_dir, point['file'])
shutil.copy(src, dst) dst_path = os.path.join(ret_dir, point['file'])
self.logger.info(f"网格 {grid_idx + 1} 包含 {len(points)} 张图像")
def merge_tif(self, grid_points: Dict[int, pd.DataFrame]): try:
"""合并所有网格的影像产品""" shutil.move(src_path, dst_path)
self.logger.info("开始合并所有影像产品") except Exception as e:
merger = MergeTif(self.config.output_dir) self.logger.warning(f"移动图片 {point['file']} 时发生错误: {str(e)}")
merger.merge_all_tifs(grid_points)
def merge_obj(self, grid_points: Dict[int, pd.DataFrame]): self.logger.info(f"完成移动 {len(removed_points)} 张被{step_name}过滤的图片")
"""合并所有网格的OBJ模型"""
self.logger.info("开始合并OBJ模型")
merger = MergeObj(self.config.output_dir)
merger.merge_grid_obj(grid_points)
def merge_ply(self, grid_points: Dict[int, pd.DataFrame]): def restore_filtered_images(self):
"""合并所有网格的PLY点云""" """将ret文件夹中的图片恢复到原始图片目录"""
self.logger.info("开始合并PLY点云") try:
merger = MergePly(self.config.output_dir) # 获取ret文件夹路径
merger.merge_grid_ply(grid_points) ret_dir = os.path.join(self.grandpa_dir, 'ret')
if not os.path.exists(ret_dir):
self.logger.info("没有找到ret文件夹，跳过恢复步骤")
return
self.logger.info("开始恢复被过滤的图片")
# 获取ret文件夹中的所有图片
filtered_images = os.listdir(ret_dir)
# 将图片移回原始目录
for img in filtered_images:
src_path = os.path.join(ret_dir, img)
dst_path = os.path.join(self.config.image_dir, img)
try:
shutil.move(src_path, dst_path)
except Exception as e:
self.logger.warning(f"恢复图片 {img} 时发生错误: {str(e)}")
self.logger.info(f"成功恢复 {len(filtered_images)} 张图片")
except Exception as e:
self.logger.error(f"恢复图片过程中发生错误: {str(e)}", exc_info=True)
raise
def process(self): def process(self):
"""执行完整的预处理流程""" """执行完整的预处理流程"""
try: try:
self.extract_gps() self.extract_gps()
self.cluster() self.gps_points = self.cluster(self.gps_points)
# self.filter_time_group_overlap() # self.gps_points = self.filter_time_group_overlap(self.gps_points)
self.filter_points() self.gps_points = self.filter_points(self.gps_points)
grid_points = self.divide_grids() self.divide_grids()
self.copy_images(grid_points)
self.logger.info("预处理任务完成") self.logger.info("预处理任务完成")
self.odm_monitor.process_all_grids(grid_points) self.odm_monitor.run_odm_with_monitor(
self.merge_tif(grid_points) self.grandpa_dir, self.config.fast_mode)
self.merge_obj(grid_points)
self.merge_ply(grid_points) self.restore_filtered_images()
except Exception as e: except Exception as e:
self.logger.error(f"处理过程中发生错误: {str(e)}", exc_info=True) self.logger.error(f"处理过程中发生错误: {str(e)}", exc_info=True)
raise raise
@ -290,8 +318,7 @@ class ImagePreprocessor:
if __name__ == "__main__": if __name__ == "__main__":
# 创建配置 # 创建配置
config = PreprocessConfig( config = PreprocessConfig(
image_dir=r"E:\datasets\UAV\134\project\images", image_dir=r"G:\error_data\20241104140457\project\images",
output_dir=r"G:\ODM_output\134_test",
cluster_eps=0.01, cluster_eps=0.01,
cluster_min_samples=5, cluster_min_samples=5,
@ -307,11 +334,11 @@ if __name__ == "__main__":
filter_dense_distance_threshold=10, filter_dense_distance_threshold=10,
filter_time_threshold=timedelta(minutes=5), filter_time_threshold=timedelta(minutes=5),
grid_size=300, grid_size=1000,
grid_overlap=0.1, grid_overlap=0.05,
mode="重建模式", fast_mode=False,
) )
# 创建处理器并执行 # 创建处理器并执行

41
utils/odm_monitor.py
View File

@ -8,33 +8,31 @@ import pandas as pd
class ODMProcessMonitor: class ODMProcessMonitor:
"""ODM处理监控器""" """ODM处理监控器"""
def __init__(self, output_dir: str, mode: str = "快拼模式"): def __init__(self, output_dir: str, fast_mode: bool):
self.output_dir = output_dir self.output_dir = output_dir
self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor') self.logger = logging.getLogger('UAV_Preprocess.ODMMonitor')
self.mode = mode self.fast_mode = fast_mode
def _check_success(self, grid_dir: str) -> bool: def _check_success(self, grid_dir: str) -> bool:
"""检查ODM是否执行成功""" """检查ODM是否执行成功"""
success_markers = ['odm_orthophoto', 'odm_georeferencing'] success_markers = ['odm_orthophoto', 'odm_georeferencing']
if self.mode != "快拼模式": if not self.fast_mode:
success_markers.append('odm_texturing') success_markers.append('odm_texturing')
return all(os.path.exists(os.path.join(grid_dir, 'project', marker)) for marker in success_markers) return all(os.path.exists(os.path.join(grid_dir, 'project', marker)) for marker in success_markers)
def run_odm_with_monitor(self, grid_dir: str, grid_idx: int, fast_mode: bool = True) -> Tuple[bool, str]: def run_odm_with_monitor(self, project_dir: str, fast_mode: bool = True) -> Tuple[bool, str]:
"""运行ODM命令""" """运行ODM命令"""
self.logger.info(f"开始处理网格 {grid_idx + 1}")
# 构建Docker命令 # 构建Docker命令
grid_dir = grid_dir[0].lower()+grid_dir[1:].replace('\\', '/')
docker_command = ( docker_command = (
f"docker run --gpus all -ti --rm " f"docker run --gpus all -ti --rm "
f"-v {grid_dir}:/datasets " f"-v {project_dir}:/datasets "
f"opendronemap/odm:gpu " f"opendronemap/odm:gpu "
f"--project-path /datasets project " f"--project-path /datasets project "
f"--max-concurrency 10 " f"--max-concurrency 15 "
f"--force-gps " f"--force-gps "
f"--feature-quality lowest " f"--feature-quality lowest "
f"--orthophoto-resolution 10 " f"--orthophoto-resolution 10 "
f"--split-overlap 0 "
) )
if fast_mode: if fast_mode:
@ -53,26 +51,7 @@ class ODMProcessMonitor:
self.logger.info(f"==========stdout==========: {stdout}") self.logger.info(f"==========stdout==========: {stdout}")
self.logger.error(f"==========stderr==========: {stderr}") self.logger.error(f"==========stderr==========: {stderr}")
# 检查执行结果 # 检查执行结果
if self._check_success(grid_dir): if self._check_success(image_dir):
self.logger.info(f"网格 {grid_idx + 1} 处理成功") self.logger.info(f"处理成功")
return True, ""
else: else:
self.logger.error(f"网格 {grid_idx + 1} 处理失败") self.logger.error(f"处理失败")
return False, f"网格 {grid_idx + 1} 处理失败"
def process_all_grids(self, grid_points: Dict[int, pd.DataFrame]):
"""处理所有网格"""
self.logger.info("开始执行网格处理")
for grid_idx in grid_points.keys():
grid_dir = os.path.join(
self.output_dir, f'grid_{grid_idx + 1}'
)
success, error_msg = self.run_odm_with_monitor(
grid_dir=grid_dir,
grid_idx=grid_idx,
fast_mode=(self.mode == "快拼模式")
)
if not success:
raise Exception(f"网格 {grid_idx + 1} 处理失败: {error_msg}")

46
utils/visualizer.py
View File

@ -19,56 +19,53 @@ class FilterVisualizer:
self.logger = logging.getLogger('UAV_Preprocess.Visualizer') self.logger = logging.getLogger('UAV_Preprocess.Visualizer')
def visualize_filter_step(self, def visualize_filter_step(self,
current_points: pd.DataFrame, retained_points: pd.DataFrame,
previous_points: pd.DataFrame, filtered_points: pd.DataFrame,
step_name: str, step_name: str,
save_name: Optional[str] = None): save_name: Optional[str] = None):
""" """
可视化单个过滤步骤的结果 可视化单个过滤步骤的结果
Args: Args:
current_points: 当前步骤后的点 retained_points: 留下的点
previous_points: 上一步骤的点 filtered_points: 过滤掉的点
step_name: 步骤名称 step_name: 步骤名称
save_name: 保存文件名，默认为step_name save_name: 保存文件名，默认为step_name
""" """
total_points_len = len(retained_points) + len(filtered_points)
self.logger.info(f"开始生成{step_name}的可视化结果") self.logger.info(f"开始生成{step_name}的可视化结果")
# 找出被过滤掉的点
filtered_files = set(previous_points['file']) - set(current_points['file'])
filtered_points = previous_points[previous_points['file'].isin(filtered_files)]
# 创建图形 # 创建图形
plt.figure(figsize=(20, 16)) plt.figure(figsize=(20, 16))
# 绘制保留的点 # 绘制保留的点
plt.scatter(current_points['lon'], current_points['lat'], plt.scatter(retained_points['lon'], retained_points['lat'],
color='blue', label='Retained Points', color='blue', label='Retained Points',
alpha=0.6, s=50) alpha=0.6, s=50)
# 绘制被过滤的点 # 绘制被过滤的点
if not filtered_points.empty: if not filtered_points.empty:
plt.scatter(filtered_points['lon'], filtered_points['lat'], plt.scatter(filtered_points['lon'], filtered_points['lat'],
color='red', marker='x', label='Filtered Points', color='red', marker='x', label='Filtered Points',
alpha=0.6, s=100) alpha=0.6, s=100)
# 设置图形属性 # 设置图形属性
plt.title(f"GPS Points After {step_name}\n" plt.title(f"GPS Points After {step_name}\n"
f"(Filtered: {len(filtered_points)}, Retained: {len(current_points)})", f"(Filtered: {len(filtered_points)}, Retained: {len(retained_points)})",
fontsize=14) fontsize=14)
plt.xlabel("Longitude", fontsize=12) plt.xlabel("Longitude", fontsize=12)
plt.ylabel("Latitude", fontsize=12) plt.ylabel("Latitude", fontsize=12)
plt.grid(True) plt.grid(True)
# 添加统计信息 # 添加统计信息
stats_text = ( stats_text = (
f"Original Points: {len(previous_points)}\n" f"Original Points: {total_points_len}\n"
f"Filtered Points: {len(filtered_points)}\n" f"Filtered Points: {len(filtered_points)}\n"
f"Remaining Points: {len(current_points)}\n" f"Remaining Points: {len(retained_points)}\n"
f"Filter Rate: {len(filtered_points)/len(previous_points)*100:.1f}%" f"Filter Rate: {len(filtered_points)/total_points_len*100:.1f}%"
) )
plt.figtext(0.02, 0.02, stats_text, fontsize=10, plt.figtext(0.02, 0.02, stats_text, fontsize=10,
bbox=dict(facecolor='white', alpha=0.8)) bbox=dict(facecolor='white', alpha=0.8))
# 添加图例 # 添加图例
plt.legend(loc='upper right', fontsize=10) plt.legend(loc='upper right', fontsize=10)
@ -78,15 +75,16 @@ class FilterVisualizer:
# 保存图形 # 保存图形
save_name = save_name or step_name.lower().replace(' ', '_') save_name = save_name or step_name.lower().replace(' ', '_')
save_path = os.path.join(self.output_dir, 'filter_imgs', f'filter_{save_name}.png') save_path = os.path.join(
self.output_dir, 'filter_imgs_visual', f'filter_{save_name}.png')
plt.savefig(save_path, dpi=300, bbox_inches='tight') plt.savefig(save_path, dpi=300, bbox_inches='tight')
plt.close() plt.close()
self.logger.info( self.logger.info(
f"{step_name}过滤可视化结果已保存至 {save_path}\n" f"{step_name}过滤可视化结果已保存至 {save_path}\n"
f"过滤掉 {len(filtered_points)} 个点，" f"过滤掉 {len(filtered_points)} 个点，"
f"保留 {len(current_points)} 个点，" f"保留 {len(retained_points)} 个点，"
f"过滤率 {len(filtered_points)/len(previous_points)*100:.1f}%" f"过滤率 {len(filtered_points)/total_points_len*100:.1f}%"
) )