diff --git a/odm_preprocess.py b/odm_preprocess.py index c128261..1b1a076 100644 --- a/odm_preprocess.py +++ b/odm_preprocess.py @@ -223,7 +223,10 @@ class ImagePreprocessor: for grid_id, points in grid_points.items(): output_dir = os.path.join( - self.config.output_dir, f"grid_{grid_id[0]}_{grid_id[1]}", "project", "images" + self.config.output_dir, + f"grid_{grid_id[0]}_{grid_id[1]}", + "project", + "images" ) os.makedirs(output_dir, exist_ok=True) @@ -232,8 +235,7 @@ class ImagePreprocessor: src = os.path.join(self.config.image_dir, point["file"]) dst = os.path.join(output_dir, point["file"]) shutil.copy(src, dst) - self.logger.info( - f"网格 ({grid_id[0]},{grid_id[1]}) 包含 {len(points)} 张图像") + self.logger.info(f"网格 ({grid_id[0]},{grid_id[1]}) 包含 {len(points)} 张图像") def merge_tif(self, grid_points: Dict[tuple, pd.DataFrame]): """合并所有网格的影像产品""" diff --git a/odm_preprocess_fast.py b/odm_preprocess_fast.py deleted file mode 100644 index 59e3b59..0000000 --- a/odm_preprocess_fast.py +++ /dev/null @@ -1,305 +0,0 @@ -import os -import shutil -from datetime import timedelta -from dataclasses import dataclass -from typing import Dict, Tuple - -import matplotlib.pyplot as plt -import pandas as pd -from tqdm import tqdm - -from filter.cluster_filter import GPSCluster -from filter.time_group_overlap_filter import TimeGroupOverlapFilter -from filter.gps_filter import GPSFilter -from utils.odm_monitor import ODMProcessMonitor -from utils.gps_extractor import GPSExtractor -from utils.grid_divider import GridDivider -from utils.logger import setup_logger -from utils.visualizer import FilterVisualizer -from post_pro.merge_tif import MergeTif -from tools.test_docker_run import run_docker_command -from post_pro.merge_obj import MergeObj -from post_pro.merge_laz import MergePly - - -@dataclass -class PreprocessConfig: - """预处理配置类""" - - image_dir: str - output_dir: str - # 聚类过滤参数 - cluster_eps: float = 0.01 - cluster_min_samples: int = 5 - # 时间组重叠过滤参数 - time_group_overlap_threshold: float = 0.7 - time_group_interval: timedelta = timedelta(minutes=5) - # 孤立点过滤参数 - filter_distance_threshold: float = 0.001 # 经纬度距离 - filter_min_neighbors: int = 6 - # 密集点过滤参数 - filter_grid_size: float = 0.001 - filter_dense_distance_threshold: float = 10 # 普通距离,单位:米 - filter_time_threshold: timedelta = timedelta(minutes=5) - # 网格划分参数 - grid_overlap: float = 0.05 - grid_size: float = 500 - # 几个pipline过程是否开启 - mode: str = "快拼模式" - - -class ImagePreprocessor: - def __init__(self, config: PreprocessConfig): - self.config = config - - # # 清理并重建输出目录 - # if os.path.exists(config.output_dir): - # self._clean_output_dir() - # self._setup_output_dirs() - - # 初始化其他组件 - self.logger = setup_logger(config.output_dir) - self.gps_points = None - self.odm_monitor = ODMProcessMonitor( - config.output_dir, mode=config.mode) - self.visualizer = FilterVisualizer(config.output_dir) - - def _clean_output_dir(self): - """清理输出目录""" - try: - shutil.rmtree(self.config.output_dir) - print(f"已清理输出目录: {self.config.output_dir}") - except Exception as e: - print(f"清理输出目录时发生错误: {str(e)}") - raise - - def _setup_output_dirs(self): - """创建必要的输出目录结构""" - try: - # 创建主输出目录 - os.makedirs(self.config.output_dir) - - # 创建过滤图像保存目录 - os.makedirs(os.path.join(self.config.output_dir, 'filter_imgs')) - - # 创建日志目录 - os.makedirs(os.path.join(self.config.output_dir, 'logs')) - - print(f"已创建输出目录结构: {self.config.output_dir}") - except Exception as e: - print(f"创建输出目录时发生错误: {str(e)}") - raise - - def extract_gps(self) -> pd.DataFrame: - """提取GPS数据""" - self.logger.info("开始提取GPS数据") - extractor = GPSExtractor(self.config.image_dir) - self.gps_points = extractor.extract_all_gps() - self.logger.info(f"成功提取 {len(self.gps_points)} 个GPS点") - return self.gps_points - - def cluster(self) -> pd.DataFrame: - """使用DBSCAN对GPS点进行聚类,只保留最大的类""" - self.logger.info("开始聚类") - previous_points = self.gps_points.copy() - - # 创建聚类器并执行聚类 - clusterer = GPSCluster( - self.gps_points, output_dir=self.config.output_dir, - eps=self.config.cluster_eps, min_samples=self.config.cluster_min_samples) - - # 获取主要类别的点 - self.clustered_points = clusterer.fit() - self.gps_points = clusterer.get_main_cluster(self.clustered_points) - - # 获取统计信息并记录 - stats = clusterer.get_cluster_stats(self.clustered_points) - self.logger.info( - f"聚类完成:主要类别包含 {stats['main_cluster_points']} 个点," - f"噪声点 {stats['noise_points']} 个" - ) - - # 可视化聚类结果 - self.visualizer.visualize_filter_step( - self.gps_points, previous_points, "1-Clustering") - - return self.gps_points - - def filter_time_group_overlap(self) -> pd.DataFrame: - """过滤重叠的时间组""" - self.logger.info("开始过滤重叠时间组") - - self.logger.info("开始过滤重叠时间组") - previous_points = self.gps_points.copy() - - filter = TimeGroupOverlapFilter( - self.config.image_dir, - self.config.output_dir, - overlap_threshold=self.config.time_group_overlap_threshold - ) - - deleted_files = filter.filter_overlapping_groups( - time_threshold=self.config.time_group_interval - ) - - # 更新GPS点数据,移除被删除的图像 - self.gps_points = self.gps_points[~self.gps_points['file'].isin( - deleted_files)] - self.logger.info(f"重叠时间组过滤后剩余 {len(self.gps_points)} 个GPS点") - - # 可视化过滤结果 - self.visualizer.visualize_filter_step( - self.gps_points, previous_points, "2-Time Group Overlap") - - return self.gps_points - - # TODO 过滤算法还需要更新 - def filter_points(self) -> pd.DataFrame: - """过滤GPS点""" - - self.logger.info("开始过滤GPS点") - filter = GPSFilter(self.config.output_dir) - - # 过滤孤立点 - previous_points = self.gps_points.copy() - self.logger.info( - f"开始过滤孤立点(距离阈值: {self.config.filter_distance_threshold}, " - f"最小邻居数: {self.config.filter_min_neighbors})" - ) - self.gps_points = filter.filter_isolated_points( - self.gps_points, - self.config.filter_distance_threshold, - self.config.filter_min_neighbors, - ) - self.logger.info(f"孤立点过滤后剩余 {len(self.gps_points)} 个GPS点") - - # 可视化孤立点过滤结果 - self.visualizer.visualize_filter_step( - self.gps_points, previous_points, "3-Isolated Points") - - # # 过滤密集点 - # previous_points = self.gps_points.copy() - # self.logger.info( - # f"开始过滤密集点(网格大小: {self.config.filter_grid_size}, " - # f"距离阈值: {self.config.filter_dense_distance_threshold})" - # ) - # self.gps_points = filter.filter_dense_points( - # self.gps_points, - # grid_size=self.config.filter_grid_size, - # distance_threshold=self.config.filter_dense_distance_threshold, - # time_threshold=self.config.filter_time_threshold, - # ) - # self.logger.info(f"密集点过滤后剩余 {len(self.gps_points)} 个GPS点") - - # # 可视化密集点过滤结果 - # self.visualizer.visualize_filter_step( - # self.gps_points, previous_points, "4-Dense Points") - - return self.gps_points - - def divide_grids(self) -> Tuple[Dict[tuple, pd.DataFrame], Dict[tuple, tuple]]: - """划分网格 - Returns: - tuple: (grid_points, translations) - - grid_points: 网格点数据字典 - - translations: 网格平移量字典 - """ - self.logger.info(f"开始划分网格 (重叠率: {self.config.grid_overlap})") - grid_divider = GridDivider( - overlap=self.config.grid_overlap, - output_dir=self.config.output_dir - ) - grids, translations = grid_divider.divide_grids( - self.gps_points, grid_size=self.config.grid_size - ) - grid_points = grid_divider.assign_to_grids(self.gps_points, grids) - self.logger.info(f"成功划分为 {len(grid_points)} 个网格") - - return grid_points, translations - - def copy_images(self, grid_points: Dict[tuple, pd.DataFrame]): - """复制图像到目标文件夹""" - self.logger.info("开始复制图像文件") - - for grid_id, points in grid_points.items(): - output_dir = os.path.join( - self.config.output_dir, f"grid_{grid_id[0]}_{grid_id[1]}", "project", "images" - ) - - os.makedirs(output_dir, exist_ok=True) - - for point in tqdm(points, desc=f"复制网格 ({grid_id[0]},{grid_id[1]}) 的图像"): - src = os.path.join(self.config.image_dir, point["file"]) - dst = os.path.join(output_dir, point["file"]) - shutil.copy(src, dst) - self.logger.info( - f"网格 ({grid_id[0]},{grid_id[1]}) 包含 {len(points)} 张图像") - - def merge_tif(self, grid_points: Dict[tuple, pd.DataFrame]): - """合并所有网格的影像产品""" - self.logger.info("开始合并所有影像产品") - merger = MergeTif(self.config.output_dir) - merger.merge_all_tifs(grid_points) - - def merge_ply(self, grid_points: Dict[tuple, pd.DataFrame]): - """合并所有网格的PLY点云""" - self.logger.info("开始合并PLY点云") - merger = MergePly(self.config.output_dir) - merger.merge_grid_laz(grid_points) - - def merge_obj(self, grid_points: Dict[tuple, pd.DataFrame], translations: Dict[tuple, tuple]): - """合并所有网格的OBJ模型""" - self.logger.info("开始合并OBJ模型") - merger = MergeObj(self.config.output_dir) - merger.merge_grid_obj(grid_points, translations) - - def process(self): - """执行完整的预处理流程""" - try: - self.extract_gps() - self.cluster() - # self.filter_time_group_overlap() - self.filter_points() - grid_points, translations = self.divide_grids() - # self.copy_images(grid_points) - self.logger.info("预处理任务完成") - - # self.odm_monitor.process_all_grids(grid_points) - # self.merge_tif(grid_points) - # self.merge_ply(grid_points) - self.merge_obj(grid_points, translations) - except Exception as e: - self.logger.error(f"处理过程中发生错误: {str(e)}", exc_info=True) - raise - - -if __name__ == "__main__": - # 创建配置 - config = PreprocessConfig( - image_dir=r"E:\datasets\UAV\1009\project\images", - output_dir=r"G:\ODM_output\1009", - - cluster_eps=0.01, - cluster_min_samples=5, - - # 添加时间组重叠过滤参数 - time_group_overlap_threshold=0.7, - time_group_interval=timedelta(minutes=5), - - filter_distance_threshold=0.001, - filter_min_neighbors=6, - - filter_grid_size=0.001, - filter_dense_distance_threshold=10, - filter_time_threshold=timedelta(minutes=5), - - grid_size=500, - grid_overlap=0.1, - - - mode="重建模式", - ) - - # 创建处理器并执行 - processor = ImagePreprocessor(config) - processor.process() diff --git a/post_pro/merge_laz.py b/post_pro/merge_laz.py index d934b5f..c101614 100644 --- a/post_pro/merge_laz.py +++ b/post_pro/merge_laz.py @@ -11,32 +11,30 @@ class MergePly: self.output_dir = output_dir self.logger = logging.getLogger('UAV_Preprocess.MergePly') - def merge_grid_laz(self, grid_points: Dict[tuple, list]): - """合并所有网格的点云""" - self.logger.info("开始合并所有网格的laz点云") - - if len(grid_points) < 2: + def merge_grid_laz(self, grid_points: Dict[tuple, pd.DataFrame]): + """合并所有网格的点云数据""" + if len(grid_points) == 1: self.logger.info("只有一个网格,无需合并") return try: - laz_lt = [] + # 获取所有点云文件路径 + laz_files = [] for grid_id, points in grid_points.items(): - grid_laz = os.path.join( + laz_path = os.path.join( self.output_dir, f"grid_{grid_id[0]}_{grid_id[1]}", "project", "odm_georeferencing", "odm_georeferenced_model.laz" ) - - if not os.path.exists(grid_laz): - self.logger.warning(f"网格 ({grid_id[0]},{grid_id[1]}) 的laz文件不存在: {grid_laz}") - continue - laz_lt.append(grid_laz) + if os.path.exists(laz_path): + laz_files.append(laz_path) + else: + self.logger.warning(f"网格 ({grid_id[0]},{grid_id[1]}) 的点云文件不存在") kwargs = { - 'all_inputs': " ".join(laz_lt), + 'all_inputs': " ".join(laz_files), 'output': os.path.join(self.output_dir, 'merged_pointcloud.laz') } diff --git a/post_pro/merge_obj.py b/post_pro/merge_obj.py index a3b10f1..a192605 100644 --- a/post_pro/merge_obj.py +++ b/post_pro/merge_obj.py @@ -104,7 +104,7 @@ class MergeObj: """平移顶点""" return [[v[0] + translation[0], v[1] + translation[1], v[2] + translation[2]] for v in vertices] - def merge_two_objs(self, obj1_path: str, obj2_path: str, output_path: str, translation): + def merge_two_objs(self, obj1_path: str, obj2_path: str, output_path: str, translation, grid_id1: tuple, grid_id2: tuple): """合并两个OBJ文件""" try: self.logger.info(f"开始合并OBJ模型:\n输入1: {obj1_path}\n输入2: {obj2_path}") @@ -124,8 +124,8 @@ class MergeObj: materials2 = self.read_mtl(mtl2_path) # 创建材质名称映射(使用与MTL文件相同的命名格式) - material_map1 = {old_name: f"material_0_0_{old_name}" for old_name in materials1.keys()} - material_map2 = {old_name: f"material_0_1_{old_name}" for old_name in materials2.keys()} + material_map1 = {old_name: f"material_{grid_id1[0]}_{grid_id1[1]}_{old_name}" for old_name in materials1.keys()} + material_map2 = {old_name: f"material_{grid_id2[0]}_{grid_id2[1]}_{old_name}" for old_name in materials2.keys()} # 平移第二个模型的顶点 vertices2_translated = self.translate_vertices(vertices2, translation) @@ -249,6 +249,10 @@ class MergeObj: def merge_grid_obj(self, grid_points: Dict[tuple, pd.DataFrame], translations: Dict[tuple, tuple]): """合并所有网格的OBJ模型""" + if len(grid_points) == 1: + self.logger.info("只有一个网格,无需合并") + return + try: # 创建输出目录 output_model_dir = os.path.join(self.output_dir, "merged_model") @@ -321,7 +325,7 @@ class MergeObj: f"merged_model_{reference_id[0]}_{reference_id[1]}_{grid_id[0]}_{grid_id[1]}.obj" ) - self.merge_two_objs(merged_obj, files['obj'], output_obj, translation) + self.merge_two_objs(merged_obj, files['obj'], output_obj, translation, reference_id, grid_id) merged_obj = output_obj # 最终结果 diff --git a/requirements.txt b/requirements.txt index 9bc2977..ff696df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,3 @@ piexif geopy psutil docker>=6.1.3 -open3d diff --git a/utils/grid_divider.py b/utils/grid_divider.py index 2ce817c..c871b57 100644 --- a/utils/grid_divider.py +++ b/utils/grid_divider.py @@ -52,12 +52,12 @@ class GridDivider: grid_min_lon = min_lon + j * lon_step - self.overlap * lon_step grid_max_lon = min_lon + (j + 1) * lon_step + self.overlap * lon_step - grid_id = (j, i) # 使用(width_idx, height_idx)元组作为网格标识 + grid_id = (i, j) # 使用(i,j)作为网格标识,i代表行,j代表列 grid_bounds = (grid_min_lat, grid_max_lat, grid_min_lon, grid_max_lon) grids.append(grid_bounds) self.logger.debug( - f"网格[{j},{i}]: 纬度[{grid_min_lat:.6f}, {grid_max_lat:.6f}], " + f"网格[{i},{j}]: 纬度[{grid_min_lat:.6f}, {grid_max_lat:.6f}], " f"经度[{grid_min_lon:.6f}, {grid_max_lon:.6f}]" ) @@ -65,7 +65,7 @@ class GridDivider: reference_grid = grids[0] for i in range(self.num_grids_height): for j in range(self.num_grids_width): - grid_id = (j, i) + grid_id = (i, j) grid_idx = i * self.num_grids_width + j if grid_idx == 0: # 参考网格 grid_translations[grid_id] = (0, 0) @@ -73,7 +73,7 @@ class GridDivider: translation = self.calculate_grid_translation(reference_grid, grids[grid_idx]) grid_translations[grid_id] = translation self.logger.debug( - f"网格[{j},{i}]相对于参考网格的平移量: x={translation[0]:.2f}m, y={translation[1]:.2f}m" + f"网格[{i},{j}]相对于参考网格的平移量: x={translation[0]:.2f}m, y={translation[1]:.2f}m" ) self.logger.info( @@ -95,7 +95,7 @@ class GridDivider: for i in range(self.num_grids_height): for j in range(self.num_grids_width): - grid_points[(j, i)] = [] # 使用(width_idx, height_idx)元组 + grid_points[(i, j)] = [] # 使用(i,j)元组 for _, point in points_df.iterrows(): point_assigned = False @@ -105,7 +105,7 @@ class GridDivider: min_lat, max_lat, min_lon, max_lon = grids[grid_idx] if min_lat <= point['lat'] <= max_lat and min_lon <= point['lon'] <= max_lon: - grid_points[(j, i)].append(point.to_dict()) + grid_points[(i, j)].append(point.to_dict()) if point_assigned: multiple_grid_points += 1 else: @@ -146,7 +146,7 @@ class GridDivider: # 在网格中心添加网格编号 center_lon = (min_lon + max_lon) / 2 center_lat = (min_lat + max_lat) / 2 - plt.text(center_lon, center_lat, f"({j},{i})", # 显示(width_idx, height_idx) + plt.text(center_lon, center_lat, f"({i},{j})", # 显示(i,j) horizontalalignment='center', verticalalignment='center') plt.title('网格划分与GPS点分布图')