commit 22a041db6ccd061625f1b26db349e5a014d25358 Author: long.ao Date: Tue Dec 17 22:09:47 2024 +0800 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..2ed8f52 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# ODM_Pro +无人机三维重建 diff --git a/odm_preprocess.py b/odm_preprocess.py new file mode 100644 index 0000000..bdc56c7 --- /dev/null +++ b/odm_preprocess.py @@ -0,0 +1,163 @@ +from gps_extractor import GPSExtractor +from gps_filter import GPSFilter +from grid_divider import GridDivider +from logger import setup_logger +import os +import pandas as pd +import shutil +import matplotlib.pyplot as plt +from typing import List, Dict, Optional +from dataclasses import dataclass +from tqdm import tqdm + + +@dataclass +class PreprocessConfig: + """预处理配置类""" + image_dir: str + output_dir: str + filter_grid_size: float = 0.001 + filter_dense_distance_threshold: float = 10 + filter_distance_threshold: float = 0.001 + filter_min_neighbors: int = 6 + grid_overlap: float = 0.05 + enable_filter: bool = True + enable_grid_division: bool = True + enable_visualization: bool = True + + +class ImagePreprocessor: + def __init__(self, config: PreprocessConfig): + self.config = config + self.logger = setup_logger(config.output_dir) + self.gps_points = [] + + def extract_gps(self) -> List[Dict]: + """提取GPS数据""" + self.logger.info("开始提取GPS数据") + extractor = GPSExtractor(self.config.image_dir) + self.gps_points = extractor.extract_all_gps() + self.logger.info(f"成功提取 {len(self.gps_points)} 个GPS点") + return self.gps_points + + def filter_points(self) -> List[Dict]: + """过滤GPS点""" + if not self.config.enable_filter: + return self.gps_points + + self.logger.info("开始过滤GPS点") + filter = GPSFilter(self.config.output_dir) + + self.logger.info(f"开始过滤孤立点 (距离阈值: {self.config.filter_distance_threshold}, 最小邻居数: { + self.config.filter_min_neighbors})") + self.gps_points = filter.filter_isolated_points( + self.gps_points, + self.config.filter_distance_threshold, + self.config.filter_min_neighbors + ) + self.logger.info(f"孤立点过滤后剩余 {len(self.gps_points)} 个GPS点") + + self.logger.info(f"开始过滤密集点 (网格大小: {self.config.filter_grid_size}, 距离阈值: { + self.config.filter_dense_distance_threshold})") + self.gps_points = filter.filter_dense_points( + self.gps_points, + grid_size=self.config.filter_grid_size, + distance_threshold=self.config.filter_dense_distance_threshold + ) + self.logger.info(f"密集点过滤后剩余 {len(self.gps_points)} 个GPS点") + return self.gps_points + + def divide_grids(self) -> Dict[int, List[Dict]]: + """划分网格""" + if not self.config.enable_grid_division: + return {0: self.gps_points} # 不划分网格时,所有点放在一个网格中 + + self.logger.info(f"开始划分网格 (重叠率: {self.config.grid_overlap})") + grid_divider = GridDivider(overlap=self.config.grid_overlap) + grids = grid_divider.divide_grids(self.gps_points) + grid_points = grid_divider.assign_to_grids(self.gps_points, grids) + self.logger.info(f"成功划分为 {len(grid_points)} 个网格") + return grid_points + + def copy_images(self, grid_points: Dict[int, List[Dict]]): + """复制图像到目标文件夹""" + self.logger.info("开始复制图像文件") + os.makedirs(self.config.output_dir, exist_ok=True) + + for grid_idx, points in grid_points.items(): + if self.config.enable_grid_division: + output_dir = os.path.join(self.config.output_dir, f'grid_{ + grid_idx + 1}', 'images') + else: + output_dir = os.path.join(self.config.output_dir, 'images') + + os.makedirs(output_dir, exist_ok=True) + + for point in tqdm(points, desc=f"复制网格 {grid_idx + 1} 的图像"): + src = os.path.join(self.config.image_dir, point['file']) + dst = os.path.join(output_dir, point['file']) + shutil.copy(src, dst) + self.logger.info(f"网格 {grid_idx + 1} 包含 {len(points)} 张图像") + + def visualize_results(self): + """可视化处理结果""" + if not self.config.enable_visualization: + return + + self.logger.info("开始生成可视化结果") + extractor = GPSExtractor(self.config.image_dir) + original_points = extractor.extract_all_gps() + + with open(os.path.join(self.config.output_dir, 'del_imgs.txt'), "r", encoding="utf-8") as file: + filtered_file = [line.strip() for line in file] + + # 绘制散点图 + plt.figure(figsize=(10, 8)) + plt.scatter([p['lon'] for p in original_points], + [p['lat'] for p in original_points], + color='blue', label="Original Points", alpha=0.6) + plt.scatter([p['lon'] for p in original_points if p['file'] in filtered_file], + [p['lat'] + for p in original_points if p['file'] in filtered_file], + color="red", label="Filtered Points", alpha=0.6) + plt.title("GPS Coordinates of Images", fontsize=14) + plt.xlabel("Longitude", fontsize=12) + plt.ylabel("Latitude", fontsize=12) + plt.grid(True) + plt.legend() + plt.savefig(os.path.join(self.config.output_dir, 'filter_GPS.png')) + plt.close() + self.logger.info("预处理结果图已保存") + + def process(self): + """执行完整的预处理流程""" + try: + self.extract_gps() + self.filter_points() + grid_points = self.divide_grids() + self.copy_images(grid_points) + self.visualize_results() + self.logger.info("预处理任务完成") + except Exception as e: + self.logger.error(f"处理过程中发生错误: {str(e)}", exc_info=True) + raise + + +if __name__ == '__main__': + # 创建配置 + config = PreprocessConfig( + image_dir=r'C:\datasets\1815\output\grid_5\images', + output_dir=r'C:\datasets\1815\output\grid_5', + filter_grid_size=0.001, + filter_dense_distance_threshold=10, + filter_distance_threshold=0.001, + filter_min_neighbors=6, + grid_overlap=0.05, + enable_filter=False, + enable_grid_division=True, + enable_visualization=False + ) + + # 创建处理器并执行 + processor = ImagePreprocessor(config) + processor.process() diff --git a/preprocess/__pycache__/gps_extractor.cpython-312.pyc b/preprocess/__pycache__/gps_extractor.cpython-312.pyc new file mode 100644 index 0000000..0a9f3f8 Binary files /dev/null and b/preprocess/__pycache__/gps_extractor.cpython-312.pyc differ diff --git a/preprocess/__pycache__/gps_extractor.cpython-39.pyc b/preprocess/__pycache__/gps_extractor.cpython-39.pyc new file mode 100644 index 0000000..b4369dc Binary files /dev/null and b/preprocess/__pycache__/gps_extractor.cpython-39.pyc differ diff --git a/preprocess/__pycache__/gps_filter.cpython-312.pyc b/preprocess/__pycache__/gps_filter.cpython-312.pyc new file mode 100644 index 0000000..e1c83c9 Binary files /dev/null and b/preprocess/__pycache__/gps_filter.cpython-312.pyc differ diff --git a/preprocess/__pycache__/gps_filter.cpython-39.pyc b/preprocess/__pycache__/gps_filter.cpython-39.pyc new file mode 100644 index 0000000..2350bc0 Binary files /dev/null and b/preprocess/__pycache__/gps_filter.cpython-39.pyc differ diff --git a/preprocess/__pycache__/grid_divider.cpython-312.pyc b/preprocess/__pycache__/grid_divider.cpython-312.pyc new file mode 100644 index 0000000..e6b8794 Binary files /dev/null and b/preprocess/__pycache__/grid_divider.cpython-312.pyc differ diff --git a/preprocess/__pycache__/grid_divider.cpython-39.pyc b/preprocess/__pycache__/grid_divider.cpython-39.pyc new file mode 100644 index 0000000..0111ed2 Binary files /dev/null and b/preprocess/__pycache__/grid_divider.cpython-39.pyc differ diff --git a/preprocess/__pycache__/logger.cpython-312.pyc b/preprocess/__pycache__/logger.cpython-312.pyc new file mode 100644 index 0000000..c45ca20 Binary files /dev/null and b/preprocess/__pycache__/logger.cpython-312.pyc differ diff --git a/preprocess/gps_extractor.py b/preprocess/gps_extractor.py new file mode 100644 index 0000000..5b217db --- /dev/null +++ b/preprocess/gps_extractor.py @@ -0,0 +1,55 @@ +import os +from PIL import Image +import piexif +import logging + + +class GPSExtractor: + """从图像文件提取GPS坐标""" + + def __init__(self, image_dir): + self.image_dir = image_dir + self.logger = logging.getLogger('UAV_Preprocess.GPSExtractor') + + @staticmethod + def _dms_to_decimal(dms): + """将DMS格式转换为十进制度""" + return dms[0][0] / dms[0][1] + (dms[1][0] / dms[1][1]) / 60 + (dms[2][0] / dms[2][1]) / 3600 + + def get_gps(self, image_path): + """提取单张图片的GPS坐标""" + try: + image = Image.open(image_path) + exif_data = piexif.load(image.info['exif']) + gps_info = exif_data.get("GPS", {}) + if gps_info: + lat = self._dms_to_decimal(gps_info.get(2, [])) + lon = self._dms_to_decimal(gps_info.get(4, [])) + self.logger.debug(f"成功提取图片GPS坐标: {image_path} - 纬度: {lat}, 经度: {lon}") + return lat, lon + else: + self.logger.warning(f"图片无GPS信息: {image_path}") + return None, None + except Exception as e: + self.logger.error(f"提取GPS坐标时发生错误: {image_path} - {str(e)}") + return None, None + + def extract_all_gps(self): + """提取所有图片的GPS坐标""" + self.logger.info(f"开始从目录提取GPS坐标: {self.image_dir}") + gps_points = [] + total_images = 0 + successful_extractions = 0 + + for image_file in os.listdir(self.image_dir): + if image_file.lower().endswith('.jpg'): + total_images += 1 + image_path = os.path.join(self.image_dir, image_file) + lat, lon = self.get_gps(image_path) + if lat and lon: + successful_extractions += 1 + gps_points.append( + {'file': image_file, 'lat': lat, 'lon': lon}) + + self.logger.info(f"GPS坐标提取完成 - 总图片数: {total_images}, 成功提取: {successful_extractions}, 失败: {total_images - successful_extractions}") + return gps_points diff --git a/preprocess/gps_filter.py b/preprocess/gps_filter.py new file mode 100644 index 0000000..8c61088 --- /dev/null +++ b/preprocess/gps_filter.py @@ -0,0 +1,146 @@ +import os +import math +from itertools import combinations +import numpy as np +from scipy.spatial import KDTree +import logging + + +class GPSFilter: + """过滤密集点及孤立点""" + + def __init__(self, output_dir): + self.log_file = os.path.join(output_dir, 'del_imgs.txt') + self.logger = logging.getLogger('UAV_Preprocess.GPSFilter') + + @staticmethod + def _haversine(lat1, lon1, lat2, lon2): + """计算两点之间的地理距离(单位:米)""" + R = 6371000 # 地球平均半径,单位:米 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + delta_phi = math.radians(lat2 - lat1) + delta_lambda = math.radians(lon2 - lon1) + + a = math.sin(delta_phi / 2) ** 2 + math.cos(phi1) * \ + math.cos(phi2) * math.sin(delta_lambda / 2) ** 2 + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + return R * c + + @staticmethod + def _assign_to_grid(lat, lon, grid_size, min_lat, min_lon): + """根据经纬度和网格大小,将点分配到网格""" + grid_x = int((lat - min_lat) // grid_size) + grid_y = int((lon - min_lon) // grid_size) + return grid_x, grid_y + + def _get_distances(self, points, grid_size): + """读取图片 GPS 坐标,计算点对之间的距离并排序""" + # 确定经纬度范围 + coords = np.array([[p['lat'], p['lon']] for p in points]) + min_lat, min_lon = np.min(coords, axis=0) + max_lat, max_lon = np.max(coords, axis=0) + self.logger.info( + f"经纬度范围:纬度[{min_lat:.6f}, {max_lat:.6f}],纬度范围[{max_lat-min_lat:.6f}]," + f"经度[{min_lon:.6f}, {max_lon:.6f}],经度范围[{max_lon-min_lon:.6f}]") + + # 分配到网格 + grid_map = {} + for img_info_dict in points: + grid = self._assign_to_grid( + img_info_dict['lat'], img_info_dict['lon'], grid_size, min_lat, min_lon) + if grid not in grid_map: + grid_map[grid] = [] + grid_map[grid].append( + (img_info_dict['file'], img_info_dict['lat'], img_info_dict['lon'])) + + self.logger.info(f"图像点已分配到 {len(grid_map)} 个网格中") + + # 在每个网格中计算两两距离并排序 + sorted_distances = {} + for grid, images in grid_map.items(): + distances = [] + for (img1, lat1, lon1), (img2, lat2, lon2) in combinations(images, 2): + dist = self._haversine(lat1, lon1, lat2, lon2) + distances.append((img1, img2, dist)) + distances.sort(key=lambda x: x[2]) # 按距离升序排序 + sorted_distances[grid] = distances + self.logger.debug(f"网格 {grid} 中计算了 {len(distances)} 个距离对") + + return sorted_distances + + def filter_dense_points(self, points, grid_size=0.001, distance_threshold=13): + """过滤密集点,根据提供的距离阈值""" + self.logger.info(f"开始过滤密集点 (网格大小: {grid_size}, 距离阈值: {distance_threshold}米)") + + # 获取每个网格中的图片的两两距离信息 + sorted_distances = self._get_distances(points, grid_size) + + to_del_imgs = [] + """遍历每个网格,删除网格中距离小于阈值的点""" + for grid, distances in sorted_distances.items(): + grid_del_count = 0 + while distances: + candidate_img1, candidate_img2, dist = distances[0] + if dist < distance_threshold: + # 有小于阈值的距离,先删除最近的距离,这两个点作为候选点,要删掉一个 + distances.pop(0) + + # 获取候选图片1和图片2倒数第二短的距离 + candidate_img1_dist = None + candidate_img2_dist = None + for distance in distances: + if candidate_img1 in distance: + candidate_img1_dist = distance[2] + break + for distance in distances: + if candidate_img2 in distance: + candidate_img2_dist = distance[2] + break + + # 谁短删掉谁 + if candidate_img1_dist and candidate_img2_dist: + if candidate_img1_dist < candidate_img2_dist: + to_del_img = candidate_img1 + else: + to_del_img = candidate_img2 + to_del_imgs.append(to_del_img) + grid_del_count += 1 + self.logger.debug(f"在网格 {grid} 中删除密集点: {to_del_img} (距离: {dist:.2f}米)") + # 从距离列表中删除与被删除图片相关的记录 + distances = [ + distance for distance in distances if to_del_img not in distance] + else: + break + if grid_del_count > 0: + self.logger.info(f"网格 {grid} 中删除了 {grid_del_count} 个密集点") + + # 过滤掉删除的图片,写入日志 + with open(self.log_file, 'a', encoding='utf-8') as f: + for img in to_del_imgs: + f.write(img+'\n') + + filtered_points = [point for point in points if point['file'] not in to_del_imgs] + self.logger.info(f"密集点过滤完成,共删除 {len(to_del_imgs)} 个点,剩余 {len(filtered_points)} 个点") + return filtered_points + + def filter_isolated_points(self, points, threshold_distance=0.001, min_neighbors=6): + """过滤孤立点""" + self.logger.info(f"开始过滤孤立点 (距离阈值: {threshold_distance}, 最小邻居数: {min_neighbors})") + + coords = np.array([[p['lat'], p['lon']] for p in points]) + kdtree = KDTree(coords) + neighbors_count = [len(kdtree.query_ball_point( + coord, threshold_distance)) for coord in coords] + + isolated_points = [] + with open(self.log_file, 'a', encoding='utf-8') as f: + for i, p in enumerate(points): + if neighbors_count[i] < min_neighbors: + isolated_points.append(p['file']) + f.write(p['file']+'\n') + self.logger.debug(f"删除孤立点: {p['file']} (邻居数: {neighbors_count[i]})") + f.write('\n') + + filtered_points = [p for i, p in enumerate(points) if neighbors_count[i] >= min_neighbors] + self.logger.info(f"孤立点过滤完成,共删除 {len(isolated_points)} 个点,剩余 {len(filtered_points)} 个点") + return filtered_points diff --git a/preprocess/grid_divider.py b/preprocess/grid_divider.py new file mode 100644 index 0000000..733ce8c --- /dev/null +++ b/preprocess/grid_divider.py @@ -0,0 +1,84 @@ +import logging + +class GridDivider: + """划分九宫格,并将图片分配到对应网格""" + + def __init__(self, overlap=0.1): + self.overlap = overlap + self.logger = logging.getLogger('UAV_Preprocess.GridDivider') + self.logger.info(f"初始化网格划分器,重叠率: {overlap}") + + def divide_grids(self, points): + """计算边界框并划分九宫格""" + self.logger.info("开始划分九宫格") + + lats = [p['lat'] for p in points] + lons = [p['lon'] for p in points] + min_lat, max_lat = min(lats), max(lats) + min_lon, max_lon = min(lons), max(lons) + + self.logger.info( + f"区域边界: 纬度[{min_lat:.6f}, {max_lat:.6f}], " + f"经度[{min_lon:.6f}, {max_lon:.6f}]" + ) + + lat_step = (max_lat - min_lat) / 3 + lon_step = (max_lon - min_lon) / 3 + + self.logger.debug(f"网格步长: 纬度{lat_step:.6f}, 经度{lon_step:.6f}") + + grids = [] + for i in range(3): + for j in range(3): + grid_min_lat = min_lat + i * lat_step - self.overlap * lat_step + grid_max_lat = min_lat + \ + (i + 1) * lat_step + self.overlap * lat_step + grid_min_lon = min_lon + j * lon_step - self.overlap * lon_step + grid_max_lon = min_lon + \ + (j + 1) * lon_step + self.overlap * lon_step + grids.append((grid_min_lat, grid_max_lat, + grid_min_lon, grid_max_lon)) + + self.logger.debug( + f"网格[{i},{j}]: 纬度[{grid_min_lat:.6f}, {grid_max_lat:.6f}], " + f"经度[{grid_min_lon:.6f}, {grid_max_lon:.6f}]" + ) + + self.logger.info(f"成功划分为 {len(grids)} 个网格") + return grids + + def assign_to_grids(self, points, grids): + """将点分配到对应网格""" + self.logger.info(f"开始将 {len(points)} 个点分配到网格中") + + grid_points = {i: [] for i in range(len(grids))} + points_assigned = 0 + multiple_grid_points = 0 + + for point in points: + point_assigned = False + for i, (min_lat, max_lat, min_lon, max_lon) in enumerate(grids): + if min_lat <= point['lat'] <= max_lat and min_lon <= point['lon'] <= max_lon: + grid_points[i].append(point) + if point_assigned: + multiple_grid_points += 1 + else: + points_assigned += 1 + point_assigned = True + + self.logger.debug( + f"点 {point['file']} (纬度: {point['lat']:.6f}, 经度: {point['lon']:.6f}) " + f"被分配到网格" + ) + + # 记录每个网格的点数 + for grid_idx, points in grid_points.items(): + self.logger.info(f"网格 {grid_idx} 包含 {len(points)} 个点") + + self.logger.info( + f"点分配完成: 总点数 {len(points)}, " + f"成功分配 {points_assigned} 个点, " + f"{multiple_grid_points} 个点被分配到多个网格" + ) + + return grid_points diff --git a/preprocess/logger.py b/preprocess/logger.py new file mode 100644 index 0000000..c6e0448 --- /dev/null +++ b/preprocess/logger.py @@ -0,0 +1,36 @@ +import logging +import os +from datetime import datetime + +def setup_logger(output_dir): + # 创建logs目录 + log_dir = os.path.join(output_dir, 'logs') + os.makedirs(log_dir, exist_ok=True) + + # 创建日志文件名(包含时间戳) + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + log_file = os.path.join(log_dir, f'preprocess_{timestamp}.log') + + # 配置日志格式 + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # 配置文件处理器 + file_handler = logging.FileHandler(log_file, encoding='utf-8') + file_handler.setFormatter(formatter) + + # 配置控制台处理器 + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + + # 获取根日志记录器 + logger = logging.getLogger('UAV_Preprocess') + logger.setLevel(logging.INFO) + + # 添加处理器 + logger.addHandler(file_handler) + logger.addHandler(console_handler) + + return logger \ No newline at end of file diff --git a/show_GPS.py b/show_GPS.py new file mode 100644 index 0000000..955d712 --- /dev/null +++ b/show_GPS.py @@ -0,0 +1,44 @@ +from gps_extractor import GPSExtractor +from gps_filter import GPSFilter +from grid_divider import GridDivider +import os +import pandas as pd +import shutil +import matplotlib.pyplot as plt + +DATASET = r'C:\datasets\1815\output\grid_5\grid_5\images' +IS_ORIGIN = True + +if __name__ == '__main__': + if IS_ORIGIN: + extractor = GPSExtractor(DATASET) + gps_points = extractor.extract_all_gps() + else: + # 提取九宫格中的GPS数据 + gps_points = [] + for subdir in os.listdir(DATASET): + subdir_path = os.path.join(DATASET, subdir) + if os.path.isdir(subdir_path): + extractor = GPSExtractor(subdir_path) + sub_gps_points = extractor.extract_all_gps() + gps_points = gps_points + sub_gps_points + print(len(sub_gps_points)) + + longitudes = [p['lon'] for p in gps_points] + latitudes = [p['lat'] for p in gps_points] + # 绘制散点图 + plt.figure(figsize=(10, 8)) + plt.scatter(longitudes, latitudes, color='blue', marker='o') + + # 添加标记,显示每个点的图像文件名 + for i, image in enumerate(gps_points): + plt.text(longitudes[i], latitudes[i], image['file'], fontsize=9, ha='right') + + # 添加图表标签 + plt.title("GPS Coordinates of Images", fontsize=14) + plt.xlabel("Longitude", fontsize=12) + plt.ylabel("Latitude", fontsize=12) + + # 显示图形 + plt.grid(True) + plt.show()