first commit

README.md

@@ -0,0 +1,2 @@
+# ODM_Pro
+无人机三维重建

odm_preprocess.py

@@ -0,0 +1,163 @@
+from gps_extractor import GPSExtractor
+from gps_filter import GPSFilter
+from grid_divider import GridDivider
+from logger import setup_logger
+import os
+import pandas as pd
+import shutil
+import matplotlib.pyplot as plt
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from tqdm import tqdm
+
+
+@dataclass
+class PreprocessConfig:
+    """预处理配置类"""
+    image_dir: str
+    output_dir: str
+    filter_grid_size: float = 0.001
+    filter_dense_distance_threshold: float = 10
+    filter_distance_threshold: float = 0.001
+    filter_min_neighbors: int = 6
+    grid_overlap: float = 0.05
+    enable_filter: bool = True
+    enable_grid_division: bool = True
+    enable_visualization: bool = True
+
+
+class ImagePreprocessor:
+    def __init__(self, config: PreprocessConfig):
+        self.config = config
+        self.logger = setup_logger(config.output_dir)
+        self.gps_points = []
+
+    def extract_gps(self) -> List[Dict]:
+        """提取GPS数据"""
+        self.logger.info("开始提取GPS数据")
+        extractor = GPSExtractor(self.config.image_dir)
+        self.gps_points = extractor.extract_all_gps()
+        self.logger.info(f"成功提取 {len(self.gps_points)} 个GPS点")
+        return self.gps_points
+
+    def filter_points(self) -> List[Dict]:
+        """过滤GPS点"""
+        if not self.config.enable_filter:
+            return self.gps_points
+
+        self.logger.info("开始过滤GPS点")
+        filter = GPSFilter(self.config.output_dir)
+
+        self.logger.info(f"开始过滤孤立点 (距离阈值: {self.config.filter_distance_threshold}, 最小邻居数: {
+                         self.config.filter_min_neighbors})")
+        self.gps_points = filter.filter_isolated_points(
+            self.gps_points,
+            self.config.filter_distance_threshold,
+            self.config.filter_min_neighbors
+        )
+        self.logger.info(f"孤立点过滤后剩余 {len(self.gps_points)} 个GPS点")
+
+        self.logger.info(f"开始过滤密集点 (网格大小: {self.config.filter_grid_size}, 距离阈值: {
+                         self.config.filter_dense_distance_threshold})")
+        self.gps_points = filter.filter_dense_points(
+            self.gps_points,
+            grid_size=self.config.filter_grid_size,
+            distance_threshold=self.config.filter_dense_distance_threshold
+        )
+        self.logger.info(f"密集点过滤后剩余 {len(self.gps_points)} 个GPS点")
+        return self.gps_points
+
+    def divide_grids(self) -> Dict[int, List[Dict]]:
+        """划分网格"""
+        if not self.config.enable_grid_division:
+            return {0: self.gps_points}  # 不划分网格时，所有点放在一个网格中
+
+        self.logger.info(f"开始划分网格 (重叠率: {self.config.grid_overlap})")
+        grid_divider = GridDivider(overlap=self.config.grid_overlap)
+        grids = grid_divider.divide_grids(self.gps_points)
+        grid_points = grid_divider.assign_to_grids(self.gps_points, grids)
+        self.logger.info(f"成功划分为 {len(grid_points)} 个网格")
+        return grid_points
+
+    def copy_images(self, grid_points: Dict[int, List[Dict]]):
+        """复制图像到目标文件夹"""
+        self.logger.info("开始复制图像文件")
+        os.makedirs(self.config.output_dir, exist_ok=True)
+
+        for grid_idx, points in grid_points.items():
+            if self.config.enable_grid_division:
+                output_dir = os.path.join(self.config.output_dir, f'grid_{
+                                          grid_idx + 1}', 'images')
+            else:
+                output_dir = os.path.join(self.config.output_dir, 'images')
+
+            os.makedirs(output_dir, exist_ok=True)
+
+            for point in tqdm(points, desc=f"复制网格 {grid_idx + 1} 的图像"):
+                src = os.path.join(self.config.image_dir, point['file'])
+                dst = os.path.join(output_dir, point['file'])
+                shutil.copy(src, dst)
+            self.logger.info(f"网格 {grid_idx + 1} 包含 {len(points)} 张图像")
+
+    def visualize_results(self):
+        """可视化处理结果"""
+        if not self.config.enable_visualization:
+            return
+
+        self.logger.info("开始生成可视化结果")
+        extractor = GPSExtractor(self.config.image_dir)
+        original_points = extractor.extract_all_gps()
+
+        with open(os.path.join(self.config.output_dir, 'del_imgs.txt'), "r", encoding="utf-8") as file:
+            filtered_file = [line.strip() for line in file]
+
+        # 绘制散点图
+        plt.figure(figsize=(10, 8))
+        plt.scatter([p['lon'] for p in original_points],
+                    [p['lat'] for p in original_points],
+                    color='blue', label="Original Points", alpha=0.6)
+        plt.scatter([p['lon'] for p in original_points if p['file'] in filtered_file],
+                    [p['lat']
+                        for p in original_points if p['file'] in filtered_file],
+                    color="red", label="Filtered Points", alpha=0.6)
+        plt.title("GPS Coordinates of Images", fontsize=14)
+        plt.xlabel("Longitude", fontsize=12)
+        plt.ylabel("Latitude", fontsize=12)
+        plt.grid(True)
+        plt.legend()
+        plt.savefig(os.path.join(self.config.output_dir, 'filter_GPS.png'))
+        plt.close()
+        self.logger.info("预处理结果图已保存")
+
+    def process(self):
+        """执行完整的预处理流程"""
+        try:
+            self.extract_gps()
+            self.filter_points()
+            grid_points = self.divide_grids()
+            self.copy_images(grid_points)
+            self.visualize_results()
+            self.logger.info("预处理任务完成")
+        except Exception as e:
+            self.logger.error(f"处理过程中发生错误: {str(e)}", exc_info=True)
+            raise
+
+
+if __name__ == '__main__':
+    # 创建配置
+    config = PreprocessConfig(
+        image_dir=r'C:\datasets\1815\output\grid_5\images',
+        output_dir=r'C:\datasets\1815\output\grid_5',
+        filter_grid_size=0.001,
+        filter_dense_distance_threshold=10,
+        filter_distance_threshold=0.001,
+        filter_min_neighbors=6,
+        grid_overlap=0.05,
+        enable_filter=False,
+        enable_grid_division=True,
+        enable_visualization=False
+    )
+
+    # 创建处理器并执行
+    processor = ImagePreprocessor(config)
+    processor.process()

preprocess/gps_extractor.py

@@ -0,0 +1,55 @@
+import os
+from PIL import Image
+import piexif
+import logging
+
+
+class GPSExtractor:
+    """从图像文件提取GPS坐标"""
+
+    def __init__(self, image_dir):
+        self.image_dir = image_dir
+        self.logger = logging.getLogger('UAV_Preprocess.GPSExtractor')
+
+    @staticmethod
+    def _dms_to_decimal(dms):
+        """将DMS格式转换为十进制度"""
+        return dms[0][0] / dms[0][1] + (dms[1][0] / dms[1][1]) / 60 + (dms[2][0] / dms[2][1]) / 3600
+
+    def get_gps(self, image_path):
+        """提取单张图片的GPS坐标"""
+        try:
+            image = Image.open(image_path)
+            exif_data = piexif.load(image.info['exif'])
+            gps_info = exif_data.get("GPS", {})
+            if gps_info:
+                lat = self._dms_to_decimal(gps_info.get(2, []))
+                lon = self._dms_to_decimal(gps_info.get(4, []))
+                self.logger.debug(f"成功提取图片GPS坐标: {image_path} - 纬度: {lat}, 经度: {lon}")
+                return lat, lon
+            else:
+                self.logger.warning(f"图片无GPS信息: {image_path}")
+                return None, None
+        except Exception as e:
+            self.logger.error(f"提取GPS坐标时发生错误: {image_path} - {str(e)}")
+            return None, None
+
+    def extract_all_gps(self):
+        """提取所有图片的GPS坐标"""
+        self.logger.info(f"开始从目录提取GPS坐标: {self.image_dir}")
+        gps_points = []
+        total_images = 0
+        successful_extractions = 0
+        
+        for image_file in os.listdir(self.image_dir):
+            if image_file.lower().endswith('.jpg'):
+                total_images += 1
+                image_path = os.path.join(self.image_dir, image_file)
+                lat, lon = self.get_gps(image_path)
+                if lat and lon:
+                    successful_extractions += 1
+                    gps_points.append(
+                        {'file': image_file, 'lat': lat, 'lon': lon})
+        
+        self.logger.info(f"GPS坐标提取完成 - 总图片数: {total_images}, 成功提取: {successful_extractions}, 失败: {total_images - successful_extractions}")
+        return gps_points

preprocess/gps_filter.py

@@ -0,0 +1,146 @@
+import os
+import math
+from itertools import combinations
+import numpy as np
+from scipy.spatial import KDTree
+import logging
+
+
+class GPSFilter:
+    """过滤密集点及孤立点"""
+
+    def __init__(self, output_dir):
+        self.log_file = os.path.join(output_dir, 'del_imgs.txt')
+        self.logger = logging.getLogger('UAV_Preprocess.GPSFilter')
+
+    @staticmethod
+    def _haversine(lat1, lon1, lat2, lon2):
+        """计算两点之间的地理距离（单位：米）"""
+        R = 6371000  # 地球平均半径，单位：米
+        phi1, phi2 = math.radians(lat1), math.radians(lat2)
+        delta_phi = math.radians(lat2 - lat1)
+        delta_lambda = math.radians(lon2 - lon1)
+
+        a = math.sin(delta_phi / 2) ** 2 + math.cos(phi1) * \
+            math.cos(phi2) * math.sin(delta_lambda / 2) ** 2
+        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
+        return R * c
+
+    @staticmethod
+    def _assign_to_grid(lat, lon, grid_size, min_lat, min_lon):
+        """根据经纬度和网格大小，将点分配到网格"""
+        grid_x = int((lat - min_lat) // grid_size)
+        grid_y = int((lon - min_lon) // grid_size)
+        return grid_x, grid_y
+
+    def _get_distances(self, points, grid_size):
+        """读取图片 GPS 坐标，计算点对之间的距离并排序"""
+        # 确定经纬度范围
+        coords = np.array([[p['lat'], p['lon']] for p in points])
+        min_lat, min_lon = np.min(coords, axis=0)
+        max_lat, max_lon = np.max(coords, axis=0)
+        self.logger.info(
+            f"经纬度范围：纬度[{min_lat:.6f}, {max_lat:.6f}]，纬度范围[{max_lat-min_lat:.6f}]，"
+            f"经度[{min_lon:.6f}, {max_lon:.6f}]，经度范围[{max_lon-min_lon:.6f}]")
+
+        # 分配到网格
+        grid_map = {}
+        for img_info_dict in points:
+            grid = self._assign_to_grid(
+                img_info_dict['lat'], img_info_dict['lon'], grid_size, min_lat, min_lon)
+            if grid not in grid_map:
+                grid_map[grid] = []
+            grid_map[grid].append(
+                (img_info_dict['file'], img_info_dict['lat'], img_info_dict['lon']))
+        
+        self.logger.info(f"图像点已分配到 {len(grid_map)} 个网格中")
+
+        # 在每个网格中计算两两距离并排序
+        sorted_distances = {}
+        for grid, images in grid_map.items():
+            distances = []
+            for (img1, lat1, lon1), (img2, lat2, lon2) in combinations(images, 2):
+                dist = self._haversine(lat1, lon1, lat2, lon2)
+                distances.append((img1, img2, dist))
+            distances.sort(key=lambda x: x[2])  # 按距离升序排序
+            sorted_distances[grid] = distances
+            self.logger.debug(f"网格 {grid} 中计算了 {len(distances)} 个距离对")
+
+        return sorted_distances
+
+    def filter_dense_points(self, points, grid_size=0.001, distance_threshold=13):
+        """过滤密集点，根据提供的距离阈值"""
+        self.logger.info(f"开始过滤密集点 (网格大小: {grid_size}, 距离阈值: {distance_threshold}米)")
+        
+        # 获取每个网格中的图片的两两距离信息
+        sorted_distances = self._get_distances(points, grid_size)
+
+        to_del_imgs = []
+        """遍历每个网格，删除网格中距离小于阈值的点"""
+        for grid, distances in sorted_distances.items():
+            grid_del_count = 0
+            while distances:
+                candidate_img1, candidate_img2, dist = distances[0]
+                if dist < distance_threshold:
+                    # 有小于阈值的距离，先删除最近的距离，这两个点作为候选点，要删掉一个
+                    distances.pop(0)
+
+                    # 获取候选图片1和图片2倒数第二短的距离
+                    candidate_img1_dist = None
+                    candidate_img2_dist = None
+                    for distance in distances:
+                        if candidate_img1 in distance:
+                            candidate_img1_dist = distance[2]
+                            break
+                    for distance in distances:
+                        if candidate_img2 in distance:
+                            candidate_img2_dist = distance[2]
+                            break
+
+                    # 谁短删掉谁
+                    if candidate_img1_dist and candidate_img2_dist:
+                        if candidate_img1_dist < candidate_img2_dist:
+                            to_del_img = candidate_img1
+                        else:
+                            to_del_img = candidate_img2
+                        to_del_imgs.append(to_del_img)
+                        grid_del_count += 1
+                        self.logger.debug(f"在网格 {grid} 中删除密集点: {to_del_img} (距离: {dist:.2f}米)")
+                        # 从距离列表中删除与被删除图片相关的记录
+                        distances = [
+                            distance for distance in distances if to_del_img not in distance]
+                else:
+                    break
+            if grid_del_count > 0:
+                self.logger.info(f"网格 {grid} 中删除了 {grid_del_count} 个密集点")
+
+        # 过滤掉删除的图片，写入日志
+        with open(self.log_file, 'a', encoding='utf-8') as f:
+            for img in to_del_imgs:
+                f.write(img+'\n')
+        
+        filtered_points = [point for point in points if point['file'] not in to_del_imgs]
+        self.logger.info(f"密集点过滤完成，共删除 {len(to_del_imgs)} 个点，剩余 {len(filtered_points)} 个点")
+        return filtered_points
+
+    def filter_isolated_points(self, points, threshold_distance=0.001, min_neighbors=6):
+        """过滤孤立点"""
+        self.logger.info(f"开始过滤孤立点 (距离阈值: {threshold_distance}, 最小邻居数: {min_neighbors})")
+        
+        coords = np.array([[p['lat'], p['lon']] for p in points])
+        kdtree = KDTree(coords)
+        neighbors_count = [len(kdtree.query_ball_point(
+            coord, threshold_distance)) for coord in coords]
+        
+        isolated_points = []
+        with open(self.log_file, 'a', encoding='utf-8') as f:
+            for i, p in enumerate(points):
+                if neighbors_count[i] < min_neighbors:
+                    isolated_points.append(p['file'])
+                    f.write(p['file']+'\n')
+                    self.logger.debug(f"删除孤立点: {p['file']} (邻居数: {neighbors_count[i]})")
+            f.write('\n')
+        
+        filtered_points = [p for i, p in enumerate(points) if neighbors_count[i] >= min_neighbors]
+        self.logger.info(f"孤立点过滤完成，共删除 {len(isolated_points)} 个点，剩余 {len(filtered_points)} 个点")
+        return filtered_points

preprocess/grid_divider.py

@@ -0,0 +1,84 @@
+import logging
+
+class GridDivider:
+    """划分九宫格，并将图片分配到对应网格"""
+
+    def __init__(self, overlap=0.1):
+        self.overlap = overlap
+        self.logger = logging.getLogger('UAV_Preprocess.GridDivider')
+        self.logger.info(f"初始化网格划分器，重叠率: {overlap}")
+
+    def divide_grids(self, points):
+        """计算边界框并划分九宫格"""
+        self.logger.info("开始划分九宫格")
+        
+        lats = [p['lat'] for p in points]
+        lons = [p['lon'] for p in points]
+        min_lat, max_lat = min(lats), max(lats)
+        min_lon, max_lon = min(lons), max(lons)
+        
+        self.logger.info(
+            f"区域边界: 纬度[{min_lat:.6f}, {max_lat:.6f}], "
+            f"经度[{min_lon:.6f}, {max_lon:.6f}]"
+        )
+
+        lat_step = (max_lat - min_lat) / 3
+        lon_step = (max_lon - min_lon) / 3
+        
+        self.logger.debug(f"网格步长: 纬度{lat_step:.6f}, 经度{lon_step:.6f}")
+
+        grids = []
+        for i in range(3):
+            for j in range(3):
+                grid_min_lat = min_lat + i * lat_step - self.overlap * lat_step
+                grid_max_lat = min_lat + \
+                    (i + 1) * lat_step + self.overlap * lat_step
+                grid_min_lon = min_lon + j * lon_step - self.overlap * lon_step
+                grid_max_lon = min_lon + \
+                    (j + 1) * lon_step + self.overlap * lon_step
+                grids.append((grid_min_lat, grid_max_lat,
+                             grid_min_lon, grid_max_lon))
+                
+                self.logger.debug(
+                    f"网格[{i},{j}]: 纬度[{grid_min_lat:.6f}, {grid_max_lat:.6f}], "
+                    f"经度[{grid_min_lon:.6f}, {grid_max_lon:.6f}]"
+                )
+
+        self.logger.info(f"成功划分为 {len(grids)} 个网格")
+        return grids
+
+    def assign_to_grids(self, points, grids):
+        """将点分配到对应网格"""
+        self.logger.info(f"开始将 {len(points)} 个点分配到网格中")
+        
+        grid_points = {i: [] for i in range(len(grids))}
+        points_assigned = 0
+        multiple_grid_points = 0
+        
+        for point in points:
+            point_assigned = False
+            for i, (min_lat, max_lat, min_lon, max_lon) in enumerate(grids):
+                if min_lat <= point['lat'] <= max_lat and min_lon <= point['lon'] <= max_lon:
+                    grid_points[i].append(point)
+                    if point_assigned:
+                        multiple_grid_points += 1
+                    else:
+                        points_assigned += 1
+                        point_assigned = True
+                        
+            self.logger.debug(
+                f"点 {point['file']} (纬度: {point['lat']:.6f}, 经度: {point['lon']:.6f}) "
+                f"被分配到网格"
+            )
+
+        # 记录每个网格的点数
+        for grid_idx, points in grid_points.items():
+            self.logger.info(f"网格 {grid_idx} 包含 {len(points)} 个点")
+
+        self.logger.info(
+            f"点分配完成: 总点数 {len(points)}, "
+            f"成功分配 {points_assigned} 个点, "
+            f"{multiple_grid_points} 个点被分配到多个网格"
+        )
+        
+        return grid_points

preprocess/logger.py

@@ -0,0 +1,36 @@
+import logging
+import os
+from datetime import datetime
+
+def setup_logger(output_dir):
+    # 创建logs目录
+    log_dir = os.path.join(output_dir, 'logs')
+    os.makedirs(log_dir, exist_ok=True)
+    
+    # 创建日志文件名（包含时间戳）
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    log_file = os.path.join(log_dir, f'preprocess_{timestamp}.log')
+    
+    # 配置日志格式
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    
+    # 配置文件处理器
+    file_handler = logging.FileHandler(log_file, encoding='utf-8')
+    file_handler.setFormatter(formatter)
+    
+    # 配置控制台处理器
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    
+    # 获取根日志记录器
+    logger = logging.getLogger('UAV_Preprocess')
+    logger.setLevel(logging.INFO)
+    
+    # 添加处理器
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+    
+    return logger 

show_GPS.py

@@ -0,0 +1,44 @@
+from gps_extractor import GPSExtractor
+from gps_filter import GPSFilter
+from grid_divider import GridDivider
+import os
+import pandas as pd
+import shutil
+import matplotlib.pyplot as plt
+
+DATASET = r'C:\datasets\1815\output\grid_5\grid_5\images'
+IS_ORIGIN = True
+
+if __name__ == '__main__':
+    if IS_ORIGIN:
+        extractor = GPSExtractor(DATASET)
+        gps_points = extractor.extract_all_gps()
+    else:
+        # 提取九宫格中的GPS数据
+        gps_points = []
+        for subdir in os.listdir(DATASET):
+            subdir_path = os.path.join(DATASET, subdir)
+            if os.path.isdir(subdir_path):
+                extractor = GPSExtractor(subdir_path)
+                sub_gps_points = extractor.extract_all_gps()
+                gps_points = gps_points + sub_gps_points
+                print(len(sub_gps_points))
+
+    longitudes = [p['lon'] for p in gps_points]
+    latitudes = [p['lat'] for p in gps_points]
+    # 绘制散点图
+    plt.figure(figsize=(10, 8))
+    plt.scatter(longitudes, latitudes, color='blue', marker='o')
+
+    # 添加标记，显示每个点的图像文件名
+    for i, image in enumerate(gps_points):
+        plt.text(longitudes[i], latitudes[i], image['file'], fontsize=9, ha='right')
+
+    # 添加图表标签
+    plt.title("GPS Coordinates of Images", fontsize=14)
+    plt.xlabel("Longitude", fontsize=12)
+    plt.ylabel("Latitude", fontsize=12)
+
+    # 显示图形
+    plt.grid(True)
+    plt.show()