#!/usr/bin/env python3
"""
Filtered OSM processor for significant Berlin green spaces.
Processes only meaningful green spaces (>1000 sqm) with real tree and toilet data.
"""

import json
import asyncio
import xml.etree.ElementTree as ET
from pathlib import Path
from datetime import datetime
import sys
import math
from typing import List, Dict, Optional, Tuple

# Add the app directory to Python path
sys.path.append(str(Path(__file__).parent.parent))

from app.services.street_tree_service import StreetTreeService
from app.services.berlin_data_service import BerlinDataService


class FilteredOSMProcessor:
    def __init__(self, data_dir: str = "app/data"):
        self.data_dir = Path(data_dir)
        self.osm_raw_dir = self.data_dir / "osm-raw"
        self.processed_dir = self.data_dir / "processed"
        
        # Initialize services
        self.tree_service = StreetTreeService()
        self.berlin_data = BerlinDataService()
        
        # Berlin bounding box
        self.berlin_bbox = {
            'min_lat': 52.3370, 'max_lat': 52.6755,
            'min_lon': 13.0882, 'max_lon': 13.7611
        }
        
        # Filtering criteria
        self.min_area_sqm = 1000  # Minimum area to be considered significant
        self.max_spaces = 800     # Maximum number of spaces to process
    
    def parse_and_filter_osm_data(self) -> List[Dict]:
        """Parse OSM data and filter for significant green spaces."""
        osm_file = self.osm_raw_dir / "berlin_green_spaces.osm"
        
        if not osm_file.exists():
            print(f"❌ OSM file not found: {osm_file}")
            print("Please run the download first or ensure the file exists.")
            return []
        
        print(f"📂 Parsing OSM data from {osm_file}")
        
        try:
            tree = ET.parse(osm_file)
            root = tree.getroot()
            ways = root.findall('.//way')
            
            print(f"📊 Found {len(ways)} total ways in OSM file")
            print(f"🔍 Filtering for significant green spaces (≥{self.min_area_sqm} sqm)...")
            
            filtered_spaces = []
            processed_count = 0
            
            for way in ways:
                processed_count += 1
                
                if processed_count % 5000 == 0:
                    print(f"   Processed {processed_count}/{len(ways)} ways... Found {len(filtered_spaces)} significant spaces")
                
                try:
                    space_data = self._process_osm_way(way, root)
                    if space_data and space_data['area_sqm'] >= self.min_area_sqm:
                        filtered_spaces.append(space_data)
                        
                        # Stop if we have enough spaces
                        if len(filtered_spaces) >= self.max_spaces:
                            print(f"✅ Reached target of {self.max_spaces} significant spaces")
                            break
                            
                except Exception as e:
                    continue
            
            # Sort by area (largest first) to prioritize important spaces
            filtered_spaces.sort(key=lambda x: x['area_sqm'], reverse=True)
            
            print(f"🎯 Filtered to {len(filtered_spaces)} significant green spaces")
            print(f"📏 Area range: {filtered_spaces[-1]['area_sqm']:,} - {filtered_spaces[0]['area_sqm']:,} sqm")
            
            return filtered_spaces
            
        except Exception as e:
            print(f"❌ Error parsing OSM file: {e}")
            return []
    
    def _process_osm_way(self, way, root) -> Optional[Dict]:
        """Process a single OSM way into green space format."""
        # Get tags
        tags = {}
        for tag in way.findall('tag'):
            tags[tag.get('k')] = tag.get('v')
        
        # Check if it's a significant green space
        green_space_type = self._get_green_space_type(tags)
        if not green_space_type:
            return None
        
        # Skip certain types that are usually small or not parks
        skip_types = ['grave_yard', 'cemetery', 'allotments']
        if green_space_type in skip_types:
            return None
        
        # Get node references
        nd_refs = [nd.get('ref') for nd in way.findall('nd')]
        if len(nd_refs) < 3:  # Need at least 3 points for an area
            return None
        
        # Find node coordinates
        coordinates = []
        for nd_ref in nd_refs:
            node = root.find(f".//node[@id='{nd_ref}']")
            if node is not None:
                lat = float(node.get('lat'))
                lon = float(node.get('lon'))
                
                # Check if within Berlin bounds
                if (self.berlin_bbox['min_lat'] <= lat <= self.berlin_bbox['max_lat'] and
                    self.berlin_bbox['min_lon'] <= lon <= self.berlin_bbox['max_lon']):
                    coordinates.append((lat, lon))
        
        if len(coordinates) < 3:
            return None
        
        # Calculate centroid and area
        centroid_lat, centroid_lon = self._calculate_centroid(coordinates)
        area_sqm = self._calculate_area(coordinates)
        
        # Skip if too small
        if area_sqm < self.min_area_sqm:
            return None
        
        # Get name
        name = tags.get('name')
        if not name:
            name = f"{green_space_type.title()} near {centroid_lat:.3f}, {centroid_lon:.3f}"
        
        # Estimate district
        district = self._estimate_district(centroid_lat, centroid_lon)
        
        return {
            'id': f"osm_way_{way.get('id')}",
            'name': name,
            'fclass': green_space_type,
            'lat': centroid_lat,
            'lng': centroid_lon,
            'area_sqm': int(area_sqm),
            'district': district,
            'osm_tags': tags,
            'osm_id': way.get('id'),
            'has_name': bool(tags.get('name'))  # Track if it has a real name
        }
    
    def _get_green_space_type(self, tags: Dict) -> Optional[str]:
        """Determine if tags represent a significant green space."""
        # Prioritize leisure tags (usually parks)
        leisure = tags.get('leisure', '')
        if leisure in ['park', 'garden', 'nature_reserve', 'recreation_ground', 'playground', 'common']:
            return leisure
        
        # Check landuse tags
        landuse = tags.get('landuse', '')
        if landuse in ['forest', 'grass', 'meadow', 'recreation_ground', 'village_green']:
            return landuse
        
        # Check natural tags (forests, etc.)
        natural = tags.get('natural', '')
        if natural in ['forest', 'wood', 'heath']:
            return natural
        
        return None
    
    def _calculate_centroid(self, coordinates: List[Tuple[float, float]]) -> Tuple[float, float]:
        """Calculate centroid of polygon."""
        lat_sum = sum(coord[0] for coord in coordinates)
        lon_sum = sum(coord[1] for coord in coordinates)
        count = len(coordinates)
        return lat_sum / count, lon_sum / count
    
    def _calculate_area(self, coordinates: List[Tuple[float, float]]) -> float:
        """Calculate area using shoelace formula (approximate for Berlin)."""
        if len(coordinates) < 3:
            return 0
        
        # Convert to approximate meters for Berlin latitude
        lat_to_m = 111000  # meters per degree latitude
        lon_to_m = 111000 * math.cos(math.radians(52.5))  # adjust for Berlin
        
        # Convert to meters
        coords_m = [(lat * lat_to_m, lon * lon_to_m) for lat, lon in coordinates]
        
        # Shoelace formula
        area = 0
        n = len(coords_m)
        
        for i in range(n):
            j = (i + 1) % n
            area += coords_m[i][0] * coords_m[j][1]
            area -= coords_m[j][0] * coords_m[i][1]
        
        return abs(area) / 2
    
    def _estimate_district(self, lat: float, lng: float) -> str:
        """Estimate Berlin district from coordinates."""
        # Simplified district boundaries
        if lat > 52.55:
            return "Pankow" if lng < 13.45 else "Lichtenberg"
        elif lat > 52.52:
            if lng < 13.25:
                return "Charlottenburg-Wilmersdorf"
            elif lng < 13.42:
                return "Mitte"
            else:
                return "Friedrichshain-Kreuzberg"
        elif lat > 52.45:
            if lng < 13.25:
                return "Steglitz-Zehlendorf"
            elif lng < 13.42:
                return "Tempelhof-Schöneberg"
            else:
                return "Neukölln"
        else:
            return "Treptow-Köpenick"
    
    async def enhance_green_space_with_real_data(self, space_data: Dict):
        """Enhance green space with real tree and toilet data."""
        try:
            lat = space_data['lat']
            lng = space_data['lng']
            area_sqm = space_data['area_sqm']
            
            # Adaptive radius based on space size
            radius = min(400, max(150, int((area_sqm ** 0.5) * 0.8)))
            
            # Get real data using existing services
            tree_response = await self.tree_service.get_trees_near_location(
                lat, lng, radius_m=radius
            )
            
            nearby_toilets = await self.berlin_data.get_toilets_near_point(lat, lng, 600)
            
            # Calculate scores
            toilet_score = self._score_toilet_accessibility(nearby_toilets)
            space_type = self._map_to_space_type(space_data.get('fclass', ''))
            
            enhanced_space = {
                "id": space_data['id'],
                "name": space_data['name'],
                "description": f"Significant Berlin {space_data.get('fclass', 'green space')} from OSM data",
                "type": space_type,
                "coordinates": {
                    "lat": float(lat),
                    "lng": float(lng)
                },
                "neighborhood": space_data.get('district', 'Unknown'),
                "area_sqm": area_sqm,
                "perimeter_m": int(4 * (area_sqm ** 0.5)),  # Rough estimate
                
                # Environmental features from real tree data
                "environmental": {
                    "tree_coverage_percent": max(5, int(tree_response.shade_analysis.estimated_shade_coverage)),
                    "shade_quality": tree_response.shade_analysis.shade_quality_score,
                    "noise_level": self._estimate_noise_level(space_data),
                    "wildlife_diversity_score": tree_response.metrics.species_diversity_score,
                    "water_features": self._detect_water_features(space_data),
                    "natural_surface_percent": self._estimate_natural_surface(space_data.get('fclass', ''))
                },
                
                # Real tree metrics from your existing service
                "tree_data": {
                    "total_trees": tree_response.metrics.total_trees,
                    "trees_per_hectare": tree_response.metrics.trees_per_hectare,
                    "species_count": len(tree_response.metrics.dominant_species),
                    "species_diversity_score": tree_response.metrics.species_diversity_score,
                    "mature_trees_count": tree_response.metrics.mature_trees_count,
                    "young_trees_count": tree_response.metrics.young_trees_count,
                    "average_tree_age": tree_response.metrics.average_tree_age,
                    "average_height": tree_response.metrics.average_height,
                    "average_crown_diameter": tree_response.metrics.average_crown_diameter,
                    "shade_coverage_percent": tree_response.metrics.shade_coverage_percent,
                    "dominant_species": tree_response.metrics.dominant_species[:3]  # Top 3
                },
                
                # Real toilet accessibility from your existing service
                "toilet_accessibility": {
                    "nearby_toilets_count": len(nearby_toilets),
                    "accessibility_score": toilet_score,
                    "nearest_distance_m": nearby_toilets[0]['distance_meters'] if nearby_toilets else None,
                    "free_toilets_count": len([t for t in nearby_toilets if t.get('is_free', False)]),
                    "accessible_toilets_count": len([t for t in nearby_toilets if t.get('wheelchair_accessible', False)])
                },
                
                # Accessibility features
                "accessibility": {
                    "wheelchair_accessible": True,
                    "public_transport_score": self._estimate_transport_score(space_data.get('district', '')),
                    "cycling_infrastructure": area_sqm > 5000,
                    "parking_availability": 3 if area_sqm > 50000 else 2,
                    "lighting_quality": 3 if 'mitte' in space_data.get('district', '').lower() else 2
                },
                
                "recreation": {
                    "playground_quality": self._estimate_playground_quality(space_data),
                    "sports_facilities": self._estimate_sports_facilities(space_data),
                    "running_paths": area_sqm > 8000,
                    "cycling_paths": area_sqm > 15000,
                    "dog_friendly": True,
                    "bbq_allowed": self._allows_bbq(space_data)
                },
                
                # OSM metadata
                "osm_metadata": {
                    "osm_id": space_data.get('osm_id'),
                    "has_official_name": space_data.get('has_name', False),
                    "tags": space_data.get('osm_tags', {}),
                    "source": "filtered_osm_extract"
                },
                
                "last_updated": datetime.now().isoformat(),
                "data_sources": ["filtered_osm_extract", "berlin_tree_cadastre", "berlin_toilets"],
                "confidence_score": 95 if space_data.get('has_name') else 85
            }
            
            return enhanced_space
            
        except Exception as e:
            print(f"❌ Error enhancing {space_data['name']}: {e}")
            return None
    
    def _score_toilet_accessibility(self, nearby_toilets: List[Dict]) -> int:
        if not nearby_toilets:
            return 25
        
        nearest = nearby_toilets[0]['distance_meters']
        if nearest <= 200:
            score = 95
        elif nearest <= 400:
            score = 80
        elif nearest <= 600:
            score = 65
        else:
            score = 45
        
        # Quality bonuses
        free = len([t for t in nearby_toilets if t.get('is_free', False)])
        accessible = len([t for t in nearby_toilets if t.get('wheelchair_accessible', False)])
        score += min(10, free * 5 + accessible * 3)
        
        return min(100, score)
    
    def _map_to_space_type(self, fclass: str) -> str:
        mapping = {
            'park': 'PARK', 'forest': 'FOREST', 'garden': 'GARDEN', 'wood': 'FOREST',
            'nature_reserve': 'NATURE_RESERVE', 'playground': 'PLAYGROUND',
            'meadow': 'MEADOW', 'grass': 'GRASS', 'recreation_ground': 'PARK',
            'common': 'PARK', 'village_green': 'GRASS', 'heath': 'HEATH'
        }
        return mapping.get(fclass, 'PARK')
    
    def _detect_water_features(self, space_data: Dict) -> bool:
        name = space_data.get('name', '').lower()
        tags = space_data.get('osm_tags', {})
        
        water_keywords = ['see', 'teich', 'pond', 'lake', 'bach', 'spree', 'wasser', 'fluss']
        return (any(keyword in name for keyword in water_keywords) or 
                'water' in str(tags.values()).lower())
    
    def _estimate_noise_level(self, space_data: Dict) -> int:
        fclass = space_data.get('fclass', '')
        district = space_data.get('district', '')
        area = space_data.get('area_sqm', 0)
        
        base = {'forest': 1, 'wood': 1, 'nature_reserve': 1, 'heath': 1,
               'meadow': 2, 'grass': 2, 'park': 2, 'garden': 2, 
               'playground': 3, 'recreation_ground': 3}.get(fclass, 2)
        
        # Central districts are noisier
        if any(busy in district.lower() for busy in ['mitte', 'kreuzberg', 'friedrichshain']):
            base += 1
        
        # Larger spaces are usually quieter inside
        if area > 50000:
            base = max(1, base - 1)
        
        return min(5, base)
    
    def _estimate_natural_surface(self, fclass: str) -> int:
        return {'forest': 95, 'wood': 95, 'nature_reserve': 95, 'heath': 90,
               'meadow': 95, 'grass': 90, 'park': 80, 'garden': 70, 
               'playground': 45, 'recreation_ground': 75}.get(fclass, 75)
    
    def _estimate_transport_score(self, district: str) -> int:
        district_lower = district.lower()
        if 'mitte' in district_lower:
            return 5
        elif any(name in district_lower for name in ['charlottenburg', 'kreuzberg', 'friedrichshain', 'pankow']):
            return 4
        else:
            return 3
    
    def _estimate_playground_quality(self, space_data: Dict) -> int:
        fclass = space_data.get('fclass', '')
        tags = space_data.get('osm_tags', {})
        area = space_data.get('area_sqm', 0)
        
        if fclass == 'playground':
            return 85
        elif 'playground' in str(tags.values()).lower():
            return 75
        elif fclass == 'park':
            # Larger parks more likely to have good playgrounds
            return 60 if area > 10000 else 45
        else:
            return 30
    
    def _estimate_sports_facilities(self, space_data: Dict) -> bool:
        fclass = space_data.get('fclass', '')
        tags = space_data.get('osm_tags', {})
        name = space_data.get('name', '').lower()
        area = space_data.get('area_sqm', 0)
        
        # Explicit indicators
        if (fclass == 'recreation_ground' or 
            'sport' in str(tags.values()).lower() or
            any(term in name for term in ['sport', 'football', 'tennis', 'recreation'])):
            return True
        
        # Large parks often have sports facilities
        return fclass == 'park' and area > 20000
    
    def _allows_bbq(self, space_data: Dict) -> bool:
        fclass = space_data.get('fclass', '')
        tags = space_data.get('osm_tags', {})
        area = space_data.get('area_sqm', 0)
        
        # Check explicit BBQ tags
        bbq_tag = tags.get('bbq', '').lower()
        if bbq_tag == 'yes':
            return True
        elif bbq_tag == 'no':
            return False
        
        # Default based on type and size
        return fclass in ['park', 'recreation_ground'] and area > 5000
    
    async def process_filtered_green_spaces(self):
        """Main processing pipeline for filtered green spaces."""
        print("🌳 Processing Significant Berlin Green Spaces")
        print("=" * 55)
        print(f"• Filtering for spaces ≥ {self.min_area_sqm:,} sqm")
        print(f"• Processing up to {self.max_spaces} significant spaces")
        print(f"• Enhancing with real Berlin tree + toilet data")
        print("=" * 55)
        
        # Step 1: Parse and filter OSM data
        filtered_spaces = self.parse_and_filter_osm_data()
        
        if not filtered_spaces:
            print("❌ No significant green spaces found")
            return []
        
        print(f"\n🔧 Enhancing {len(filtered_spaces)} significant spaces with real data...")
        
        # Step 2: Enhance with real data
        enhanced_spaces = []
        
        for i, space_data in enumerate(filtered_spaces, 1):
            area_ha = space_data['area_sqm'] / 10000
            print(f"[{i:3d}/{len(filtered_spaces)}] {space_data['name'][:40]:40} ({area_ha:.1f} ha)")
            
            result = await self.enhance_green_space_with_real_data(space_data)
            if result:
                enhanced_spaces.append(result)
                trees = result["tree_data"]["total_trees"]
                toilets = result["toilet_accessibility"]["nearby_toilets_count"]
                print(f"              ✅ {trees:3d} trees, {toilets} toilets")
            else:
                print(f"              ❌ Enhancement failed")
            
            # Progress update every 50 spaces
            if i % 50 == 0:
                print(f"\n   📊 Progress: {len(enhanced_spaces)}/{i} enhanced successfully")
            
            # Small delay to be nice to services
            await asyncio.sleep(0.1)
        
        print(f"\n🎉 Successfully enhanced {len(enhanced_spaces)} significant green spaces!")
        return enhanced_spaces
    
    def save_enhanced_data(self, enhanced_spaces: List[Dict]):
        """Save the filtered and enhanced dataset."""
        output_file = self.processed_dir / "significant_berlin_green_spaces.json"
        
        # Calculate comprehensive statistics
        with_trees = len([s for s in enhanced_spaces if s["tree_data"]["total_trees"] > 0])
        with_toilets = len([s for s in enhanced_spaces if s["toilet_accessibility"]["nearby_toilets_count"] > 0])
        total_trees = sum(s["tree_data"]["total_trees"] for s in enhanced_spaces)
        total_area = sum(s["area_sqm"] for s in enhanced_spaces)
        
        # Named vs unnamed spaces
        named_spaces = len([s for s in enhanced_spaces if s["osm_metadata"]["has_official_name"]])
        
        # Area distribution
        large_spaces = len([s for s in enhanced_spaces if s["area_sqm"] > 50000])  # > 5 hectares
        medium_spaces = len([s for s in enhanced_spaces if 10000 <= s["area_sqm"] <= 50000])  # 1-5 hectares
        small_spaces = len([s for s in enhanced_spaces if s["area_sqm"] < 10000])  # < 1 hectare
        
        # District breakdown
        by_district = {}
        for space in enhanced_spaces:
            district = space['neighborhood']
            if district not in by_district:
                by_district[district] = []
            by_district[district].append(space)
        
        data = {
            "green_spaces": enhanced_spaces,
            "total_count": len(enhanced_spaces),
            "last_updated": datetime.now().isoformat(),
            "data_sources": [
                "filtered_osm_extract_significant_spaces_only",
                "berlin_tree_cadastre_via_street_tree_service", 
                "berlin_toilet_locations_via_berlin_data_service"
            ],
            "processing_info": {
                "filtering_criteria": {
                    "minimum_area_sqm": self.min_area_sqm,
                    "maximum_spaces_processed": self.max_spaces,
                    "includes_only_significant_spaces": True
                },
                "enhancement_method": "real_berlin_tree_and_toilet_data",
                "coordinate_system": "WGS84"
            },
            "summary_stats": {
                "total_spaces": len(enhanced_spaces),
                "spaces_with_tree_data": with_trees,
                "spaces_with_toilet_data": with_toilets,
                "total_trees_analyzed": total_trees,
                "total_area_hectares": round(total_area / 10000, 1),
                "coverage_rates": {
                    "tree_data": f"{round((with_trees/len(enhanced_spaces))*100, 1)}%",
                    "toilet_data": f"{round((with_toilets/len(enhanced_spaces))*100, 1)}%"
                },
                "space_categories": {
                    "named_spaces": named_spaces,
                    "unnamed_spaces": len(enhanced_spaces) - named_spaces,
                    "large_spaces_over_5ha": large_spaces,
                    "medium_spaces_1_5ha": medium_spaces,
                    "smaller_spaces_under_1ha": small_spaces
                }
            },
            "district_breakdown": {
                district: len(spaces) for district, spaces in by_district.items()
            }
        }
        
        with open(output_file, 'w', encoding='utf-8') as f:
            json.dump(data, f, indent=2, ensure_ascii=False)
        
        print(f"\n📁 Comprehensive dataset saved: {output_file}")
        print(f"\n📊 Final Statistics:")
        print(f"   🌳 {len(enhanced_spaces)} significant green spaces")
        print(f"   📛 {named_spaces} with official names, {len(enhanced_spaces) - named_spaces} discovered areas")
        print(f"   🌲 {with_trees} spaces with tree data ({round((with_trees/len(enhanced_spaces))*100)}%)")
        print(f"   🚻 {with_toilets} spaces with toilet data ({round((with_toilets/len(enhanced_spaces))*100)}%)")
        print(f"   🌿 {total_trees:,} total trees analyzed")
        print(f"   📏 {round(total_area/10000, 1)} hectares total area")
        
        print(f"\n🏙️ District Distribution:")
        for district, spaces in sorted(by_district.items(), key=lambda x: len(x[1]), reverse=True):
            print(f"   • {district}: {len(spaces)} spaces")
        
        print(f"\n📈 Size Categories:")
        print(f"   • Large (>5 ha): {large_spaces} spaces")
        print(f"   • Medium (1-5 ha): {medium_spaces} spaces") 
        print(f"   • Smaller (<1 ha): {small_spaces} spaces")
        
        print(f"\n✨ This dataset provides comprehensive coverage of Berlin's")
        print(f"    significant green spaces with real tree and toilet data!")
        
        return output_file


async def main():
    processor = FilteredOSMProcessor()
    
    try:
        enhanced_spaces = await processor.process_filtered_green_spaces()
        
        if enhanced_spaces:
            processor.save_enhanced_data(enhanced_spaces)
            print(f"\n🎯 SUCCESS! Ready to use in your API for accurate personality scoring!")
        else:
            print("❌ No spaces were successfully processed.")
        
    except KeyboardInterrupt:
        print("\n⚠️ Process interrupted by user")
    except Exception as e:
        print(f"❌ Error: {e}")
        raise


if __name__ == "__main__":
    asyncio.run(main())