""" Management command to import / update RoadDrain objects from external/avaloirs_2026.csv. CSV → RoadDrain field mapping ───────────────────────────── geom → geom (PointField srid=3812, dim=3) + lon / lat / geojson DR_ID → code (e.g. "DR-001") DR_STREET → street (ForeignKey to RoadStreet, matched by name_fr) DR_SOURCE → positionning_source (with DR_SURVEY_TYPE appended) DR_SOILTYPE → soil_type DR_LOCATION → drain_location_type DR_VEGETATION → nearby_vegetation DR_ROOTING → possible_rooting DR_STATE → is_defect DR_BAD_STATE_TYPE → defect_types (all tokens as list) DR_STATE_NOTE → defect_note (appended) DR_Z → altitude DR_BEGIN_LIFE → installation_date DR_MEASURE_POSITION / DR_POSITION / DR_BAD_POSITION_TYPE / DR_POSITION_NOTE → positionning_note (combined) DR_MODEL → model (RoadDrainModel, looked-up or created by code) DR_MODEL_NOTE / DR_NOTE → note (combined) DR_TYPE → related_image_type DR_OPERATOR → brand """ import csv import json import os import re from datetime import datetime from django.contrib.gis.gdal import CoordTransform, SpatialReference from django.contrib.gis.geos import Point from django.core.management.base import BaseCommand from django.db import transaction from assets.models.roads import RoadDrain, RoadDrainModel, RoadStreet # ── Path to CSV ─────────────────────────────────────────────────────────────── CSV_PATH = os.path.normpath( os.path.join( os.path.dirname(__file__), # …/assets/management/commands/ "..", "..", "..", # → repo root "scripts", "data", "avaloirs_2026.csv", ) ) BULK_BATCH_SIZE = 500 # ── Lookup / mapping tables ─────────────────────────────────────────────────── SOIL_TYPE_MAP = { "CONCRETE": "concrete", "ASPHALT": "asphalt", "STONE": "stone", "CONCRETE SLABS": "concrete_slabs", "CONCRETE PAVING": "concrete_paving", "STONE SLABS": "stone_slabs", "STONE PAVING": "stone_paving", "CONCRETE/GRASS": "concrete_grass", "GRAVEL": "gravel", "GRASS": "grass", "EARTH": "earth", } LOCATION_TYPE_MAP = { "WATERWAY": "waterway", "SIDEWALK": "sidewalk", "BERME": "berme", "SIDEWALK-FACADE TO ROAD": "sidewalk_facade_to_road", "SQUARE": "square", "ALONG SIDEWALK": "along_sidewalk", "MIDDLE ROAD": "middle_road", } DEFECT_TYPE_MAP = { "OBSTRUCTED": "obstructed", "SMELL CUT MISSING": "smell_cut_missing", "GRATE BROKEN": "grate_broken", "GRATE MISSING": "grate_missing", "POOR WATER FLOW": "poor_water_flow", "OTHER": "other", } # Fields updated on an existing drain (update path) UPDATE_FIELDS = [ "street_id", "model_id", "soil_type", "drain_location_type", "nearby_vegetation", "possible_rooting", "is_defect", "defect_types", "defect_note", "positionning_source", "positionning_note", "positionning_date", "altitude", "related_image_type", "brand", "note", "installation_date", "geom", "lon", "lat", "geojson", ] # ── Helpers ─────────────────────────────────────────────────────────────────── def _parse_bool(val: str): """Return True/False/None from 'true'/'false'/'' strings.""" v = (val or "").strip().lower() if v == "true": return True if v == "false": return False return None def _parse_defect_tokens(raw: str) -> list[str]: """ Parse values like '{OBSTRUCTED}', '{GRATE BROKEN, OTHER}', '{}'. Returns a list of upper-cased token strings. """ raw = raw.strip() if not raw or raw == "{}": return [] inner = re.sub(r"^\{|\}$", "", raw) return [t.strip() for t in inner.split(",") if t.strip()] def _extract_model_code(model_path: str) -> str: """'MODEL/AV/AV_TYPE-002.png' → 'AV_TYPE-002'""" return os.path.splitext(os.path.basename(model_path))[0] def _parse_geom(wkt: str): """ Parse 'POINT ZM (x y z m)' (already in EPSG:3812) into a 3-D Point. Returns a GEOSGeometry Point(srid=3812) or None on failure. """ m = re.match( r"POINT\s+ZM\s*\(\s*([\d.+-]+)\s+([\d.+-]+)\s+([\d.+-]+)\s+([\d.+-]+)\s*\)", wkt, re.IGNORECASE, ) if not m: return None x, y, z = float(m.group(1)), float(m.group(2)), float(m.group(3)) return Point(x, y, z, srid=3812) def _lon_lat_geojson(geom_3812): """Transform a 3812 Point to WGS84, return (lon, lat, geojson_str).""" srs_3812 = SpatialReference(3812) srs_4326 = SpatialReference(4326) ct = CoordTransform(srs_3812, srs_4326) g = geom_3812.clone() g.transform(ct) lon, lat = round(g.x, 8), round(g.y, 8) geojson = json.dumps({"type": "Point", "coordinates": [lon, lat]}) return lon, lat, geojson def _parse_date(raw: str): """Parse 'YYYY/MM/DD' → date, or None.""" raw = (raw or "").strip() if not raw: return None try: return datetime.strptime(raw, "%Y/%m/%d").date() except ValueError: return None def _build_drain(row: dict, street_map: dict, drain_model_map: dict) -> dict: """ Convert one CSV row into a dict of RoadDrain field values. Returns None if the row must be skipped. """ if (row.get("DR_TYPE") or "").strip().upper() != "DRAIN POINT": return None dr_id = (row.get("DR_ID") or "").strip() if not dr_id: return None code = f"DR-{int(dr_id):05d}" # ── Geometry ────────────────────────────────────────────────────────────── wkt = (row.get("geom") or "").strip() geom = _parse_geom(wkt) if wkt else None if geom is None: return None lon, lat, geojson = _lon_lat_geojson(geom) # ── Street ──────────────────────────────────────────────────────────────── # street_name = (row.get("DR_STREET") or "").strip() # street = street_map.get(street_name) # may be None street = None # ── Drain model ─────────────────────────────────────────────────────────── model_path = (row.get("DR_MODEL") or "").strip() drain_model = None if model_path: model_code = _extract_model_code(model_path) drain_model = drain_model_map.get(model_code) # ── Soil type ───────────────────────────────────────────────────────────── soil_raw = (row.get("DR_SOILTYPE") or "").strip().upper() soil_type = SOIL_TYPE_MAP.get(soil_raw) # ── Location type ───────────────────────────────────────────────────────── loc_raw = (row.get("DR_LOCATION") or "").strip().upper() drain_location_type = LOCATION_TYPE_MAP.get(loc_raw) # ── Boolean flags ───────────────────────────────────────────────────────── nearby_vegetation = _parse_bool(row.get("DR_VEGETATION", "")) possible_rooting = _parse_bool(row.get("DR_ROOTING", "")) is_defect = _parse_bool(row.get("DR_STATE", "")) # ── Defect type / note ──────────────────────────────────────────────────── defect_tokens = _parse_defect_tokens(row.get("DR_BAD_STATE_TYPE") or "") defect_types = [DEFECT_TYPE_MAP[t] for t in defect_tokens if t in DEFECT_TYPE_MAP] or None state_note = (row.get("DR_STATE_NOTE") or "").strip() defect_note = state_note or None # ── Positionning ────────────────────────────────────────────────────────── survey_source = (row.get("DR_SOURCE") or "").strip() survey_type = (row.get("DR_SURVEY_TYPE") or "").strip() positionning_source = f"{survey_source} / {survey_type}" if survey_type else survey_source or None pos_note_parts = [] measure_pos = (row.get("DR_MEASURE_POSITION") or "").strip() if measure_pos: pos_note_parts.append(measure_pos) bad_position = _parse_bool(row.get("DR_POSITION", "")) if bad_position: bad_pos_tokens = _parse_defect_tokens(row.get("DR_BAD_POSITION_TYPE") or "") if bad_pos_tokens: pos_note_parts.append("Position: " + ", ".join(bad_pos_tokens)) pos_note_raw = (row.get("DR_POSITION_NOTE") or "").strip() if pos_note_raw: pos_note_parts.append(pos_note_raw) positionning_note = "\n".join(pos_note_parts) or None positionning_date = _parse_date(row.get("DR_BEGIN_LIFE") or "") # ── Altitude ────────────────────────────────────────────────────────────── z_raw = (row.get("DR_Z") or "").strip() altitude = float(z_raw) if z_raw else None # ── Note ───────────────────────────────────────────────────────────────── note_parts = [] dr_note = (row.get("DR_NOTE") or "").strip() model_note = (row.get("DR_MODEL_NOTE") or "").strip() veg_note = (row.get("DR_VEGETATION_NOTE") or "").strip() if dr_note: note_parts.append(dr_note) if model_note: note_parts.append(f"Modèle : {model_note}") if veg_note: note_parts.append(f"Végétation : {veg_note}") note = "\n".join(note_parts) or None # ── Misc ────────────────────────────────────────────────────────────────── related_image_type = (row.get("DR_TYPE") or "").strip() or None brand = None installation_date = None return dict( code=code, street=street, model=drain_model, soil_type=soil_type, drain_location_type=drain_location_type, nearby_vegetation=nearby_vegetation, possible_rooting=possible_rooting, is_defect=is_defect, defect_types=defect_types, defect_note=defect_note, positionning_source=positionning_source, positionning_note=positionning_note, positionning_date=positionning_date, altitude=altitude, related_image_type=related_image_type, brand=brand, note=note, installation_date=installation_date, geom=geom, lon=lon, lat=lat, geojson=geojson, ) # ── Management command ──────────────────────────────────────────────────────── class Command(BaseCommand): help = "Import / update RoadDrain objects from external/avaloirs_2026.csv" def add_arguments(self, parser): parser.add_argument( "--csv", default=CSV_PATH, help="Path to the CSV file (default: external/avaloirs_2026.csv)", ) parser.add_argument( "--dry-run", action="store_true", help="Parse and validate without writing to the database", ) def handle(self, *args, **options): csv_path = options["csv"] dry_run = options["dry_run"] if not os.path.isfile(csv_path): self.stderr.write(self.style.ERROR(f"CSV file not found: {csv_path}")) return # ── Pre-load DB lookups ─────────────────────────────────────────────── street_map = { s.name_fr: s for s in RoadStreet.objects.all() if s.name_fr } self.stdout.write(f" Streets loaded : {len(street_map)}") # Get-or-create drain models keyed by code drain_model_map: dict[str, RoadDrainModel] = { m.code: m for m in RoadDrainModel.objects.all() } self.stdout.write(f" DrainModels loaded : {len(drain_model_map)}") # Existing drains keyed by code existing_drains: dict[str, RoadDrain] = { d.code: d for d in RoadDrain.objects.all() } self.stdout.write(f" Existing drains : {len(existing_drains)}") # ── Read CSV ────────────────────────────────────────────────────────── with open(csv_path, newline="", encoding="utf-8-sig") as f: rows = list(csv.DictReader(f)) self.stdout.write(f" CSV rows : {len(rows)}") # ── Ensure drain models exist for all DR_MODEL values ───────────────── if not dry_run: needed_codes = set() for row in rows: mp = (row.get("DR_MODEL") or "").strip() if mp: needed_codes.add(_extract_model_code(mp)) new_models = [] for code in needed_codes: if code not in drain_model_map: dm = RoadDrainModel(code=code, name_fr=code) new_models.append(dm) if new_models: RoadDrainModel.objects.bulk_create(new_models, batch_size=BULK_BATCH_SIZE) # Reload drain_model_map = {m.code: m for m in RoadDrainModel.objects.all()} self.stdout.write( f" DrainModels created: {len(new_models)} " f"(total {len(drain_model_map)})" ) # ── Parse rows ──────────────────────────────────────────────────────── to_create: list[RoadDrain] = [] to_update: list[RoadDrain] = [] skipped = 0 streets_not_found: set[str] = set() for row in rows: fields = _build_drain(row, street_map, drain_model_map) if fields is None: skipped += 1 continue street_name = (row.get("DR_STREET") or "").strip() if street_name and fields["street"] is None: streets_not_found.add(street_name) code = fields.pop("code") existing = existing_drains.get(code) if existing is None: obj = RoadDrain(code=code, **fields) to_create.append(obj) else: for attr, val in fields.items(): setattr(existing, attr, val) to_update.append(existing) # ── Report unmatched streets ────────────────────────────────────────── for name in sorted(streets_not_found): self.stdout.write( self.style.WARNING(f" Street not found in DB: {name!r}") ) self.stdout.write( f" To create : {len(to_create)}, " f"to update : {len(to_update)}, " f"skipped : {skipped}" ) if dry_run: self.stdout.write(self.style.WARNING(" Dry-run: no changes written.")) return # ── Write to DB ─────────────────────────────────────────────────────── with transaction.atomic(): if to_create: RoadDrain.objects.bulk_create( to_create, batch_size=BULK_BATCH_SIZE ) if to_update: RoadDrain.objects.bulk_update( to_update, UPDATE_FIELDS, batch_size=BULK_BATCH_SIZE ) self.stdout.write( self.style.SUCCESS( f" Done — created {len(to_create)}, updated {len(to_update)} drains." ) )