418 lines
17 KiB
Python
418 lines
17 KiB
Python
"""
|
|
Management command to import / update RoadDrain objects from
|
|
external/avaloirs_2026.csv.
|
|
|
|
CSV → RoadDrain field mapping
|
|
─────────────────────────────
|
|
geom → geom (PointField srid=3812, dim=3) + lon / lat / geojson
|
|
DR_ID → code (e.g. "DR-001")
|
|
DR_STREET → street (ForeignKey to RoadStreet, matched by name_fr)
|
|
DR_SOURCE → positionning_source (with DR_SURVEY_TYPE appended)
|
|
DR_SOILTYPE → soil_type
|
|
DR_LOCATION → drain_location_type
|
|
DR_VEGETATION → nearby_vegetation
|
|
DR_ROOTING → possible_rooting
|
|
DR_STATE → is_defect
|
|
DR_BAD_STATE_TYPE → defect_types (all tokens as list)
|
|
DR_STATE_NOTE → defect_note (appended)
|
|
DR_Z → altitude
|
|
DR_BEGIN_LIFE → installation_date
|
|
DR_MEASURE_POSITION / DR_POSITION / DR_BAD_POSITION_TYPE / DR_POSITION_NOTE
|
|
→ positionning_note (combined)
|
|
DR_MODEL → model (RoadDrainModel, looked-up or created by code)
|
|
DR_MODEL_NOTE / DR_NOTE → note (combined)
|
|
DR_TYPE → related_image_type
|
|
DR_OPERATOR → brand
|
|
"""
|
|
|
|
import csv
|
|
import json
|
|
import os
|
|
import re
|
|
from datetime import datetime
|
|
|
|
from django.contrib.gis.gdal import CoordTransform, SpatialReference
|
|
from django.contrib.gis.geos import Point
|
|
from django.core.management.base import BaseCommand
|
|
from django.db import transaction
|
|
|
|
from assets.models.roads import RoadDrain, RoadDrainModel, RoadStreet
|
|
|
|
# ── Path to CSV ───────────────────────────────────────────────────────────────
|
|
CSV_PATH = os.path.normpath(
|
|
os.path.join(
|
|
os.path.dirname(__file__), # …/assets/management/commands/
|
|
"..", "..", "..", # → repo root
|
|
"scripts", "data", "avaloirs_2026.csv",
|
|
)
|
|
)
|
|
|
|
BULK_BATCH_SIZE = 500
|
|
|
|
# ── Lookup / mapping tables ───────────────────────────────────────────────────
|
|
|
|
SOIL_TYPE_MAP = {
|
|
"CONCRETE": "concrete",
|
|
"ASPHALT": "asphalt",
|
|
"STONE": "stone",
|
|
"CONCRETE SLABS": "concrete_slabs",
|
|
"CONCRETE PAVING": "concrete_paving",
|
|
"STONE SLABS": "stone_slabs",
|
|
"STONE PAVING": "stone_paving",
|
|
"CONCRETE/GRASS": "concrete_grass",
|
|
"GRAVEL": "gravel",
|
|
"GRASS": "grass",
|
|
"EARTH": "earth",
|
|
}
|
|
|
|
LOCATION_TYPE_MAP = {
|
|
"WATERWAY": "waterway",
|
|
"SIDEWALK": "sidewalk",
|
|
"BERME": "berme",
|
|
"SIDEWALK-FACADE TO ROAD": "sidewalk_facade_to_road",
|
|
"SQUARE": "square",
|
|
"ALONG SIDEWALK": "along_sidewalk",
|
|
"MIDDLE ROAD": "middle_road",
|
|
}
|
|
|
|
DEFECT_TYPE_MAP = {
|
|
"OBSTRUCTED": "obstructed",
|
|
"SMELL CUT MISSING": "smell_cut_missing",
|
|
"GRATE BROKEN": "grate_broken",
|
|
"GRATE MISSING": "grate_missing",
|
|
"POOR WATER FLOW": "poor_water_flow",
|
|
"OTHER": "other",
|
|
}
|
|
|
|
# Fields updated on an existing drain (update path)
|
|
UPDATE_FIELDS = [
|
|
"street_id", "model_id", "soil_type", "drain_location_type",
|
|
"nearby_vegetation", "possible_rooting", "is_defect", "defect_types",
|
|
"defect_note", "positionning_source", "positionning_note",
|
|
"positionning_date", "altitude", "related_image_type",
|
|
"brand", "note", "installation_date",
|
|
"geom", "lon", "lat", "geojson",
|
|
]
|
|
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
def _parse_bool(val: str):
|
|
"""Return True/False/None from 'true'/'false'/'' strings."""
|
|
v = (val or "").strip().lower()
|
|
if v == "true":
|
|
return True
|
|
if v == "false":
|
|
return False
|
|
return None
|
|
|
|
|
|
def _parse_defect_tokens(raw: str) -> list[str]:
|
|
"""
|
|
Parse values like '{OBSTRUCTED}', '{GRATE BROKEN, OTHER}', '{}'.
|
|
Returns a list of upper-cased token strings.
|
|
"""
|
|
raw = raw.strip()
|
|
if not raw or raw == "{}":
|
|
return []
|
|
inner = re.sub(r"^\{|\}$", "", raw)
|
|
return [t.strip() for t in inner.split(",") if t.strip()]
|
|
|
|
|
|
def _extract_model_code(model_path: str) -> str:
|
|
"""'MODEL/AV/AV_TYPE-002.png' → 'AV_TYPE-002'"""
|
|
return os.path.splitext(os.path.basename(model_path))[0]
|
|
|
|
|
|
def _parse_geom(wkt: str):
|
|
"""
|
|
Parse 'POINT ZM (x y z m)' (already in EPSG:3812) into a 3-D Point.
|
|
Returns a GEOSGeometry Point(srid=3812) or None on failure.
|
|
"""
|
|
m = re.match(
|
|
r"POINT\s+ZM\s*\(\s*([\d.+-]+)\s+([\d.+-]+)\s+([\d.+-]+)\s+([\d.+-]+)\s*\)",
|
|
wkt,
|
|
re.IGNORECASE,
|
|
)
|
|
if not m:
|
|
return None
|
|
x, y, z = float(m.group(1)), float(m.group(2)), float(m.group(3))
|
|
return Point(x, y, z, srid=3812)
|
|
|
|
|
|
def _lon_lat_geojson(geom_3812):
|
|
"""Transform a 3812 Point to WGS84, return (lon, lat, geojson_str)."""
|
|
srs_3812 = SpatialReference(3812)
|
|
srs_4326 = SpatialReference(4326)
|
|
ct = CoordTransform(srs_3812, srs_4326)
|
|
g = geom_3812.clone()
|
|
g.transform(ct)
|
|
lon, lat = round(g.x, 8), round(g.y, 8)
|
|
geojson = json.dumps({"type": "Point", "coordinates": [lon, lat]})
|
|
return lon, lat, geojson
|
|
|
|
|
|
def _parse_date(raw: str):
|
|
"""Parse 'YYYY/MM/DD' → date, or None."""
|
|
raw = (raw or "").strip()
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return datetime.strptime(raw, "%Y/%m/%d").date()
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _build_drain(row: dict, street_map: dict, drain_model_map: dict) -> dict:
|
|
"""
|
|
Convert one CSV row into a dict of RoadDrain field values.
|
|
Returns None if the row must be skipped.
|
|
"""
|
|
if (row.get("DR_TYPE") or "").strip().upper() != "DRAIN POINT":
|
|
return None
|
|
|
|
dr_id = (row.get("DR_ID") or "").strip()
|
|
if not dr_id:
|
|
return None
|
|
|
|
code = f"DR-{int(dr_id):05d}"
|
|
|
|
# ── Geometry ──────────────────────────────────────────────────────────────
|
|
wkt = (row.get("geom") or "").strip()
|
|
geom = _parse_geom(wkt) if wkt else None
|
|
if geom is None:
|
|
return None
|
|
lon, lat, geojson = _lon_lat_geojson(geom)
|
|
|
|
# ── Street ────────────────────────────────────────────────────────────────
|
|
# street_name = (row.get("DR_STREET") or "").strip()
|
|
# street = street_map.get(street_name) # may be None
|
|
street = None
|
|
|
|
# ── Drain model ───────────────────────────────────────────────────────────
|
|
model_path = (row.get("DR_MODEL") or "").strip()
|
|
drain_model = None
|
|
if model_path:
|
|
model_code = _extract_model_code(model_path)
|
|
drain_model = drain_model_map.get(model_code)
|
|
|
|
# ── Soil type ─────────────────────────────────────────────────────────────
|
|
soil_raw = (row.get("DR_SOILTYPE") or "").strip().upper()
|
|
soil_type = SOIL_TYPE_MAP.get(soil_raw)
|
|
|
|
# ── Location type ─────────────────────────────────────────────────────────
|
|
loc_raw = (row.get("DR_LOCATION") or "").strip().upper()
|
|
drain_location_type = LOCATION_TYPE_MAP.get(loc_raw)
|
|
|
|
# ── Boolean flags ─────────────────────────────────────────────────────────
|
|
nearby_vegetation = _parse_bool(row.get("DR_VEGETATION", ""))
|
|
possible_rooting = _parse_bool(row.get("DR_ROOTING", ""))
|
|
is_defect = _parse_bool(row.get("DR_STATE", ""))
|
|
|
|
# ── Defect type / note ────────────────────────────────────────────────────
|
|
defect_tokens = _parse_defect_tokens(row.get("DR_BAD_STATE_TYPE") or "")
|
|
defect_types = [DEFECT_TYPE_MAP[t] for t in defect_tokens if t in DEFECT_TYPE_MAP] or None
|
|
|
|
state_note = (row.get("DR_STATE_NOTE") or "").strip()
|
|
defect_note = state_note or None
|
|
|
|
# ── Positionning ──────────────────────────────────────────────────────────
|
|
survey_source = (row.get("DR_SOURCE") or "").strip()
|
|
survey_type = (row.get("DR_SURVEY_TYPE") or "").strip()
|
|
positionning_source = f"{survey_source} / {survey_type}" if survey_type else survey_source or None
|
|
|
|
pos_note_parts = []
|
|
measure_pos = (row.get("DR_MEASURE_POSITION") or "").strip()
|
|
if measure_pos:
|
|
pos_note_parts.append(measure_pos)
|
|
bad_position = _parse_bool(row.get("DR_POSITION", ""))
|
|
if bad_position:
|
|
bad_pos_tokens = _parse_defect_tokens(row.get("DR_BAD_POSITION_TYPE") or "")
|
|
if bad_pos_tokens:
|
|
pos_note_parts.append("Position: " + ", ".join(bad_pos_tokens))
|
|
pos_note_raw = (row.get("DR_POSITION_NOTE") or "").strip()
|
|
if pos_note_raw:
|
|
pos_note_parts.append(pos_note_raw)
|
|
positionning_note = "\n".join(pos_note_parts) or None
|
|
|
|
positionning_date = _parse_date(row.get("DR_BEGIN_LIFE") or "")
|
|
|
|
# ── Altitude ──────────────────────────────────────────────────────────────
|
|
z_raw = (row.get("DR_Z") or "").strip()
|
|
altitude = float(z_raw) if z_raw else None
|
|
|
|
# ── Note ─────────────────────────────────────────────────────────────────
|
|
note_parts = []
|
|
dr_note = (row.get("DR_NOTE") or "").strip()
|
|
model_note = (row.get("DR_MODEL_NOTE") or "").strip()
|
|
veg_note = (row.get("DR_VEGETATION_NOTE") or "").strip()
|
|
if dr_note:
|
|
note_parts.append(dr_note)
|
|
if model_note:
|
|
note_parts.append(f"Modèle : {model_note}")
|
|
if veg_note:
|
|
note_parts.append(f"Végétation : {veg_note}")
|
|
note = "\n".join(note_parts) or None
|
|
|
|
# ── Misc ──────────────────────────────────────────────────────────────────
|
|
related_image_type = (row.get("DR_TYPE") or "").strip() or None
|
|
brand = None
|
|
installation_date = None
|
|
|
|
return dict(
|
|
code=code,
|
|
street=street,
|
|
model=drain_model,
|
|
soil_type=soil_type,
|
|
drain_location_type=drain_location_type,
|
|
nearby_vegetation=nearby_vegetation,
|
|
possible_rooting=possible_rooting,
|
|
is_defect=is_defect,
|
|
defect_types=defect_types,
|
|
defect_note=defect_note,
|
|
positionning_source=positionning_source,
|
|
positionning_note=positionning_note,
|
|
positionning_date=positionning_date,
|
|
altitude=altitude,
|
|
related_image_type=related_image_type,
|
|
brand=brand,
|
|
note=note,
|
|
installation_date=installation_date,
|
|
geom=geom,
|
|
lon=lon,
|
|
lat=lat,
|
|
geojson=geojson,
|
|
)
|
|
|
|
|
|
# ── Management command ────────────────────────────────────────────────────────
|
|
|
|
class Command(BaseCommand):
|
|
help = "Import / update RoadDrain objects from external/avaloirs_2026.csv"
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
"--csv",
|
|
default=CSV_PATH,
|
|
help="Path to the CSV file (default: external/avaloirs_2026.csv)",
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
action="store_true",
|
|
help="Parse and validate without writing to the database",
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
csv_path = options["csv"]
|
|
dry_run = options["dry_run"]
|
|
|
|
if not os.path.isfile(csv_path):
|
|
self.stderr.write(self.style.ERROR(f"CSV file not found: {csv_path}"))
|
|
return
|
|
|
|
# ── Pre-load DB lookups ───────────────────────────────────────────────
|
|
street_map = {
|
|
s.name_fr: s
|
|
for s in RoadStreet.objects.all()
|
|
if s.name_fr
|
|
}
|
|
self.stdout.write(f" Streets loaded : {len(street_map)}")
|
|
|
|
# Get-or-create drain models keyed by code
|
|
drain_model_map: dict[str, RoadDrainModel] = {
|
|
m.code: m for m in RoadDrainModel.objects.all()
|
|
}
|
|
self.stdout.write(f" DrainModels loaded : {len(drain_model_map)}")
|
|
|
|
# Existing drains keyed by code
|
|
existing_drains: dict[str, RoadDrain] = {
|
|
d.code: d for d in RoadDrain.objects.all()
|
|
}
|
|
self.stdout.write(f" Existing drains : {len(existing_drains)}")
|
|
|
|
# ── Read CSV ──────────────────────────────────────────────────────────
|
|
with open(csv_path, newline="", encoding="utf-8-sig") as f:
|
|
rows = list(csv.DictReader(f))
|
|
self.stdout.write(f" CSV rows : {len(rows)}")
|
|
|
|
# ── Ensure drain models exist for all DR_MODEL values ─────────────────
|
|
if not dry_run:
|
|
needed_codes = set()
|
|
for row in rows:
|
|
mp = (row.get("DR_MODEL") or "").strip()
|
|
if mp:
|
|
needed_codes.add(_extract_model_code(mp))
|
|
|
|
new_models = []
|
|
for code in needed_codes:
|
|
if code not in drain_model_map:
|
|
dm = RoadDrainModel(code=code, name_fr=code)
|
|
new_models.append(dm)
|
|
if new_models:
|
|
RoadDrainModel.objects.bulk_create(new_models, batch_size=BULK_BATCH_SIZE)
|
|
# Reload
|
|
drain_model_map = {m.code: m for m in RoadDrainModel.objects.all()}
|
|
self.stdout.write(
|
|
f" DrainModels created: {len(new_models)} "
|
|
f"(total {len(drain_model_map)})"
|
|
)
|
|
|
|
# ── Parse rows ────────────────────────────────────────────────────────
|
|
to_create: list[RoadDrain] = []
|
|
to_update: list[RoadDrain] = []
|
|
skipped = 0
|
|
streets_not_found: set[str] = set()
|
|
|
|
for row in rows:
|
|
fields = _build_drain(row, street_map, drain_model_map)
|
|
if fields is None:
|
|
skipped += 1
|
|
continue
|
|
|
|
street_name = (row.get("DR_STREET") or "").strip()
|
|
if street_name and fields["street"] is None:
|
|
streets_not_found.add(street_name)
|
|
|
|
code = fields.pop("code")
|
|
existing = existing_drains.get(code)
|
|
|
|
if existing is None:
|
|
obj = RoadDrain(code=code, **fields)
|
|
to_create.append(obj)
|
|
else:
|
|
for attr, val in fields.items():
|
|
setattr(existing, attr, val)
|
|
to_update.append(existing)
|
|
|
|
# ── Report unmatched streets ──────────────────────────────────────────
|
|
for name in sorted(streets_not_found):
|
|
self.stdout.write(
|
|
self.style.WARNING(f" Street not found in DB: {name!r}")
|
|
)
|
|
|
|
self.stdout.write(
|
|
f" To create : {len(to_create)}, "
|
|
f"to update : {len(to_update)}, "
|
|
f"skipped : {skipped}"
|
|
)
|
|
|
|
if dry_run:
|
|
self.stdout.write(self.style.WARNING(" Dry-run: no changes written."))
|
|
return
|
|
|
|
# ── Write to DB ───────────────────────────────────────────────────────
|
|
with transaction.atomic():
|
|
if to_create:
|
|
RoadDrain.objects.bulk_create(
|
|
to_create, batch_size=BULK_BATCH_SIZE
|
|
)
|
|
if to_update:
|
|
RoadDrain.objects.bulk_update(
|
|
to_update, UPDATE_FIELDS, batch_size=BULK_BATCH_SIZE
|
|
)
|
|
|
|
self.stdout.write(
|
|
self.style.SUCCESS(
|
|
f" Done — created {len(to_create)}, updated {len(to_update)} drains."
|
|
)
|
|
)
|