909 lines
36 KiB
Python
909 lines
36 KiB
Python
"""
|
||
Commande Django pour importer des assets depuis un fichier Excel.
|
||
|
||
La commande lit un fichier .xlsx dont les colonnes sont :
|
||
thématique | nom_modèle_objet | code | nom_fr | nom_nl |
|
||
code_catégorie | code_modèle | code_localisation | geojson |
|
||
lon | lat | statut | codes_parents | codes_enfants |
|
||
marque | date_installation | structure
|
||
|
||
Usage :
|
||
python manage.py import_assets_from_excel chemin/vers/fichier.xlsx
|
||
python manage.py import_assets_from_excel chemin/vers/fichier.xlsx --dry-run
|
||
python manage.py import_assets_from_excel chemin/vers/fichier.xlsx --update
|
||
python manage.py import_assets_from_excel --generate-template [--template-output assets_template.xlsx]
|
||
"""
|
||
|
||
import ast
|
||
import json
|
||
import re
|
||
from datetime import datetime
|
||
|
||
import openpyxl
|
||
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
||
from openpyxl.utils import get_column_letter
|
||
|
||
from django.apps import apps
|
||
from django.contrib.contenttypes.models import ContentType
|
||
from django.contrib.gis.geos import GEOSGeometry, Point
|
||
from django.core.management.base import BaseCommand, CommandError
|
||
from django.db import transaction
|
||
from django.utils import timezone
|
||
|
||
from assets.models import (
|
||
AbstractAsset,
|
||
AbstractGeoAsset,
|
||
AbstractLocation,
|
||
AssetCategory,
|
||
ASSET_STATUS_CHOICES,
|
||
)
|
||
from common.models import Thematic
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Column definitions (normalized header → Excel label)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
COLUMNS = [
|
||
("thématique", "Thématique (code)"),
|
||
("nom_modèle_objet", "Nom du modèle d'objet (ex: structuregeoasset)"),
|
||
("code", "Code"),
|
||
("nom_fr", "Nom FR"),
|
||
("nom_nl", "Nom NL"),
|
||
("code_catégorie", "Code de la catégorie"),
|
||
("code_modèle", "Code du modèle"),
|
||
("code_localisation", "Code de la localisation"),
|
||
("geojson", "GeoJSON"),
|
||
("lon", "Lon"),
|
||
("lat", "Lat"),
|
||
("statut", "Statut"),
|
||
("codes_parents", "Codes des assets parents ( [code1, code2] )"),
|
||
("codes_enfants", "Codes des assets enfants ( [code1, code2] )"),
|
||
("marque", "Marque"),
|
||
("date_installation", "Date d'installation (YYYY-MM-DD)"),
|
||
("structure", "Structure (code)"),
|
||
]
|
||
|
||
HEADER_KEYS = [col[0] for col in COLUMNS]
|
||
|
||
COLUMN_HINTS = {
|
||
"thématique": "Code de la thématique, ex: structures, trafficlights, controlcenters…",
|
||
"nom_modèle_objet": "Nom du modèle Django en minuscules, ex: structuregeoasset, itsgeoasset, controlcentergeoasset",
|
||
"code": "Code unique de l'asset",
|
||
"nom_fr": "Nom en français",
|
||
"nom_nl": "Naam in het Nederlands",
|
||
"code_catégorie": "Code de l'AssetCategory",
|
||
"code_modèle": "Code du modèle d'asset (StructureAssetModel.code, etc.)",
|
||
"code_localisation": "Code de la localisation (StructureLocation.code, ITSLocation.code…)",
|
||
"geojson": 'GeoJSON valide (type Point/Polygon…), ex: {"type":"Point","coordinates":[4.35,50.85]}',
|
||
"lon": "Longitude (WGS84) – facultatif si geojson fourni",
|
||
"lat": "Latitude (WGS84) – facultatif si geojson fourni",
|
||
"statut": "active | planned | damaged | to_replace | removed | to_be_approved",
|
||
"codes_parents": 'Codes des assets parents entre crochets, ex: [CODE1, CODE2]',
|
||
"codes_enfants": 'Codes des assets enfants entre crochets, ex: [CODE3]',
|
||
"marque": "Marque / fabricant",
|
||
"date_installation": "Date d'installation au format YYYY-MM-DD",
|
||
"structure": "Code de l'objet Structure (uniquement pour StructureGeoAsset)",
|
||
}
|
||
|
||
EXAMPLE_ROW = {
|
||
"thématique": "structures",
|
||
"nom_modèle_objet": "structuregeoasset",
|
||
"code": "STR-001",
|
||
"nom_fr": "Pont de l'exemple",
|
||
"nom_nl": "Voorbeeldbrug",
|
||
"code_catégorie": "CAT_STRUCTURE",
|
||
"code_modèle": "MOD_PONT",
|
||
"code_localisation": "LOC_001",
|
||
"geojson": '{"type":"Point","coordinates":[4.3517,50.8503]}',
|
||
"lon": "4.3517",
|
||
"lat": "50.8503",
|
||
"statut": "active",
|
||
"codes_parents": "[STR-000]",
|
||
"codes_enfants": "[STR-002, STR-003]",
|
||
"marque": "Fabricant SA",
|
||
"date_installation": "2024-01-15",
|
||
"structure": "OA-001",
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _normalize_key(s: str) -> str:
|
||
"""Lowercases and strips a string for column matching."""
|
||
return s.strip().lower().replace(" ", "_").replace("'", "_").replace("(", "").replace(")", "")
|
||
|
||
|
||
def _parse_codes_list(raw: str) -> list[str]:
|
||
"""Parse '[CODE1, CODE2]' or 'CODE1,CODE2' into a list of strings."""
|
||
if not raw:
|
||
return []
|
||
raw = str(raw).strip()
|
||
if raw.startswith("["):
|
||
# Remove brackets and split
|
||
inner = raw.strip("[]")
|
||
parts = [p.strip() for p in inner.split(",") if p.strip()]
|
||
return parts
|
||
return [p.strip() for p in raw.split(",") if p.strip()]
|
||
|
||
|
||
def _parse_date(raw) -> datetime | None:
|
||
"""Try to parse a date string (YYYY-MM-DD or DD/MM/YYYY)."""
|
||
if not raw:
|
||
return None
|
||
if isinstance(raw, datetime):
|
||
return raw
|
||
raw = str(raw).strip()
|
||
for fmt in ("%Y-%m-%d", "%d/%m/%Y", "%d-%m-%Y", "%Y/%m/%d"):
|
||
try:
|
||
return datetime.strptime(raw, fmt)
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _get_model_class(model_name: str):
|
||
"""Return the Django model class for a given model name (case-insensitive)."""
|
||
model_name_lower = model_name.strip().lower()
|
||
cts = ContentType.objects.filter(model=model_name_lower)
|
||
if not cts.exists():
|
||
return None
|
||
# Prefer assets app
|
||
cts_sorted = sorted(cts, key=lambda ct: (0 if ct.app_label == "assets" else 1))
|
||
for ct in cts_sorted:
|
||
mc = ct.model_class()
|
||
if mc is not None:
|
||
return mc
|
||
return None
|
||
|
||
|
||
def _find_asset_by_code(code: str):
|
||
"""Search all AbstractAsset subclasses for an asset with the given code."""
|
||
for model in apps.get_models():
|
||
if model._meta.abstract:
|
||
continue
|
||
try:
|
||
if not issubclass(model, AbstractAsset):
|
||
continue
|
||
except TypeError:
|
||
continue
|
||
obj = model.objects.filter(code=code).first()
|
||
if obj:
|
||
return obj
|
||
return None
|
||
|
||
|
||
def _find_location_by_code(model_class, code: str):
|
||
"""
|
||
Find a location instance for an asset model class by code.
|
||
Looks at FK fields that point to AbstractLocation subclasses.
|
||
"""
|
||
for field in model_class._meta.get_fields():
|
||
if not hasattr(field, "related_model") or field.related_model is None:
|
||
continue
|
||
if not hasattr(field, "column"):
|
||
continue # skip reverse relations
|
||
try:
|
||
if issubclass(field.related_model, AbstractLocation):
|
||
location = field.related_model.objects.filter(code=code).first()
|
||
if location:
|
||
return field.name, location
|
||
except TypeError:
|
||
continue
|
||
return None, None
|
||
|
||
|
||
def _find_model_instance_by_code(model_class, code: str):
|
||
"""
|
||
Find an AssetModel instance for an asset class by code.
|
||
Looks for the 'model' FK field.
|
||
"""
|
||
try:
|
||
model_field = model_class._meta.get_field("model")
|
||
related_model = model_field.related_model
|
||
return related_model.objects.filter(code=code).first()
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _set_geom_from_geojson(asset, geojson_str: str):
|
||
"""Parse a GeoJSON string and set asset.geom (in SRID 3812)."""
|
||
try:
|
||
geom_data = json.loads(geojson_str)
|
||
if geom_data.get("type") == "Feature":
|
||
geom_json = json.dumps(geom_data.get("geometry", {}))
|
||
else:
|
||
geom_json = json.dumps(geom_data)
|
||
geom = GEOSGeometry(geom_json, srid=4326)
|
||
geom.transform(3812)
|
||
asset.geom = geom
|
||
# Update lon/lat from geometry centroid
|
||
geom_wgs84 = geom.transform(4326, clone=True)
|
||
if geom_wgs84.geom_type == "Point":
|
||
asset.lon = geom_wgs84.x
|
||
asset.lat = geom_wgs84.y
|
||
else:
|
||
centroid = geom_wgs84.centroid
|
||
asset.lon = centroid.x
|
||
asset.lat = centroid.y
|
||
return True
|
||
except Exception as e:
|
||
return False
|
||
|
||
|
||
def _set_geom_from_lon_lat(asset, lon: float, lat: float):
|
||
"""Set asset.geom (in SRID 3812) from WGS84 lon/lat."""
|
||
pt = Point(lon, lat, srid=4326)
|
||
pt.transform(3812)
|
||
asset.geom = pt
|
||
asset.lon = lon
|
||
asset.lat = lat
|
||
|
||
|
||
def _set_parent_fk(asset, parent_asset, model_class, excluded=None):
|
||
"""
|
||
Find the FK field on 'asset' (model_class) that points to the type of
|
||
'parent_asset' and set it.
|
||
Returns (True, field_name) on success or (False, tried_field_names) on failure.
|
||
"""
|
||
if excluded is None:
|
||
excluded = {"model", "category", "replaced_by", "marked_as_removed_by", "updated_by",
|
||
"marked_as_removed_by"}
|
||
parent_class = type(parent_asset)
|
||
tried = []
|
||
for field in model_class._meta.get_fields():
|
||
if not hasattr(field, "related_model") or field.related_model is None:
|
||
continue
|
||
if not hasattr(field, "column"):
|
||
continue # skip reverse relations
|
||
if field.name in excluded:
|
||
continue
|
||
# Only consider FK fields pointing to AbstractAsset subclasses
|
||
try:
|
||
if not issubclass(field.related_model, AbstractAsset):
|
||
continue
|
||
except TypeError:
|
||
continue
|
||
tried.append(f"{field.name}→{field.related_model.__name__}")
|
||
try:
|
||
if issubclass(parent_class, field.related_model):
|
||
setattr(asset, field.name, parent_asset)
|
||
return True, field.name
|
||
except TypeError:
|
||
continue
|
||
return False, tried
|
||
|
||
|
||
def _set_child_fk(child_asset, current_asset, excluded=None):
|
||
"""
|
||
Find the FK field on 'child_asset' that points to the type of 'current_asset'
|
||
and set it on the child.
|
||
Returns (True, field_name) on success or (False, tried_field_names) on failure.
|
||
"""
|
||
if excluded is None:
|
||
excluded = {"model", "category", "replaced_by", "marked_as_removed_by", "updated_by",
|
||
"marked_as_removed_by"}
|
||
current_class = type(current_asset)
|
||
child_class = type(child_asset)
|
||
tried = []
|
||
for field in child_class._meta.get_fields():
|
||
if not hasattr(field, "related_model") or field.related_model is None:
|
||
continue
|
||
if not hasattr(field, "column"):
|
||
continue
|
||
if field.name in excluded:
|
||
continue
|
||
# Only consider FK fields pointing to AbstractAsset subclasses
|
||
try:
|
||
if not issubclass(field.related_model, AbstractAsset):
|
||
continue
|
||
except TypeError:
|
||
continue
|
||
tried.append(f"{field.name}→{field.related_model.__name__}")
|
||
try:
|
||
if issubclass(current_class, field.related_model):
|
||
setattr(child_asset, field.name, current_asset)
|
||
return True, field.name
|
||
except TypeError:
|
||
continue
|
||
return False, tried
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Excel template generation
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def generate_template(output_path: str):
|
||
"""Generate a nicely formatted Excel template file."""
|
||
wb = openpyxl.Workbook()
|
||
ws_data = wb.active
|
||
ws_data.title = "Assets"
|
||
|
||
# --- Styles ---
|
||
header_fill = PatternFill("solid", fgColor="1F4E79")
|
||
hint_fill = PatternFill("solid", fgColor="D6E4F0")
|
||
example_fill = PatternFill("solid", fgColor="E2EFDA")
|
||
header_font = Font(bold=True, color="FFFFFF", size=11)
|
||
hint_font = Font(italic=True, color="555555", size=9)
|
||
example_font = Font(color="375623", size=10)
|
||
wrap = Alignment(wrap_text=True, vertical="top")
|
||
thin_border_side = Side(border_style="thin", color="AAAAAA")
|
||
thin_border = Border(
|
||
left=thin_border_side, right=thin_border_side,
|
||
top=thin_border_side, bottom=thin_border_side,
|
||
)
|
||
|
||
# Row 1 – Column keys (hidden row used for mapping)
|
||
for col_idx, (key, label) in enumerate(COLUMNS, start=1):
|
||
cell = ws_data.cell(row=1, column=col_idx, value=key)
|
||
cell.font = Font(size=8, color="999999")
|
||
cell.fill = PatternFill("solid", fgColor="F2F2F2")
|
||
|
||
# Row 2 – Human-readable headers
|
||
for col_idx, (key, label) in enumerate(COLUMNS, start=1):
|
||
cell = ws_data.cell(row=2, column=col_idx, value=label)
|
||
cell.font = header_font
|
||
cell.fill = header_fill
|
||
cell.alignment = wrap
|
||
cell.border = thin_border
|
||
|
||
# Row 3 – Hints
|
||
for col_idx, key in enumerate(HEADER_KEYS, start=1):
|
||
cell = ws_data.cell(row=3, column=col_idx, value=COLUMN_HINTS.get(key, ""))
|
||
cell.font = hint_font
|
||
cell.fill = hint_fill
|
||
cell.alignment = wrap
|
||
cell.border = thin_border
|
||
|
||
# Row 4 – Example
|
||
for col_idx, key in enumerate(HEADER_KEYS, start=1):
|
||
cell = ws_data.cell(row=4, column=col_idx, value=EXAMPLE_ROW.get(key, ""))
|
||
cell.font = example_font
|
||
cell.fill = example_fill
|
||
cell.alignment = wrap
|
||
cell.border = thin_border
|
||
|
||
# Row 5+ – Empty data rows
|
||
for row in range(5, 25):
|
||
for col_idx in range(1, len(COLUMNS) + 1):
|
||
cell = ws_data.cell(row=row, column=col_idx, value="")
|
||
cell.border = thin_border
|
||
cell.alignment = Alignment(vertical="top")
|
||
|
||
# Column widths
|
||
COLUMN_WIDTHS = {
|
||
1: 20, # thématique
|
||
2: 35, # nom_modèle_objet
|
||
3: 20, # code
|
||
4: 30, # nom_fr
|
||
5: 30, # nom_nl
|
||
6: 25, # code_catégorie
|
||
7: 25, # code_modèle
|
||
8: 25, # code_localisation
|
||
9: 50, # geojson
|
||
10: 12, # lon
|
||
11: 12, # lat
|
||
12: 18, # statut
|
||
13: 35, # codes_parents
|
||
14: 35, # codes_enfants
|
||
15: 25, # marque
|
||
16: 22, # date_installation
|
||
17: 20, # structure
|
||
}
|
||
for col_idx, width in COLUMN_WIDTHS.items():
|
||
ws_data.column_dimensions[get_column_letter(col_idx)].width = width
|
||
|
||
ws_data.row_dimensions[1].height = 14
|
||
ws_data.row_dimensions[2].height = 30
|
||
ws_data.row_dimensions[3].height = 50
|
||
ws_data.row_dimensions[4].height = 25
|
||
|
||
# Freeze panes below row 4 and after column A
|
||
ws_data.freeze_panes = "B5"
|
||
|
||
# --- Second sheet: reference values ---
|
||
ws_ref = wb.create_sheet("Référence")
|
||
ref_header = Font(bold=True, color="FFFFFF")
|
||
ref_fill = PatternFill("solid", fgColor="2E75B6")
|
||
|
||
ws_ref["A1"] = "Valeurs valides pour le champ 'statut'"
|
||
ws_ref["A1"].font = ref_header
|
||
ws_ref["A1"].fill = ref_fill
|
||
ws_ref["A1"].alignment = Alignment(horizontal="center")
|
||
ws_ref.merge_cells("A1:B1")
|
||
|
||
ws_ref["A2"] = "Clé"
|
||
ws_ref["B2"] = "Libellé"
|
||
ws_ref["A2"].font = Font(bold=True)
|
||
ws_ref["B2"].font = Font(bold=True)
|
||
|
||
status_labels = {
|
||
"to_be_approved": "À approuver",
|
||
"planned": "Planifié",
|
||
"active": "Actif",
|
||
"damaged": "Endommagé",
|
||
"to_replace": "À remplacer",
|
||
"removed": "Enlevé",
|
||
}
|
||
for row_idx, (k, v) in enumerate(status_labels.items(), start=3):
|
||
ws_ref.cell(row=row_idx, column=1, value=k)
|
||
ws_ref.cell(row=row_idx, column=2, value=v)
|
||
|
||
row_after_status = 3 + len(status_labels) + 2
|
||
|
||
ws_ref.cell(row=row_after_status, column=1, value="Exemples de modèles d'objets (nom_modèle_objet)").font = Font(bold=True)
|
||
ws_ref.merge_cells(f"A{row_after_status}:B{row_after_status}")
|
||
|
||
examples = [
|
||
("structuregeoasset", "StructureGeoAsset – Ouvrages d'art"),
|
||
("itsgeoasset", "ITSGeoAsset – ITS"),
|
||
("controlcentergeoasset", "ControlCenterGeoAsset – Centres de contrôle"),
|
||
("trafficlightpole", "TrafficLightPole – Feux de circulation (poteaux)"),
|
||
("trafficlightlantern", "TrafficLightLantern – Lanternes"),
|
||
("trafficlightcontroller","TrafficLightController – Régulateurs"),
|
||
("trafficlightdetector", "TrafficLightDetector – Détecteurs"),
|
||
("trafficlightcabinet", "TrafficLightElectricalCabinet – Armoires"),
|
||
("publiclightingpole", "PublicLightingPole – Éclairage public (mâts)"),
|
||
("publiclightinglightfixture", "PublicLightingLightFixture – Luminaires"),
|
||
("naturetree", "NatureTree – Arbres"),
|
||
("natureshrub", "NatureShrub – Arbustes"),
|
||
("naturebench", "NatureBench – Bancs"),
|
||
("naturegreensurface", "NatureGreenSurface – Surfaces vertes"),
|
||
("roadfurniture", "RoadFurniture – Mobilier routier"),
|
||
]
|
||
for row_idx, (model, desc) in enumerate(examples, start=row_after_status + 1):
|
||
ws_ref.cell(row=row_idx, column=1, value=model)
|
||
ws_ref.cell(row=row_idx, column=2, value=desc)
|
||
|
||
ws_ref.column_dimensions["A"].width = 35
|
||
ws_ref.column_dimensions["B"].width = 50
|
||
|
||
wb.save(output_path)
|
||
return output_path
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Main command
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class Command(BaseCommand):
|
||
help = (
|
||
"Importe des assets depuis un fichier Excel (.xlsx). "
|
||
"Utilisez --generate-template pour créer le fichier modèle."
|
||
)
|
||
|
||
def add_arguments(self, parser):
|
||
parser.add_argument(
|
||
"excel_file",
|
||
nargs="?",
|
||
type=str,
|
||
help="Chemin vers le fichier Excel à importer.",
|
||
)
|
||
parser.add_argument(
|
||
"--dry-run",
|
||
action="store_true",
|
||
default=False,
|
||
help="Simule l'import sans écrire dans la base de données.",
|
||
)
|
||
parser.add_argument(
|
||
"--update",
|
||
action="store_true",
|
||
default=False,
|
||
help="Met à jour les assets existants si le code existe déjà.",
|
||
)
|
||
parser.add_argument(
|
||
"--generate-template",
|
||
action="store_true",
|
||
default=False,
|
||
help="Génère un fichier Excel modèle.",
|
||
)
|
||
parser.add_argument(
|
||
"--template-output",
|
||
type=str,
|
||
default="assets_import_template.xlsx",
|
||
help="Chemin de sortie du fichier modèle (défaut: assets_import_template.xlsx).",
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
def handle(self, *args, **options):
|
||
# --- Template generation mode ---
|
||
if options["generate_template"]:
|
||
output = options["template_output"]
|
||
path = generate_template(output)
|
||
self.stdout.write(self.style.SUCCESS(f"✓ Template généré: {path}"))
|
||
return
|
||
|
||
# --- Import mode ---
|
||
excel_file = options.get("excel_file")
|
||
if not excel_file:
|
||
raise CommandError(
|
||
"Veuillez fournir le chemin du fichier Excel, "
|
||
"ou utilisez --generate-template pour créer le modèle."
|
||
)
|
||
|
||
dry_run = options["dry_run"]
|
||
update = options["update"]
|
||
|
||
if dry_run:
|
||
self.stdout.write(self.style.WARNING("⚠ Mode dry-run activé – aucune donnée ne sera enregistrée."))
|
||
|
||
self._import(excel_file, dry_run=dry_run, update=update)
|
||
|
||
# ------------------------------------------------------------------
|
||
def _import(self, excel_file: str, dry_run: bool, update: bool):
|
||
try:
|
||
wb = openpyxl.load_workbook(excel_file, data_only=True)
|
||
except FileNotFoundError:
|
||
raise CommandError(f"Fichier introuvable: {excel_file}")
|
||
except Exception as e:
|
||
raise CommandError(f"Impossible d'ouvrir le fichier: {e}")
|
||
|
||
ws = wb.active
|
||
|
||
# --- Detect header row ---
|
||
header_row_idx, col_map, data_start_row = self._detect_header(ws)
|
||
if header_row_idx is None:
|
||
raise CommandError(
|
||
"Impossible de trouver la ligne d'en-tête. "
|
||
"Assurez-vous que la ligne 1 ou 2 contient les clés de colonnes."
|
||
)
|
||
|
||
self.stdout.write(f"→ En-tête détectée à la ligne {header_row_idx}, données à partir de la ligne {data_start_row}.")
|
||
|
||
# Collect data rows (skip header, hint and example rows)
|
||
data_rows = []
|
||
for row in ws.iter_rows(min_row=data_start_row, values_only=True):
|
||
# Skip completely empty rows
|
||
if all(cell is None or str(cell).strip() == "" for cell in row):
|
||
continue
|
||
data_rows.append(row)
|
||
|
||
self.stdout.write(f"→ {len(data_rows)} ligne(s) à traiter.")
|
||
|
||
# Two-pass approach
|
||
# Pass 1: create / update assets
|
||
# Pass 2: resolve parent/child FK links
|
||
created_assets = {} # row_index → asset instance
|
||
rows_data = [] # store parsed row dicts
|
||
|
||
errors = []
|
||
stats = {"created": 0, "updated": 0, "skipped": 0, "errors": 0}
|
||
|
||
# Consume data_rows into parsed dicts, skip hint row(s) written by template
|
||
parsed_rows = []
|
||
for row in data_rows:
|
||
row_dict = {}
|
||
for key, col_idx in col_map.items():
|
||
value = row[col_idx] if col_idx < len(row) else None
|
||
row_dict[key] = value if value is not None else ""
|
||
# Skip if 'code' is empty
|
||
if not str(row_dict.get("code", "")).strip():
|
||
continue
|
||
parsed_rows.append(row_dict)
|
||
|
||
self.stdout.write(f"→ {len(parsed_rows)} ligne(s) valides (avec code).")
|
||
|
||
with transaction.atomic():
|
||
# ------ PASS 1: create / update assets ------
|
||
for line_num, row_dict in enumerate(parsed_rows, start=1):
|
||
try:
|
||
asset, action = self._process_row(row_dict, update=update, dry_run=dry_run)
|
||
if asset is not None:
|
||
created_assets[line_num] = asset
|
||
rows_data.append((line_num, row_dict))
|
||
stats[action] += 1
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [{action.upper()}] code={row_dict['code']}"
|
||
)
|
||
else:
|
||
stats["skipped"] += 1
|
||
self.stdout.write(
|
||
self.style.WARNING(f" Ligne {line_num}: [IGNORÉ] code={row_dict['code']}")
|
||
)
|
||
except Exception as e:
|
||
stats["errors"] += 1
|
||
msg = f" Ligne {line_num}: [ERREUR] code={row_dict.get('code', '?')} – {e}"
|
||
self.stderr.write(self.style.ERROR(msg))
|
||
errors.append(msg)
|
||
|
||
# ------ PASS 2: parent/child links ------
|
||
for line_num, row_dict in rows_data:
|
||
asset = created_assets.get(line_num)
|
||
if asset is None:
|
||
continue
|
||
|
||
model_class = type(asset)
|
||
save_needed = False
|
||
|
||
# Parent links
|
||
parent_codes = _parse_codes_list(str(row_dict.get("codes_parents", "")))
|
||
for pcode in parent_codes:
|
||
parent = _find_asset_by_code(pcode)
|
||
if parent is None:
|
||
self.stderr.write(
|
||
f" Ligne {line_num}: [LIEN] parent '{pcode}' introuvable dans la base – lien ignoré."
|
||
)
|
||
continue
|
||
if not dry_run:
|
||
# Direction 1 : FK sur le nouvel asset → parent
|
||
ok, tried_fields = _set_parent_fk(asset, parent, model_class)
|
||
if ok:
|
||
save_needed = True
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN] parent '{pcode}' ({type(parent).__name__}) lié via FK."
|
||
)
|
||
else:
|
||
# Direction 2 (fallback) : FK sur le parent → nouvel asset
|
||
ok2, tried_fields2 = _set_parent_fk(parent, asset, type(parent))
|
||
if ok2:
|
||
parent.save()
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN] parent '{pcode}' ({type(parent).__name__}) "
|
||
f"lié via FK inverse (FK sur le parent)."
|
||
)
|
||
else:
|
||
all_tried = (tried_fields or []) + (tried_fields2 or [])
|
||
self.stderr.write(
|
||
f" Ligne {line_num}: [LIEN] impossible de lier le parent '{pcode}' "
|
||
f"({type(parent).__name__}) – aucun champ FK compatible trouvé "
|
||
f"(champs vérifiés: {all_tried or 'aucun'})."
|
||
)
|
||
else:
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN dry-run] parent '{pcode}' trouvé ({type(parent).__name__})."
|
||
)
|
||
|
||
if save_needed and not dry_run:
|
||
asset.save()
|
||
save_needed = False
|
||
|
||
# Child links
|
||
child_codes = _parse_codes_list(str(row_dict.get("codes_enfants", "")))
|
||
for ccode in child_codes:
|
||
child = _find_asset_by_code(ccode)
|
||
if child is None:
|
||
self.stderr.write(
|
||
f" Ligne {line_num}: [LIEN] enfant '{ccode}' introuvable dans la base – lien ignoré."
|
||
)
|
||
continue
|
||
if not dry_run:
|
||
# Direction 1 : FK sur l'enfant → nouvel asset
|
||
ok, tried_fields = _set_child_fk(child, asset)
|
||
if ok:
|
||
child.save()
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN] enfant '{ccode}' ({type(child).__name__}) lié via FK."
|
||
)
|
||
else:
|
||
# Direction 2 (fallback) : FK sur le nouvel asset → enfant
|
||
ok2, tried_fields2 = _set_child_fk(asset, child)
|
||
if ok2:
|
||
asset.save()
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN] enfant '{ccode}' ({type(child).__name__}) "
|
||
f"lié via FK inverse (FK sur l'asset importé)."
|
||
)
|
||
else:
|
||
all_tried = (tried_fields or []) + (tried_fields2 or [])
|
||
self.stderr.write(
|
||
f" Ligne {line_num}: [LIEN] impossible de lier l'enfant '{ccode}' "
|
||
f"({type(child).__name__}) – aucun champ FK compatible trouvé "
|
||
f"(champs vérifiés: {all_tried or 'aucun'})."
|
||
)
|
||
else:
|
||
self.stdout.write(
|
||
f" Ligne {line_num}: [LIEN dry-run] enfant '{ccode}' trouvé ({type(child).__name__})."
|
||
)
|
||
|
||
if dry_run:
|
||
# Roll back so nothing is persisted
|
||
transaction.set_rollback(True)
|
||
|
||
# --- Summary ---
|
||
self.stdout.write("")
|
||
self.stdout.write(self.style.SUCCESS("=== Résumé de l'import ==="))
|
||
self.stdout.write(f" Créés : {stats['created']}")
|
||
self.stdout.write(f" Mis à jour: {stats['updated']}")
|
||
self.stdout.write(f" Ignorés : {stats['skipped']}")
|
||
self.stdout.write(f" Erreurs : {stats['errors']}")
|
||
if errors:
|
||
self.stdout.write("")
|
||
self.stdout.write(self.style.ERROR("Détail des erreurs:"))
|
||
for err in errors:
|
||
self.stdout.write(self.style.ERROR(err))
|
||
if dry_run:
|
||
self.stdout.write(self.style.WARNING("(dry-run: aucune donnée enregistrée)"))
|
||
|
||
# ------------------------------------------------------------------
|
||
def _detect_header(self, ws):
|
||
"""
|
||
Detect which row contains the column headers and where data starts.
|
||
Supports two layouts:
|
||
- Row 1: machine-key row (from our template) → data starts at row 5
|
||
- Row 1 or 2: human-readable labels → data starts the row after
|
||
Returns (header_row_idx [1-based], col_map {key: 0-based col index},
|
||
data_start_row [1-based]).
|
||
"""
|
||
# Build reverse mapping: normalised label → key
|
||
label_to_key = {_normalize_key(label): key for key, label in COLUMNS}
|
||
|
||
HEADER_KEYS_SET = set(HEADER_KEYS)
|
||
|
||
for row_idx in range(1, 4):
|
||
row_values = [
|
||
str(cell.value).strip() if cell.value is not None else ""
|
||
for cell in ws[row_idx]
|
||
]
|
||
normalized = [_normalize_key(v) for v in row_values]
|
||
matches_key = sum(1 for n in normalized if n in HEADER_KEYS_SET)
|
||
matches_label = sum(1 for n in normalized if n in label_to_key)
|
||
|
||
if matches_key >= 3:
|
||
col_map = {}
|
||
for col_idx, norm in enumerate(normalized):
|
||
if norm in HEADER_KEYS_SET:
|
||
col_map[norm] = col_idx
|
||
# If ALL values are exact machine keys, this is our template row 1.
|
||
# In that case skip rows 2-4 (human labels, hints, example).
|
||
non_empty = [n for n in normalized if n]
|
||
is_template_machine_row = non_empty and all(n in HEADER_KEYS_SET for n in non_empty)
|
||
data_start = 5 if is_template_machine_row else row_idx + 1
|
||
return row_idx, col_map, data_start
|
||
|
||
if matches_label >= 3:
|
||
col_map = {}
|
||
for col_idx, norm in enumerate(normalized):
|
||
if norm in label_to_key:
|
||
col_map[label_to_key[norm]] = col_idx
|
||
return row_idx, col_map, row_idx + 1
|
||
|
||
return None, {}, None
|
||
|
||
# ------------------------------------------------------------------
|
||
def _process_row(self, row_dict: dict, update: bool, dry_run: bool):
|
||
"""
|
||
Process a single Excel row and create/update the asset.
|
||
Returns (asset_instance, action_string) or (None, 'skipped').
|
||
"""
|
||
code = str(row_dict.get("code", "")).strip()
|
||
if not code:
|
||
return None, "skipped"
|
||
|
||
model_name = str(row_dict.get("nom_modèle_objet", "")).strip().lower()
|
||
if not model_name:
|
||
raise ValueError("La colonne 'nom_modèle_objet' est requise.")
|
||
|
||
model_class = _get_model_class(model_name)
|
||
if model_class is None:
|
||
raise ValueError(f"Modèle d'objet inconnu: '{model_name}'")
|
||
|
||
# Check for existing asset
|
||
existing = model_class.objects.filter(code=code).first()
|
||
if existing and not update:
|
||
return None, "skipped"
|
||
|
||
asset = existing if existing else model_class()
|
||
action = "updated" if existing else "created"
|
||
|
||
# --- Thematic (informational, used for validation only) ---
|
||
thematic_code = str(row_dict.get("thématique", "")).strip()
|
||
if thematic_code and not dry_run:
|
||
try:
|
||
Thematic.objects.get(code=thematic_code)
|
||
except Thematic.DoesNotExist:
|
||
raise ValueError(f"Thématique inconnue: '{thematic_code}'")
|
||
|
||
# --- Basic fields ---
|
||
asset.code = code
|
||
|
||
nom_fr = str(row_dict.get("nom_fr", "")).strip()
|
||
if nom_fr and hasattr(asset, "name_fr"):
|
||
asset.name_fr = nom_fr
|
||
|
||
nom_nl = str(row_dict.get("nom_nl", "")).strip()
|
||
if nom_nl and hasattr(asset, "name_nl"):
|
||
asset.name_nl = nom_nl
|
||
|
||
marque = str(row_dict.get("marque", "")).strip()
|
||
if marque and hasattr(asset, "brand"):
|
||
asset.brand = marque
|
||
|
||
# --- Status ---
|
||
statut = str(row_dict.get("statut", "")).strip()
|
||
if statut:
|
||
valid_keys = list(ASSET_STATUS_CHOICES.keys())
|
||
if statut in valid_keys:
|
||
asset.status = statut
|
||
else:
|
||
raise ValueError(
|
||
f"Statut invalide: '{statut}'. Valeurs acceptées: {valid_keys}"
|
||
)
|
||
else:
|
||
if not existing:
|
||
asset.status = "active"
|
||
|
||
# --- Installation date ---
|
||
date_raw = row_dict.get("date_installation", "")
|
||
parsed_date = _parse_date(date_raw)
|
||
if parsed_date and hasattr(asset, "installation_date"):
|
||
from django.utils import timezone as dj_tz
|
||
if dj_tz.is_aware(parsed_date):
|
||
asset.installation_date = parsed_date
|
||
else:
|
||
asset.installation_date = dj_tz.make_aware(parsed_date)
|
||
|
||
# --- Category ---
|
||
cat_code = str(row_dict.get("code_catégorie", "")).strip()
|
||
if cat_code and hasattr(asset, "category"):
|
||
try:
|
||
asset.category = AssetCategory.objects.get(code=cat_code)
|
||
except AssetCategory.DoesNotExist:
|
||
raise ValueError(f"Catégorie inconnue: '{cat_code}'")
|
||
|
||
# --- Asset model object (e.g. StructureAssetModel) ---
|
||
model_code = str(row_dict.get("code_modèle", "")).strip()
|
||
if model_code and hasattr(model_class, "_meta"):
|
||
model_instance = _find_model_instance_by_code(model_class, model_code)
|
||
if model_instance is None:
|
||
raise ValueError(f"Modèle d'asset introuvable: code='{model_code}'")
|
||
asset.model = model_instance
|
||
elif not existing:
|
||
# Check if 'model' field is required
|
||
try:
|
||
model_field = model_class._meta.get_field("model")
|
||
if not model_field.null and not model_field.blank:
|
||
raise ValueError(
|
||
"Le champ 'code_modèle' est requis pour ce type d'asset."
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
# --- Location ---
|
||
loc_code = str(row_dict.get("code_localisation", "")).strip()
|
||
if loc_code:
|
||
field_name, location = _find_location_by_code(model_class, loc_code)
|
||
if location:
|
||
setattr(asset, field_name, location)
|
||
else:
|
||
raise ValueError(f"Localisation introuvable: code='{loc_code}'")
|
||
|
||
# --- Structure ---
|
||
structure_code = str(row_dict.get("structure", "")).strip()
|
||
if structure_code and hasattr(asset, "structure"):
|
||
from assets.models.structures import Structure
|
||
try:
|
||
asset.structure = Structure.objects.get(code=structure_code)
|
||
except Structure.DoesNotExist:
|
||
raise ValueError(f"Structure introuvable: code='{structure_code}'")
|
||
|
||
# --- Geometry ---
|
||
geojson_raw = str(row_dict.get("geojson", "")).strip()
|
||
lon_raw = row_dict.get("lon", "")
|
||
lat_raw = row_dict.get("lat", "")
|
||
|
||
if geojson_raw and hasattr(asset, "geom"):
|
||
ok = _set_geom_from_geojson(asset, geojson_raw)
|
||
if not ok:
|
||
raise ValueError(f"GeoJSON invalide: '{geojson_raw[:80]}'")
|
||
# Also store raw geojson string
|
||
if hasattr(asset, "geojson"):
|
||
asset.geojson = geojson_raw
|
||
elif lon_raw != "" and lat_raw != "" and hasattr(asset, "geom"):
|
||
try:
|
||
lon = float(str(lon_raw).replace(",", "."))
|
||
lat = float(str(lat_raw).replace(",", "."))
|
||
_set_geom_from_lon_lat(asset, lon, lat)
|
||
if hasattr(asset, "geojson"):
|
||
asset.geojson = json.dumps({
|
||
"type": "Point",
|
||
"coordinates": [lon, lat],
|
||
})
|
||
except ValueError:
|
||
raise ValueError(f"Lon/lat invalide: lon='{lon_raw}', lat='{lat_raw}'")
|
||
|
||
if not dry_run:
|
||
asset.save()
|
||
|
||
return asset, action
|