377 lines
16 KiB
Python
377 lines
16 KiB
Python
import os
|
|
import sys
|
|
import subprocess
|
|
import gzip
|
|
import shutil
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.conf import settings
|
|
from django.utils import timezone
|
|
|
|
try:
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import ClientError, NoCredentialsError
|
|
BOTO3_AVAILABLE = True
|
|
except ImportError:
|
|
BOTO3_AVAILABLE = False
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Effectue une sauvegarde complète de la base de données PostgreSQL, puis effectue les transferts optionnels (Rsync, MinIO S3)."
|
|
|
|
def log_info(self, message: str):
|
|
self.stdout.write(f"[{timezone.now().strftime('%Y-%m-%d %H:%M:%S')}] [INFO] {message}")
|
|
|
|
def log_warning(self, message: str):
|
|
self.stdout.write(self.style.WARNING(f"[{timezone.now().strftime('%Y-%m-%d %H:%M:%S')}] [WARNING] {message}"))
|
|
|
|
def log_error(self, message: str):
|
|
self.stderr.write(self.style.ERROR(f"[{timezone.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] {message}"))
|
|
|
|
def get_backup_dir(self) -> Path:
|
|
"""Retourne le répertoire de sauvegarde, le crée si nécessaire."""
|
|
backup_dir = os.getenv("BACKUP_DIR")
|
|
if backup_dir:
|
|
backup_path = Path(backup_dir)
|
|
else:
|
|
# Répertoire 'backups' à la racine Django
|
|
backup_path = settings.BASE_DIR / "backups"
|
|
|
|
backup_path.mkdir(parents=True, exist_ok=True)
|
|
return backup_path
|
|
|
|
def get_db_config(self) -> dict:
|
|
"""Récupère la configuration de la base de données depuis l'environnement ou les settings Django."""
|
|
db_settings = settings.DATABASES.get("default", {})
|
|
|
|
# Priorité aux variables d'environnement, sinon repli sur la conf Django
|
|
config = {
|
|
"host": os.getenv("DB_HOST") or db_settings.get("HOST") or "localhost",
|
|
"port": os.getenv("DB_PORT") or db_settings.get("PORT") or "5432",
|
|
"name": os.getenv("DB_NAME") or db_settings.get("NAME"),
|
|
"user": os.getenv("DB_USER") or db_settings.get("USER"),
|
|
"password": os.getenv("DB_PASSWORD") or db_settings.get("PASSWORD"),
|
|
}
|
|
|
|
# Vérification des paramètres requis
|
|
missing = [key for key, value in config.items() if not value and key != "port"]
|
|
if missing:
|
|
raise CommandError(f"Paramètres de connexion à la base de données manquants: {', '.join(missing)}")
|
|
|
|
return config
|
|
|
|
def get_schemas(self) -> list:
|
|
"""Récupère les schémas PostgreSQL à sauvegarder."""
|
|
schemas_str = os.getenv("BACKUP_SCHEMAS")
|
|
if not schemas_str:
|
|
# Récupération automatique du schéma configuré
|
|
db_schema = os.getenv("DB_SCHEMA")
|
|
if not db_schema:
|
|
options = settings.DATABASES.get("default", {}).get("OPTIONS", {}).get("options", "")
|
|
if "search_path=" in options:
|
|
db_schema = options.split("search_path=")[1].split(",")[0].strip()
|
|
else:
|
|
db_schema = "django"
|
|
schemas_str = f"{db_schema},public"
|
|
|
|
return [s.strip() for s in schemas_str.split(",") if s.strip()]
|
|
|
|
def run_pg_dump(self, db_config: dict, output_file: Path, timeout_seconds: int, schemas: list) -> bool:
|
|
"""Exécute pg_dump pour créer la sauvegarde brute."""
|
|
pg_dump_path = os.getenv("PG_DUMP_PATH", "pg_dump")
|
|
|
|
cmd = [
|
|
pg_dump_path,
|
|
"-h", db_config["host"],
|
|
"-p", str(db_config["port"]),
|
|
"-U", db_config["user"],
|
|
"-d", db_config["name"],
|
|
"-F", "p", # Plain SQL format
|
|
"--no-password",
|
|
]
|
|
|
|
for schema in schemas:
|
|
cmd.extend(["-n", schema])
|
|
|
|
env = os.environ.copy()
|
|
env["PGPASSWORD"] = db_config["password"]
|
|
|
|
try:
|
|
self.log_info(f"Démarrage de pg_dump vers {output_file}")
|
|
self.log_info(f"Schémas : {', '.join(schemas)}")
|
|
|
|
with open(output_file, "w") as f:
|
|
result = subprocess.run(
|
|
cmd,
|
|
stdout=f,
|
|
stderr=subprocess.PIPE,
|
|
env=env,
|
|
timeout=timeout_seconds,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
err_msg = result.stderr.decode('utf-8', errors='replace')
|
|
self.log_error(f"Erreur pg_dump : {err_msg}")
|
|
if output_file.exists():
|
|
output_file.unlink()
|
|
raise CommandError(f"pg_dump a échoué (code {result.returncode}) : {err_msg}")
|
|
|
|
self.log_info("pg_dump terminé avec succès")
|
|
|
|
except subprocess.TimeoutExpired:
|
|
self.log_error(f"Timeout: pg_dump a dépassé la limite de {timeout_seconds}s")
|
|
if output_file.exists():
|
|
output_file.unlink()
|
|
raise CommandError(f"pg_dump a expiré après {timeout_seconds} secondes.")
|
|
except FileNotFoundError:
|
|
self.log_error(f"pg_dump introuvable à '{pg_dump_path}'. Assurez-vous que postgresql-client est installé.")
|
|
if output_file.exists():
|
|
output_file.unlink()
|
|
raise CommandError(f"pg_dump introuvable à '{pg_dump_path}'. Veuillez installer postgresql-client ou configurer PG_DUMP_PATH.")
|
|
except Exception as e:
|
|
self.log_error(f"Erreur inattendue durant pg_dump : {str(e)}")
|
|
if output_file.exists():
|
|
output_file.unlink()
|
|
raise CommandError(f"Erreur inattendue durant pg_dump : {str(e)}")
|
|
|
|
def compress_file(self, input_file: Path, output_file: Path):
|
|
"""Compresse la sauvegarde brute avec gzip."""
|
|
try:
|
|
self.log_info(f"Compression du dump vers {output_file}")
|
|
with open(input_file, "rb") as f_in:
|
|
with gzip.open(output_file, "wb") as f_out:
|
|
shutil.copyfileobj(f_in, f_out)
|
|
|
|
# Suppression du fichier non compressé
|
|
input_file.unlink()
|
|
|
|
size_mb = output_file.stat().st_size / (1024 * 1024)
|
|
self.log_info(f"Compression terminée. Taille : {size_mb:.2f} Mo")
|
|
except Exception as e:
|
|
self.log_error(f"Erreur lors de la compression : {str(e)}")
|
|
raise CommandError(f"Erreur lors de la compression gzip : {str(e)}")
|
|
|
|
def upload_to_minio(self, backup_file: Path) -> bool:
|
|
"""Téléverse la sauvegarde sur le serveur MinIO configuré via l'API S3."""
|
|
if not BOTO3_AVAILABLE:
|
|
self.log_error("La librairie 'boto3' est requise pour le téléversement MinIO.")
|
|
return False
|
|
|
|
endpoint_url = os.getenv("BACKUP_MINIO_ENDPOINT_URL", "https://apps.mobility.brussels")
|
|
bucket = os.getenv("BACKUP_MINIO_BUCKET", "datatransfer")
|
|
region = os.getenv("BACKUP_MINIO_REGION", "us-east-1")
|
|
access_key = os.getenv("BACKUP_MINIO_ACCESS_KEY")
|
|
secret_key = os.getenv("BACKUP_MINIO_SECRET_KEY")
|
|
|
|
if not access_key or not secret_key:
|
|
self.log_error("BACKUP_MINIO_ACCESS_KEY ou BACKUP_MINIO_SECRET_KEY non configuré dans l'environnement. Téléversement impossible.")
|
|
return False
|
|
|
|
object_key = f"{datetime.now().date().isoformat()}_{backup_file.name}"
|
|
self.log_info(f"Téléversement vers MinIO ({endpoint_url}/{bucket}/{object_key})...")
|
|
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=endpoint_url,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
region_name=region,
|
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
)
|
|
|
|
try:
|
|
from boto3.s3.transfer import TransferConfig
|
|
|
|
# Configure le multipart upload avec des blocs de 5 Mo (minimum S3 requis)
|
|
# afin de contourner la limite de taille d'entité imposée par le proxy Nginx/Apache.
|
|
transfer_config = TransferConfig(
|
|
multipart_threshold=5 * 1024 * 1024,
|
|
multipart_chunksize=5 * 1024 * 1024,
|
|
)
|
|
|
|
s3.upload_file(
|
|
Filename=str(backup_file),
|
|
Bucket=bucket,
|
|
Key=object_key,
|
|
Config=transfer_config
|
|
)
|
|
self.log_info("Téléversement MinIO réussi.")
|
|
return True
|
|
except NoCredentialsError:
|
|
self.log_error("Identifiants non valides pour MinIO/S3.")
|
|
return False
|
|
except ClientError as e:
|
|
self.log_error(f"Échec MinIO ClientError : {e}")
|
|
return False
|
|
except Exception as e:
|
|
self.log_error(f"Erreur inattendue durant l'upload MinIO : {e}")
|
|
return False
|
|
|
|
def sync_to_remote(self, backup_file: Path) -> bool:
|
|
"""Effectue le transfert distant historique (rsync) s'il est activé."""
|
|
remote_enabled = os.getenv("BACKUP_REMOTE_ENABLED", "false").lower() == "true"
|
|
if not remote_enabled:
|
|
return True
|
|
|
|
remote_host = os.getenv("BACKUP_REMOTE_HOST")
|
|
remote_dir = os.getenv("BACKUP_REMOTE_DIR")
|
|
ssh_key = os.getenv("BACKUP_SSH_KEY")
|
|
# Si le chemin configuré n'existe pas dans le conteneur mais qu'un montage volume
|
|
# a mis la clé dans /app/backup_key, on se replie sur ce montage.
|
|
if ssh_key and not os.path.exists(ssh_key) and os.path.exists("/app/backup_key"):
|
|
self.log_info("Repli automatique sur la clé SSH montée dans le conteneur (/app/backup_key)")
|
|
ssh_key = "/app/backup_key"
|
|
|
|
timeout = int(os.getenv("BACKUP_REMOTE_TIMEOUT", "3600"))
|
|
|
|
if not remote_host or not remote_dir:
|
|
self.log_warning("BACKUP_REMOTE_HOST ou BACKUP_REMOTE_DIR non configuré. Transfert Rsync ignoré.")
|
|
return False
|
|
|
|
self.log_info(f"Transfert Rsync vers {remote_host}:{remote_dir}")
|
|
|
|
# Préparer une clé SSH temporaire avec des permissions 600 pour contourner
|
|
# les restrictions de permissions d'SSH sur les clés à droits trop larges (ex: 644)
|
|
tmp_ssh_key = None
|
|
if ssh_key and os.path.exists(ssh_key):
|
|
try:
|
|
import tempfile
|
|
fd, tmp_ssh_path = tempfile.mkstemp()
|
|
with os.fdopen(fd, 'wb') as tmp_file:
|
|
with open(ssh_key, 'rb') as src_file:
|
|
tmp_file.write(src_file.read())
|
|
|
|
os.chmod(tmp_ssh_path, 0o600)
|
|
tmp_ssh_key = tmp_ssh_path
|
|
self.log_info(f"Clé SSH copiée temporairement avec les permissions 600 : {tmp_ssh_key}")
|
|
except Exception as e:
|
|
self.log_warning(f"Impossible de sécuriser la clé SSH pour le conteneur: {e}. Utilisation directe.")
|
|
tmp_ssh_key = ssh_key
|
|
else:
|
|
tmp_ssh_key = ssh_key
|
|
|
|
ssh_cmd = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
|
if tmp_ssh_key:
|
|
ssh_cmd = f"ssh -i {tmp_ssh_key} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
|
|
|
|
cmd = [
|
|
"rsync",
|
|
"-avz",
|
|
"-e", ssh_cmd,
|
|
str(backup_file),
|
|
f"{remote_host}:{remote_dir}/",
|
|
]
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
self.log_error(f"Erreur Rsync : {result.stderr}")
|
|
return False
|
|
|
|
self.log_info("Transfert Rsync réussi.")
|
|
return True
|
|
except subprocess.TimeoutExpired:
|
|
self.log_error(f"Timeout: Rsync a dépassé {timeout}s.")
|
|
return False
|
|
except FileNotFoundError:
|
|
self.log_error("rsync non trouvé sur le système.")
|
|
return False
|
|
except Exception as e:
|
|
self.log_error(f"Erreur Rsync inattendue : {str(e)}")
|
|
return False
|
|
finally:
|
|
if tmp_ssh_key and tmp_ssh_key != ssh_key and os.path.exists(tmp_ssh_key):
|
|
try:
|
|
os.unlink(tmp_ssh_key)
|
|
except OSError:
|
|
pass
|
|
|
|
def cleanup_old_backups(self, backup_dir: Path, retention_days: int):
|
|
"""Supprime les sauvegardes expirées locales."""
|
|
cutoff_date = datetime.now() - timedelta(days=retention_days)
|
|
deleted_count = 0
|
|
|
|
self.log_info(f"Nettoyage des sauvegardes locales de plus de {retention_days} jours")
|
|
|
|
for backup_file in backup_dir.glob("streetup_backup_*.sql.gz"):
|
|
try:
|
|
date_str = backup_file.name.replace("streetup_backup_", "").replace(".sql.gz", "")
|
|
# Format attendu : YYYYMMDD_HHMMSS
|
|
file_date = datetime.strptime(date_str, "%Y%m%d_%H%M%S")
|
|
|
|
if file_date < cutoff_date:
|
|
backup_file.unlink()
|
|
self.log_info(f"Supprimé : {backup_file.name}")
|
|
deleted_count += 1
|
|
except (ValueError, OSError) as e:
|
|
self.log_warning(f"Impossible de traiter/supprimer {backup_file.name} : {str(e)}")
|
|
|
|
if deleted_count > 0:
|
|
self.log_info(f"{deleted_count} sauvegarde(s) ancienne(s) supprimée(s).")
|
|
else:
|
|
self.log_info("Aucune sauvegarde expirée à supprimer.")
|
|
|
|
def handle(self, *args, **options):
|
|
# Vérification globale de l'activation des sauvegardes (évite les lancements indésirables en dev/test après import de dump)
|
|
backup_enabled = os.getenv("BACKUP_ENABLED", "false").lower() == "true"
|
|
if not backup_enabled:
|
|
self.log_warning("La sauvegarde de la base de données est désactivée via la variable d'environnement BACKUP_ENABLED.")
|
|
return
|
|
|
|
self.log_info("=" * 60)
|
|
self.log_info("SAUVEGARDE DE LA BASE DE DONNÉES STREETUP")
|
|
self.log_info("=" * 60)
|
|
|
|
db_config = self.get_db_config()
|
|
backup_dir = self.get_backup_dir()
|
|
retention_days = int(os.getenv("BACKUP_RETENTION_DAYS", "30"))
|
|
timeout_seconds = int(os.getenv("BACKUP_TIMEOUT_SECONDS", "3600"))
|
|
|
|
schemas = self.get_schemas()
|
|
|
|
# Configuration des transferts
|
|
minio_enabled = os.getenv("BACKUP_MINIO_ENABLED", "false").lower() == "true"
|
|
remote_enabled = os.getenv("BACKUP_REMOTE_ENABLED", "false").lower() == "true"
|
|
|
|
self.log_info(f"Base de données : {db_config['name']} @ {db_config['host']}")
|
|
self.log_info(f"Dossier local : {backup_dir}")
|
|
self.log_info(f"Rétention locale : {retention_days} jours")
|
|
self.log_info(f"Schémas : {', '.join(schemas)}")
|
|
self.log_info(f"Upload MinIO : {'ACTIVÉ' if minio_enabled else 'DÉSACTIVÉ'}")
|
|
self.log_info(f"Transfert Rsync : {'ACTIVÉ' if remote_enabled else 'DÉSACTIVÉ'}")
|
|
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
sql_file = backup_dir / f"streetup_backup_{timestamp}.sql"
|
|
gz_file = backup_dir / f"streetup_backup_{timestamp}.sql.gz"
|
|
|
|
# 1. pg_dump
|
|
self.run_pg_dump(db_config, sql_file, timeout_seconds, schemas)
|
|
|
|
# 2. gzip
|
|
self.compress_file(sql_file, gz_file)
|
|
|
|
# 3. MinIO
|
|
if minio_enabled:
|
|
if not self.upload_to_minio(gz_file):
|
|
self.log_warning("Sauvegarde MinIO en échec (le backup local est conservé).")
|
|
|
|
# 4. Rsync
|
|
if remote_enabled:
|
|
if not self.sync_to_remote(gz_file):
|
|
self.log_warning("Sauvegarde Rsync en échec (le backup local est conservé).")
|
|
|
|
# 5. Clean local
|
|
self.cleanup_old_backups(backup_dir, retention_days)
|
|
|
|
self.log_info("-" * 60)
|
|
self.log_info(f"✓ Sauvegarde exécutée avec succès : {gz_file.name}")
|
|
self.log_info("=" * 60)
|