loko/streetup/panoramax/views.py
2026-07-22 14:48:40 +02:00

759 lines
31 KiB
Python

import os
import zipfile
import tempfile
import traceback
import time
import requests
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta
from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.http import JsonResponse, HttpResponse, HttpResponseForbidden, HttpResponseBadRequest
from django.utils.translation import gettext as _
from django.contrib import messages
from django.core.files.base import ContentFile
from django.conf import settings
from django.core import signing
from common.models import UserConfig
from .models import PanoramaxUpload
from .utils import (
parse_gpx,
extract_frames_from_video,
write_gps_exif,
interpolate_gpx,
calculate_bearing,
calculate_stable_heading,
call_panoramax_api_create_set,
call_panoramax_api_upload_file,
call_panoramax_api_complete_set
)
from PIL import Image
# Shared thread pool executor with 2 workers to serialize heavy video/image processing tasks
_upload_executor = ThreadPoolExecutor(max_workers=2)
@login_required
def index(request):
"""
Panoramax 360 photo viewer page.
"""
user_config = get_object_or_404(UserConfig, user=request.user)
# Check if user has access to the panoramax view
if not user_config.can_access_view('panoramax'):
raise PermissionDenied(_("Vous n'avez pas accès à cette vue."))
# Self-healing: if some completed uploads don't have associated collections stored,
# fetch them from Panoramax API and save them.
unfetched_uploads = PanoramaxUpload.objects.filter(status='completed')
unfetched_uploads = [up for up in unfetched_uploads if not up.associated_collections]
headers = {}
if settings.PANORAMAX_API_KEY:
headers['Authorization'] = f'Bearer {settings.PANORAMAX_API_KEY}'
if unfetched_uploads:
for up in unfetched_uploads:
if up.upload_set_id:
try:
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/upload_sets/{up.upload_set_id}"
r = requests.get(url, headers=headers, timeout=2.0)
if r.status_code == 200:
data = r.json()
collections = data.get('associated_collections', [])
if collections:
associated = []
for c in collections:
date_val = None
try:
date_str = c.get('extent', {}).get('temporal', {}).get('interval', [[None]])[0][0]
if date_str:
date_val = date_str.split('T')[0]
except Exception:
pass
associated.append({
'id': c['id'],
'date': date_val
})
up.associated_collections = associated
up.save()
except Exception as e:
print(f"Error self-healing upload {up.id}: {e}")
# Convert legacy list of strings to list of dicts with dates
all_completed = PanoramaxUpload.objects.filter(status='completed')
for up in all_completed:
changed = False
enriched = []
for item in up.associated_collections:
if isinstance(item, str):
date_val = "2026-06-15"
try:
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/collections/{item}"
r = requests.get(url, headers=headers, timeout=2.0)
if r.status_code == 200:
data = r.json()
date_str = data.get('extent', {}).get('temporal', {}).get('interval', [[None]])[0][0]
if date_str:
date_val = date_str.split('T')[0]
except Exception:
pass
enriched.append({
'id': item,
'date': date_val
})
changed = True
elif isinstance(item, dict):
if not item.get('date'):
date_val = "2026-06-15"
try:
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/collections/{item.get('id')}"
r = requests.get(url, headers=headers, timeout=2.0)
if r.status_code == 200:
data = r.json()
date_str = data.get('extent', {}).get('temporal', {}).get('interval', [[None]])[0][0]
if date_str:
date_val = date_str.split('T')[0]
except Exception:
pass
item['date'] = date_val
changed = True
enriched.append(item)
if changed:
up.associated_collections = enriched
up.save()
# Query users who have completed uploads
from django.contrib.auth.models import User
from django.db.models import Count
import json
upload_users = User.objects.filter(
panoramaxupload__status='completed'
).annotate(
upload_count=Count('panoramaxupload')
).order_by('username')
# Construct the JSON mapping of username -> sequence_ids and flat list of all sequences
user_sequences = {}
all_sequences = []
for u in upload_users:
seqs = []
uploads = PanoramaxUpload.objects.filter(user=u, status='completed')
for up in uploads:
if up.associated_collections:
for item in up.associated_collections:
if isinstance(item, dict):
seq_id = item.get('id')
seq_date = item.get('date') or up.created_at.strftime('%Y-%m-%d')
else:
seq_id = item
seq_date = up.created_at.strftime('%Y-%m-%d')
if seq_id:
seqs.append(seq_id)
all_sequences.append({
'id': seq_id,
'date': seq_date,
'user': u.username
})
user_sequences[u.username] = seqs
from django.core import signing
image_token = signing.dumps({"user_id": request.user.id}, salt="panoramax-images")
context = {
'user_config': user_config,
'panoramax_api_url': settings.PANORAMAX_API_URL if hasattr(settings, 'PANORAMAX_API_URL') else '',
'image_token': image_token,
'upload_users': upload_users,
'user_sequences_json': json.dumps(user_sequences),
'all_sequences_json': json.dumps(all_sequences),
}
return render(request, "panoramax/viewer.html", context)
@login_required
def upload(request):
"""
Shows list of uploads and handles photo/video uploads.
"""
user_config = get_object_or_404(UserConfig, user=request.user)
if not user_config.can_access_view('panoramax') or not user_config.panoramax_can_upload:
raise PermissionDenied(_("Vous n'avez pas l'autorisation d'uploader des photos."))
if request.method == 'POST':
title = request.POST.get('title')
if not title:
title = f"Upload du {datetime.now().strftime('%d/%m/%Y %H:%M')}"
upload_type = request.POST.get('upload_type')
gpx_file = request.FILES.get('gpx_file')
camera_offset_val = request.POST.get('camera_offset', '0')
try:
camera_offset = int(camera_offset_val)
except ValueError:
camera_offset = 0
upload_task = PanoramaxUpload(
user=request.user,
title=title,
upload_type=upload_type,
camera_offset=camera_offset,
status='pending',
progress=0
)
if gpx_file:
upload_task.gpx_file = gpx_file
uploaded_files = request.FILES.getlist('uploaded_file')
if not uploaded_files:
messages.error(request, _("Veuillez sélectionner des fichiers à uploader."))
return redirect('panoramax:upload')
# Handle file storage based on upload type
if upload_type in ['video', 'zip']:
# Single file expected
upload_task.uploaded_file = uploaded_files[0]
else:
# Multiple files: files selection or folder selection
# We compress multiple files into a single zip file on the fly to avoid loose files in storage
temp_zip_file = tempfile.NamedTemporaryFile(suffix='.zip', delete=False)
try:
with zipfile.ZipFile(temp_zip_file.name, 'w', zipfile.ZIP_DEFLATED) as zip_ref:
for idx, file in enumerate(uploaded_files):
# Ensure only image files are zipped
if file.name.lower().endswith(('.jpg', '.jpeg')):
zip_ref.writestr(file.name, file.read())
# Reopen file and save as FileField content
temp_zip_file.seek(0)
upload_task.uploaded_file.save(f"photos_{int(time.time())}.zip", ContentFile(temp_zip_file.read()), save=False)
finally:
temp_zip_file.close()
try:
os.unlink(temp_zip_file.name)
except OSError:
pass
upload_task.save()
# Submit task to the single worker thread queue
_upload_executor.submit(process_upload_task, upload_task.id)
messages.success(request, _("Votre demande de téléversement a été enregistrée. Le traitement est lancé en arrière-plan."))
return redirect('panoramax:upload')
# GET: display history and form
uploads = PanoramaxUpload.objects.filter(user=request.user).order_by('-created_at')
context = {
'user_config': user_config,
'uploads': uploads
}
return render(request, "panoramax/upload.html", context)
@login_required
def upload_status(request, upload_id):
"""
JSON API view returning the status and progress of a task.
"""
user_config = get_object_or_404(UserConfig, user=request.user)
if not user_config.can_access_view('panoramax') or not user_config.panoramax_can_upload:
return JsonResponse({'error': 'Unauthorized'}, status=403)
upload_task = get_object_or_404(PanoramaxUpload, id=upload_id, user=request.user)
panoramax_status = None
if upload_task.status == 'completed' and upload_task.upload_set_id:
try:
url = f"{settings.PANORAMAX_API_URL}/upload_sets/{upload_task.upload_set_id}"
headers = {}
if settings.PANORAMAX_API_KEY:
headers['Authorization'] = f'Bearer {settings.PANORAMAX_API_KEY}'
r = requests.get(url, headers=headers, timeout=1.5)
if r.status_code == 200:
data = r.json()
is_ready = data.get('ready', False)
# Extract associated collections
collections = data.get('associated_collections', [])
if collections and not upload_task.associated_collections:
upload_task.associated_collections = [c['id'] for c in collections]
upload_task.save()
status = data.get('items_status', {})
prepared = status.get('prepared', 0)
preparing = status.get('preparing', 0)
not_processed = status.get('not_processed', 0)
broken = status.get('broken', 0)
rejected = status.get('rejected', 0)
total = prepared + preparing + not_processed + broken + rejected
pct = int((prepared / total * 100)) if total > 0 else 0
panoramax_status = {
'ready': is_ready,
'prepared': prepared,
'preparing': preparing,
'not_processed': not_processed,
'broken': broken,
'rejected': rejected,
'total': total,
'pct': pct
}
except Exception as e:
print(f"Error checking external Panoramax status: {e}")
return JsonResponse({
'status': upload_task.status,
'status_display': upload_task.get_status_display(),
'progress': upload_task.progress,
'error_message': upload_task.error_message or '',
'upload_set_id': upload_task.upload_set_id or '',
'panoramax_status': panoramax_status
})
def process_upload_task(upload_id):
"""
Background worker function running in a separate thread.
Parses files, processes EXIF geo-tagging and uploads to Panoramax.
"""
task = None
try:
task = PanoramaxUpload.objects.get(id=upload_id)
task.status = 'processing'
task.progress = 5
task.save()
# Step 1: Initialize temporary directory
with tempfile.TemporaryDirectory() as temp_dir:
image_paths = []
# Step 2: Parse GPS track points
gps_points = []
if task.gpx_file:
gps_points = parse_gpx(task.gpx_file.path)
elif task.upload_type == 'video':
# Attempt to extract GPS track from the video itself (e.g. GoPro GPMD)
from .utils import extract_gpmf_gps
gps_points = extract_gpmf_gps(task.uploaded_file.path)
# Step 3: Process inputs based on type
if task.upload_type == 'video':
# Video processing
if not gps_points:
raise ValueError(_("Aucune donnée GPS n'a été trouvée dans la vidéo et aucun fichier GPX n'a été fourni."))
def progress_cb(pct):
task.progress = int(5 + pct * 0.7) # Map 0-80% of utils progress to 5-61% of total task
task.save()
image_paths = extract_frames_from_video(
task.uploaded_file.path,
gps_points,
temp_dir,
camera_offset=task.camera_offset,
progress_callback=progress_cb
)
else:
# Zip/photos extraction
task.progress = 10
task.save()
with zipfile.ZipFile(task.uploaded_file.path, 'r') as zip_ref:
# Extract zip file
zip_ref.extractall(temp_dir)
# Scan directory for images
raw_paths = []
for root, subdirs, files in os.walk(temp_dir):
for file in files:
if file.lower().endswith(('.jpg', '.jpeg')):
raw_paths.append(os.path.join(root, file))
total_images = len(raw_paths)
if total_images == 0:
raise ValueError(_("Aucune image JPEG n'a été trouvée dans le fichier d'upload."))
# Geotag images if GPX is provided
photo_metadata = []
for idx, path in enumerate(raw_paths):
photo_time = None
try:
img = Image.open(path)
exif = img.getexif()
time_str = exif.get(36867) # DateTimeOriginal
if time_str:
try:
photo_time = datetime.strptime(time_str, "%Y:%m:%d %H:%M:%S")
except ValueError:
pass
except Exception:
pass
if not photo_time:
mtime = os.path.getmtime(path)
photo_time = datetime.fromtimestamp(mtime)
pos = None
if gps_points:
pos = interpolate_gpx(gps_points, photo_time)
photo_metadata.append({
'path': path,
'time': photo_time,
'pos': pos
})
# Sort photos chronologically by capture time to calculate heading along the sequence
photo_metadata.sort(key=lambda x: x['time'] or datetime.min)
total_offset = task.camera_offset
num_photos = len(photo_metadata)
for idx, item in enumerate(photo_metadata):
path = item['path']
pos = item['pos']
if pos:
if num_photos > 1:
if idx < num_photos - 1:
next_pos = photo_metadata[idx + 1]['pos']
if next_pos:
bearing = calculate_bearing(pos['lat'], pos['lon'], next_pos['lat'], next_pos['lon'])
else:
bearing = 0.0
else:
prev_pos = photo_metadata[idx - 1]['pos']
if prev_pos:
bearing = calculate_bearing(prev_pos['lat'], prev_pos['lon'], pos['lat'], pos['lon'])
else:
bearing = 0.0
heading = (bearing + total_offset) % 360
else:
heading = (0.0 + total_offset) % 360
write_gps_exif(path, pos['lat'], pos['lon'], pos['ele'], pos['time'], heading=heading)
image_paths.append(path)
# Update progress during photo preprocessing (reaches up to 50%)
task.progress = int(10 + (idx / num_photos) * 40)
task.save()
time.sleep(0.01) # Yield CPU control slightly
# Step 4: Upload to Panoramax
task.progress = 60
task.save()
if not image_paths:
raise ValueError(_("Aucune image valide n'a été générée pour l'upload."))
# Create UploadSet in Panoramax API
upload_set_id = call_panoramax_api_create_set(task.title, len(image_paths), creator=task.user.username)
task.upload_set_id = upload_set_id
task.progress = 70
task.save()
# Upload files
for idx, path in enumerate(image_paths):
call_panoramax_api_upload_file(upload_set_id, path)
# Update progress during upload phase (70% -> 95%)
task.progress = int(70 + (idx / len(image_paths)) * 25)
task.save()
# Voluntary yield to avoid network card & server saturation
time.sleep(0.02)
# Mark upload set completed
call_panoramax_api_complete_set(upload_set_id)
task.progress = 100
task.status = 'completed'
task.save()
except Exception as e:
traceback.print_exc()
if task:
task.status = 'failed'
task.error_message = str(e)
task.save()
finally:
# Nettoyage des fichiers média volumineux une fois l'importation terminée
if task:
try:
task.refresh_from_db()
if task.uploaded_file:
task.uploaded_file.delete(save=False)
if task.gpx_file:
task.gpx_file.delete(save=False)
task.save()
except Exception as cleanup_err:
print(f"Erreur lors du nettoyage des fichiers panoramax: {cleanup_err}")
traceback.print_exc()
from django.core.signing import BadSignature, SignatureExpired
from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404
import boto3
@login_required
def serve_panoramax_picture(request, picture_id, path):
"""
Validates a 30-minute signed token and redirects the browser directly to a
temporary presigned S3 URL.
- Method 1: If PANORAMAX_AWS_S3_BUCKET is configured in settings, constructs the
S3 key directly (fastest, zero Panoramax query overhead).
- Method 2: Otherwise, queries the Panoramax API to inspect if it redirects to S3,
and parses the S3 URL dynamically.
- Method 3: Falls back to streaming the image bytes directly if no S3 redirection
or credentials are configured.
"""
token = request.GET.get('token')
if not token:
return HttpResponseForbidden("Missing authorization token")
try:
# Validate the token (max age = 1800 seconds / 30 minutes)
signing.loads(token, salt="panoramax-images", max_age=1800)
except (BadSignature, SignatureExpired):
return HttpResponseForbidden("Expired or invalid authorization token")
aws_key = getattr(settings, 'PANORAMAX_AWS_ACCESS_KEY_ID', None)
aws_secret = getattr(settings, 'PANORAMAX_AWS_SECRET_ACCESS_KEY', None)
bucket = getattr(settings, 'PANORAMAX_AWS_S3_BUCKET', None)
prefix = getattr(settings, 'PANORAMAX_AWS_S3_PREFIX', 'photos').strip('/')
# Method 1: Direct construction if bucket name is set in settings (Option B direct)
if aws_key and aws_secret and bucket:
uuid_str = str(picture_id).replace("-", "").lower()
prefix1 = uuid_str[0:2]
prefix2 = uuid_str[2:4]
prefix3 = uuid_str[4:6]
prefix4 = uuid_str[6:8]
rest_uuid = str(picture_id)[9:]
if path == 'hd.jpg':
s3_key = f"{prefix}/permanent/{prefix1}/{prefix2}/{prefix3}/{prefix4}/{rest_uuid}.jpg"
elif path in ['sd.jpg', 'thumb.jpg']:
s3_key = f"{prefix}/derivates/{prefix1}/{prefix2}/{prefix3}/{prefix4}/{rest_uuid}/{path}"
elif path.startswith('tiled/'):
tiles_path = path.replace('tiled/', 'tiles/')
s3_key = f"{prefix}/derivates/{prefix1}/{prefix2}/{prefix3}/{prefix4}/{rest_uuid}/{tiles_path}"
else:
s3_key = f"{prefix}/derivates/{prefix1}/{prefix2}/{prefix3}/{prefix4}/{rest_uuid}/{path}"
endpoint_url = getattr(settings, 'PANORAMAX_AWS_S3_ENDPOINT_URL', 'https://s3.rbx.io.cloud.ovh.net')
try:
s3_client = boto3.client(
's3',
aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret,
endpoint_url=endpoint_url
)
presigned_url = s3_client.generate_presigned_url(
'get_object',
Params={'Bucket': bucket, 'Key': s3_key},
ExpiresIn=1800
)
return HttpResponseRedirect(presigned_url)
except Exception as e:
import traceback
traceback.print_exc()
# If direct construction signing fails, fall through to query method
# Method 2: Query Panoramax API to inspect redirect behavior
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/pictures/{picture_id}/{path}"
headers = {}
if settings.PANORAMAX_API_KEY:
headers['Authorization'] = f'Bearer {settings.PANORAMAX_API_KEY}'
try:
# Perform a request on the Panoramax API without following redirects.
# This is extremely fast because it stops immediately upon receiving the redirect headers.
r = requests.get(url, headers=headers, allow_redirects=False, timeout=5)
# If Panoramax redirects to an external S3 storage and we have AWS credentials to sign it:
if r.status_code in [301, 302, 303, 307, 308] and 'Location' in r.headers and aws_key and aws_secret:
s3_redirect_url = r.headers['Location']
# Parse the direct S3 URL to extract bucket, endpoint, and key
from urllib.parse import urlparse
parsed_url = urlparse(s3_redirect_url)
# virtual-hosted style (e.g. bucket.s3.region.domain)
netloc_parts = parsed_url.netloc.split('.')
parsed_bucket = netloc_parts[0]
endpoint_host = ".".join(netloc_parts[1:])
endpoint_url = f"{parsed_url.scheme}://{endpoint_host}"
# Use environment override if defined
if getattr(settings, 'PANORAMAX_AWS_S3_ENDPOINT_URL', None):
endpoint_url = settings.PANORAMAX_AWS_S3_ENDPOINT_URL
s3_key = parsed_url.path.lstrip('/')
# Generate the S3 Presigned URL (valid for 30 minutes)
s3_client = boto3.client(
's3',
aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret,
endpoint_url=endpoint_url
)
presigned_url = s3_client.generate_presigned_url(
'get_object',
Params={'Bucket': parsed_bucket, 'Key': s3_key},
ExpiresIn=1800
)
return HttpResponseRedirect(presigned_url)
else:
# Method 3: Fallback to streaming the image bytes directly
r_get = requests.get(url, headers=headers, stream=True, timeout=15)
if r_get.status_code == 200:
from django.http import StreamingHttpResponse
response = StreamingHttpResponse(
r_get.iter_content(chunk_size=8192),
content_type=r_get.headers.get('Content-Type', 'image/jpeg')
)
if 'Cache-Control' in r_get.headers:
response['Cache-Control'] = r_get.headers['Cache-Control']
return response
else:
raise Http404("Image not found in Panoramax")
except Exception as e:
import traceback
traceback.print_exc()
raise Http404(f"Error fetching picture: {e}")
@login_required
def delete_collection(request, collection_id):
"""
Deletes an entire collection (sequence) from Panoramax if the user owns it or is admin.
"""
if request.method != 'POST':
return HttpResponseBadRequest("Only POST method is allowed")
user_config = get_object_or_404(UserConfig, user=request.user)
if not user_config.can_access_view('panoramax'):
return HttpResponseForbidden("Access denied to Panoramax")
collection_id_str = str(collection_id)
# Check authorization: user must own the sequence OR be staff/superuser
is_authorized = request.user.is_staff or request.user.is_superuser
if not is_authorized:
user_uploads = PanoramaxUpload.objects.filter(user=request.user, status='completed')
for up in user_uploads:
if up.associated_collections:
for item in up.associated_collections:
item_id = item.get('id') if isinstance(item, dict) else item
if item_id == collection_id_str:
is_authorized = True
break
if is_authorized:
break
if not is_authorized:
return HttpResponseForbidden("You are not authorized to delete this collection")
# Send DELETE request to Panoramax API
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/collections/{collection_id_str}"
headers = {}
if settings.PANORAMAX_API_KEY:
headers['Authorization'] = f'Bearer {settings.PANORAMAX_API_KEY}'
try:
response = requests.delete(url, headers=headers, timeout=15)
if response.status_code not in [200, 204, 404]:
return HttpResponse(
f"API error: {response.text}",
status=response.status_code
)
# Clean up database: remove from associated_collections in uploader tasks
for up in PanoramaxUpload.objects.all():
if up.associated_collections:
new_cols = []
changed = False
for item in up.associated_collections:
item_id = item.get('id') if isinstance(item, dict) else item
if item_id == collection_id_str:
changed = True
else:
new_cols.append(item)
if changed:
up.associated_collections = new_cols
up.save()
return HttpResponse("Collection deleted successfully")
except Exception as e:
traceback.print_exc()
return HttpResponse(f"Error calling Panoramax API: {e}", status=500)
@login_required
def delete_picture(request, collection_id, picture_id):
"""
Deletes a single picture from a collection in Panoramax if the user owns the collection or is admin.
"""
if request.method != 'POST':
return HttpResponseBadRequest("Only POST method is allowed")
user_config = get_object_or_404(UserConfig, user=request.user)
if not user_config.can_access_view('panoramax'):
return HttpResponseForbidden("Access denied to Panoramax")
collection_id_str = str(collection_id)
picture_id_str = str(picture_id)
# Check authorization: user must own the sequence OR be staff/superuser
is_authorized = request.user.is_staff or request.user.is_superuser
if not is_authorized:
user_uploads = PanoramaxUpload.objects.filter(user=request.user, status='completed')
for up in user_uploads:
if up.associated_collections:
for item in up.associated_collections:
item_id = item.get('id') if isinstance(item, dict) else item
if item_id == collection_id_str:
is_authorized = True
break
if is_authorized:
break
if not is_authorized:
return HttpResponseForbidden("You are not authorized to delete pictures from this collection")
# Send DELETE request to Panoramax API
url = f"{settings.PANORAMAX_API_URL.rstrip('/')}/collections/{collection_id_str}/items/{picture_id_str}"
headers = {}
if settings.PANORAMAX_API_KEY:
headers['Authorization'] = f'Bearer {settings.PANORAMAX_API_KEY}'
try:
response = requests.delete(url, headers=headers, timeout=15)
if response.status_code not in [200, 204, 404]:
return HttpResponse(
f"API error: {response.text}",
status=response.status_code
)
return HttpResponse("Picture deleted successfully")
except Exception as e:
traceback.print_exc()
return HttpResponse(f"Error calling Panoramax API: {e}", status=500)