import os
import base64
import hashlib
import importlib.util
import json
import logging
import mimetypes
import re
import shutil
import sqlite3
import threading
import uuid
import io
import zipfile
import subprocess
import sys
import signal
import time
import ipaddress
import smtplib
import ssl
from email.message import EmailMessage
from html import escape, unescape
from urllib.parse import urlparse
from datetime import datetime, timedelta, date
from functools import wraps
from pathlib import Path

from flask import (
    Flask,
    abort,
    flash,
    g,
    has_request_context,
    jsonify,
    redirect,
    render_template,
    request,
    send_file,
    send_from_directory,
    session,
    url_for,
)
from werkzeug.security import check_password_hash, generate_password_hash
from werkzeug.utils import secure_filename
import ai_descriptions
import inspection_reports
import lead_replies
import lead_dispatch
import turbo_listing
import inbox_reader
import page_content_gen
from chatbot_defaults import default_chatbot_greeting, default_chatbot_fallback_message, default_chatbot_system_prompt
from trust_features import audit as trust_audit, migrate_trust_schema, register_trust_features
from migrations.car_dealer_migration import migrate_car_dealer_schema
from migrations.homepage_cards import migrate as migrate_homepage_cards

try:
    import requests
except ImportError:  # pragma: no cover - dependency is required for Fireworks content generation.
    requests = None

try:
    from cryptography.fernet import Fernet, InvalidToken
except ImportError:  # pragma: no cover - dependency is required for encrypted API keys.
    Fernet = None
    InvalidToken = Exception

try:
    from openai import OpenAI
except ImportError:  # pragma: no cover - dependency is required for AI image editing.
    OpenAI = None

try:
    from PIL import Image, ImageOps
except ImportError:  # pragma: no cover - dependency is required for local image optimization.
    Image = None
    ImageOps = None


BASE_DIR = Path(__file__).resolve().parent
BUNDLED_INSTANCE_DIR = BASE_DIR / "instance"
BUNDLED_UPLOADS_DIR = BASE_DIR / "static" / "uploads"
CARFAX_REPORTS_DIR = os.path.join(BASE_DIR, "carfax_reports")
PERSISTENT_DATA_ROOT = Path(
    os.environ.get("PERSISTENT_DATA_DIR")
    or os.environ.get("RAILWAY_VOLUME_MOUNT_PATH")
    or ""
).resolve() if (os.environ.get("PERSISTENT_DATA_DIR") or os.environ.get("RAILWAY_VOLUME_MOUNT_PATH")) else None
INSTANCE_DIR = (PERSISTENT_DATA_ROOT / "instance") if PERSISTENT_DATA_ROOT else BUNDLED_INSTANCE_DIR
UPLOADS_DIR = (PERSISTENT_DATA_ROOT / "uploads") if PERSISTENT_DATA_ROOT else BUNDLED_UPLOADS_DIR
SITE_UPLOADS_DIR = UPLOADS_DIR / "site"
LISTING_UPLOADS_DIR = UPLOADS_DIR / "listings"
AI_EDIT_UPLOADS_DIR = UPLOADS_DIR / "ai_edits"
GENERATED_VISUALS_UPLOADS_DIR = UPLOADS_DIR / "generated_visuals"
DATABASE = INSTANCE_DIR / "inventory.sqlite3"
LOCAL_OPENAI_SECRET_FILE = INSTANCE_DIR / "openai_settings_secret.key"
TRANSLATION_JOB_FILE = INSTANCE_DIR / "translation_job.json"
TRANSLATION_JOB_LOG_FILE = INSTANCE_DIR / "translation_job.log"
TRANSLATION_JOB_CONFIG_FILE = INSTANCE_DIR / "translation_job_config.json"
IMAGE_VARIANT_JOB_FILE = INSTANCE_DIR / "image_variant_job.json"
IMAGE_VARIANT_LOG_FILE = INSTANCE_DIR / "image_variant_job.log"
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
ADMIN_URL_PREFIX = os.environ.get("ADMIN_URL_PREFIX", "/stock-room-7f4c2a91").strip() or "/stock-room-7f4c2a91"
if not ADMIN_URL_PREFIX.startswith("/"):
    ADMIN_URL_PREFIX = "/" + ADMIN_URL_PREFIX
ADMIN_URL_PREFIX = ADMIN_URL_PREFIX.rstrip("/")
DEFAULT_AI_IMAGE_EDIT_PROMPT = """TASK: Replace only the existing dealership/storefront sign on the building with the provided website logo.

ABSOLUTE RULES:
- The base image is a vehicles/product photo. The logo image is only a reference for the building sign.
- Use ONLY the provided website logo. Do not redesign, restyle, recolor, redraw, or invent branding.
- Do NOT place the logo on the vehicle, sedan boom, cab, counterweight, bucket, stickers, windows, vehicles, ground, sky, or any object except the existing building sign/signboard.
- Do NOT replace or alter manufacturer decals such as SANY, CAT, JCB, Kubota, model numbers, safety stickers, or any vehicle markings.
- If no suitable building/storefront sign area is clearly visible, leave the image essentially unchanged rather than inventing a sign.
- Keep the camera angle, crop, object positions, vehicle shape, building geometry, windows, doors, yard, vehicles, sky, and lighting unchanged.
- Replace the existing sign in its original physical position only. Match the exact perspective, size, material, shadows, reflections, and lighting of that sign surface.
- Preserve the logo proportions, typography, spacing, and colors as closely as possible.
- The result must be consistent across images: the same sign/logo style, not new decals or random placements.

OUTPUT:
- A photorealistic edit where only the visible building sign is replaced with the provided logo.
- All other parts of the image should remain untouched.
"""

DEFAULT_AI_ANGLE_GENERATION_PROMPT = """TASK: Generate exactly one new photorealistic inventory/gallery image of the same exact vehicle from the requested camera angle.

ABSOLUTE IDENTITY RULES:
- Use the provided reference image as the only identity source for the vehicle.
- Preserve the same vehicle type, make, model, trim, body style, wheels, mirrors, decals, wear, dirt, color, damage, and condition.
- Do NOT invent a replacement vehicle, showroom render, brochure image, collage, diagram, or stylized illustration.
- Do NOT change manufacturer decals, model numbers, registration plates, warning labels, or visible wear patterns.
- Keep the dealership yard / building / ground / lighting plausible and consistent with the reference.

ANGLE RULES:
- The requested angle is mandatory: {angle_name}.
- The new image must be clearly different from the reference image and clearly different from common near-duplicates.
- Do NOT make tiny angle changes such as 65, 70, and 75 degrees. If the request says side view, make it a true side view around 90 degrees. If it says rear three-quarter, make it clearly rear-biased around 135 degrees.
- Keep the whole vehicle visible unless the reference image itself makes that impossible.
- Keep a realistic camera height for vehicle sales photography.

OUTPUT:
- One realistic sales photo from {angle_name}.
- No text overlay, no watermark, no extra logos, no price badge, no people.

Listing title: {listing_title}.
"""


def default_ai_image_edit_prompt():
    return globals().get(
        "DEFAULT_AI_IMAGE_EDIT_PROMPT",
        "TASK: Replace only the existing dealership/storefront sign on the building with the provided website logo.\n"
        "Do not place the logo on the vehicle, vehicle, ground, sky, windows, or any other surface.\n"
        "Do not alter manufacturer decals or model numbers. If no building sign is visible, leave the image essentially unchanged.",
    )


def default_ai_angle_generation_prompt():
    return globals().get("DEFAULT_AI_ANGLE_GENERATION_PROMPT", "")


LANGUAGES = ("en", "fr")
LANGUAGE_NAMES = {
    "en": "English",
    "fr": "Français",
}

# Internal language codes must remain ISO language codes for routes/DB columns.
# Public/UI abbreviations can use familiar country-style labels where buyers expect them.
LANGUAGE_DISPLAY_CODES = {
    "en": "EN",
    "fr": "FR",
}

def language_display_code(code):
    return LANGUAGE_DISPLAY_CODES.get((code or "").lower(), (code or "").upper())
TRANS_FIELDS = ("title", "short_description", "description_html", "seo_title", "meta_description")
PAGE_TRANS_FIELDS = ("title", "excerpt", "content_html", "seo_title", "meta_description")
FAQ_TRANS_FIELDS = ("question", "answer")
IMPORT_SUMMARY_FIELDS = (
    ("total_source_listings", "Source listings"),
    ("listings_selected", "Selected"),
    ("listings_created", "Created"),
    ("listings_updated", "Updated"),
    ("listings_skipped", "Skipped"),
    ("images_found", "Images found"),
    ("images_imported", "Images imported"),
    ("images_copied", "Images copied"),
    ("missing_image_files", "Missing files"),
    ("primary_images_set", "Primary set"),
    ("featured_images_updated", "Featured updated"),
    ("repaired_listings", "Repaired"),
    ("image_variants_generated", "Variants"),
)
FIREWORKS_CHAT_COMPLETIONS_URL = "https://api.fireworks.ai/inference/v1/chat/completions"
FIREWORKS_MODELS = (
    ("GLM 5.2 - recommended for all text tasks", "accounts/fireworks/models/glm-5p2"),
    ("DeepSeek V4 Pro - quality alternative", "accounts/fireworks/models/deepseek-v4-pro"),
    ("DeepSeek V4 Flash - speed/chat alternative", "accounts/fireworks/models/deepseek-v4-flash"),
    ("Kimi K2.6 - multimodal/agentic alternative", "accounts/fireworks/models/kimi-k2p6"),
    ("Kimi K2.6 Fast", "accounts/fireworks/routers/kimi-k2p6-fast"),
    ("GLM 5.1", "accounts/fireworks/models/glm-5p1"),
    ("GLM 5.1 Fast", "accounts/fireworks/routers/glm-5p1-fast"),
    ("Kimi K2.5 - legacy cost-balanced", "accounts/fireworks/models/kimi-k2p5"),
)
DEFAULT_FIREWORKS_MODEL = "accounts/fireworks/models/glm-5p2"
DEFAULT_TRANSLATION_MODEL = "accounts/fireworks/models/glm-5p2"
DEFAULT_CONTENT_MODEL = "accounts/fireworks/models/glm-5p2"
DEFAULT_CHATBOT_MODEL = "accounts/fireworks/models/glm-5p2"
DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL = "accounts/fireworks/models/kimi-k2p6"
FIREWORKS_IMAGE_REVIEW_MODELS = (
    ("Kimi K2.5 Vision - shared Fireworks reviewer", "accounts/fireworks/models/kimi-k2p5"),
    ("GLM-4.5V - Z.ai vision reviewer, requires Fireworks on-demand access", "accounts/fireworks/models/glm-4p5v"),
)

INTERFACE_TRANSLATION_DEFAULTS = {
    "home": {"en":"Home","fr":"Accueil"},
    "inventory": {"en":"Inventory","fr":"Inventaire"},
    "back_to_inventory": {"en":"Back to inventory","fr":"Retour à l'inventaire"},
    "vehicles": {"en":"Vehicles","fr":"Véhicules"},
    "available": {"en":"Available","fr":"Disponible"},
    "reserved": {"en":"Reserved","fr":"Réservé"},
    "sold": {"en":"Sold","fr":"Vendu"},
    "price_on_request": {"en":"Price on request","fr":"Prix sur demande"},
    "enquire_now": {"en":"Enquire now","fr":"Demande d'information"},
    "enquiry_copy": {"en":"Ask about inspection details, financing, Canada-wide delivery, or availability for this vehicle.","fr":"Renseignez-vous sur les détails d'inspection, le financement, la livraison partout au Canada ou la disponibilité de ce véhicule."},
    "call": {"en":"Call","fr":"Appeler"},
    "email": {"en":"Email","fr":"Courriel"},
    "name": {"en":"Name","fr":"Nom"},
    "phone": {"en":"Phone","fr":"Téléphone"},
    "message": {"en":"Message","fr":"Message"},
    "default_enquiry": {"en":"I am interested in {title} ({reference}).","fr":"Je suis intéressé par {title} ({reference})."},
    "send_enquiry": {"en":"Send Enquiry","fr":"Envoyer la demande"},
    "machine_highlights": {"en":"Vehicle highlights","fr":"Points forts du véhicule"},
    "quick_buyer_summary": {"en":"Quick buyer summary","fr":"Résumé rapide pour l'acheteur"},
    "model_year": {"en":"{value} model year","fr":"Année modèle {value}"},
    "recorded_hours": {"en":"{value} kilometers","fr":"{value} kilomètres"},
    "hours_on_request": {"en":"Odometer on request","fr":"Kilométrage sur demande"},
    "reference_tracking": {"en":"CA reference {reference} for clear enquiry tracking","fr":"Référence CA {reference} pour un suivi clair des demandes"},
    "extra_media": {"en":"Additional photos, video or inspection details available on request","fr":"Photos, vidéos ou détails d'inspection supplémentaires disponibles sur demande"},
    "delivery_support": {"en":"Delivery and export support can be discussed before purchase","fr":"Les options de livraison et d'exportation peuvent être discutées avant l'achat"},
    "machine_overview": {"en":"Vehicle overview","fr":"Aperçu du véhicule"},
    "basic_specifications": {"en":"Basic specifications","fr":"Spécifications de base"},
    "reference": {"en":"Reference","fr":"Référence"},
    "year": {"en":"Year","fr":"Année"},
    "make": {"en":"Make","fr":"Marque"},
    "model": {"en":"Model","fr":"Modèle"},
    "hours": {"en":"Hours","fr":"Kilométrage"},
    "category": {"en":"Category","fr":"Catégorie"},
    "status": {"en":"Status","fr":"Statut"},
    "similar_vehicles": {"en":"Similar Vehicles","fr":"Véhicules similaires"},
    "view": {"en":"View","fr":"Voir"},
    "view_details": {"en":"View details","fr":"Voir les détails"},
    "guide_price": {"en":"Guide price","fr":"Prix indicatif"},
    "why_buy": {"en":"Why buy from us?","fr":"Pourquoi acheter chez nous ?"},
    "support_title": {"en":"{COMPANY_NAME} support","fr":"Soutien {COMPANY_NAME}"},
    "shopping_tools": {"en":"Shopping tools","fr":"Outils d'achat"},
    "before_purchase": {"en":"Before purchase","fr":"Avant l'achat"},
    "category_sedan": {"en":"Sedan","fr":"Berline"},
    "category_SUV": {"en":"SUV","fr":"VUS"},
    "category_wheel_loader": {"en":"Truck","fr":"Camionnette"},
}

INTERFACE_TRANSLATION_DEFAULTS.update({
    "supplier_bullet": {"en":"Canadian used vehicles supplier.","fr":"Fournisseur canadien de véhicules d'occasion."},
    "vat_bullet": {"en":"OMVIC-registered Canadian dealer.","fr":"Concessionnaire canadien inscrit auprès de l'OMVIC."},
    "clear_reference_bullet": {"en":"Clear CA reference for each vehicle.","fr":"Référence CA claire pour chaque véhicule."},
    "safety_inspection_bullet": {"en":"Independent inspections welcome where practical.","fr":"Inspections indépendantes bienvenues lorsque c'est possible."},
    "delivery_region_bullet": {"en":"Canada-wide delivery/export support.","fr":"Soutien pour la livraison et l'exportation partout au Canada."},
    "confirm_location_bullet": {"en":"Confirm current location and delivery route.","fr":"Confirmez l'emplacement actuel et l'itinéraire de livraison."},
    "request_media_bullet": {"en":"Request any extra photos or video you need.","fr":"Demandez les photos ou vidéos supplémentaires dont vous avez besoin."},
    "confirm_tax_bullet": {"en":"Confirm HST and licensing details case by case.","fr":"Confirmer les détails de TPS/TVH et d'immatriculation au cas par cas."},
    "company_details": {"en":"Company details","fr":"Détails de l'entreprise"},
    "contact": {"en":"Contact","fr":"Contact"},
    "information": {"en":"Information","fr":"Informations"},
    "buyer_reminder": {"en":"Buyer reminder","fr":"Rappel à l'acheteur"},
    "buyer_reminder_text": {"en":"When enquiring, include the CA reference, destination and any inspection questions so we can respond with the correct vehicle details.","fr":"Lors de votre demande, incluez la référence CA, la destination et toute question d'inspection afin que nous puissions répondre avec les bons détails du véhicule."}
})

INTERFACE_TRANSLATION_DEFAULTS.update({
    "listing_translation_fallback_notice": {
        "en": "This vehicle description is currently available in English only.",
        "fr": "La description de ce véhicule est actuellement disponible en anglais seulement.",
        "it": "La descrizione di questa macchina è attualmente disponibile solo in inglese.",
        "de": "Diese Maschinenbeschreibung ist derzeit nur auf Englisch verfügbar.",
        "nl": "Deze machinebeschrijving is momenteel alleen in het Engels beschikbaar.",
        "no": "Denne maskinbeskrivelsen er foreløpig bare tilgjengelig på engelsk.",
        "da": "Denne maskinbeskrivelse er i øjeblikket kun tilgængelig på engelsk.",
        "sv": "Den här maskinbeskrivningen är för närvarande endast tillgänglig på engelska.",
        "pl": "Opis tej maszyny jest obecnie dostępny tylko w języku angielskim."
    }
})

INTERFACE_TRANSLATION_DEFAULTS.update({
    "previous_image": {"en":"Previous image","fr":"Image précédente"},
    "next_image": {"en":"Next image","fr":"Image suivante"},
    "related_empty": {"en":"Related stock will appear here as inventory grows.","fr":"Le stock associé apparaîtra ici à mesure que l'inventaire s'agrandit."}
})

INTERFACE_TRANSLATION_DEFAULTS.update({
    "nav_home": {"en":"Home","fr":"Accueil"},
    "nav_inventory": {"en":"Inventory","fr":"Inventaire"},
    "nav_about_us": {"en":"About Us","fr":"À propos"},
    "nav_delivery": {"en":"Delivery","fr":"Livraison"},
    "nav_shipping": {"en":"Delivery","fr":"Livraison"},
    "nav_contact": {"en":"Contact","fr":"Contact"},
    "language": {"en":"Language","fr":"Langue"},
})

INTERFACE_TRANSLATION_DEFAULTS.update({

    "home_export_ready": {"en":"Export-ready vehicles","fr":"Véhicules prêts pour l'exportation"},
    "view_inventory": {"en":"View Inventory","fr":"Voir l'inventaire"},
    "request_export_quote": {"en":"Request Export Quote","fr":"Demander un devis d'exportation"},
    "trust_uk_based": {"en":"Canada based","fr":"Basé au Canada"},
    "trust_vat_registered": {"en":"HST registered","fr":"Inscrit à la TVH"},
    "trust_clear_references": {"en":"Clear CA references","fr":"Références CA claires"},
    "trust_detailed_galleries": {"en":"Detailed galleries","fr":"Galeries détaillées"},
    "trust_safety_inspected": {"en":"Safety inspected","fr":"Inspection de sécurité"},
    "trust_europe_delivery": {"en":"Canada-wide delivery","fr":"Livraison partout au Canada"},
    "featured_listings": {"en":"Featured listings","fr":"Annonces en vedette"},
    "ready_to_work_vehicles": {"en":"Ready-to-work vehicles","fr":"Véhicules prêts à prendre la route"},
    "browse_all_inventory": {"en":"Browse all inventory","fr":"Parcourir tout l'inventaire"},
    "no_featured_vehicles": {"en":"No featured vehicles are published yet.","fr":"Aucun véhicule en vedette n'est encore publié."},
    "clear_references_heading": {"en":"Clear References","fr":"Références claires"},
    "clear_references_text": {"en":"Every published vehicle uses a CA reference so enquiries, photos, inspection questions and paperwork stay connected.","fr":"Chaque véhicule publié utilise une référence CA afin que les demandes, photos, questions d'inspection et documents restent liés."},
    "export_support_heading": {"en":"Canada-Wide Support","fr":"Soutien partout au Canada"},
    "export_support_text": {"en":"Canada-wide delivery and paperwork support.","fr":"Livraison et documentation partout au Canada."},
    "safety_inspected_heading": {"en":"Inspection Friendly","fr":"Inspections bienvenues"},
    "safety_inspected_text": {"en":"Request additional photos, videos, written notes or third-party inspection options before finalising a vehicle.","fr":"Demandez des photos, vidéos, notes écrites ou options d'inspection par un tiers avant de finaliser un véhicule."},
    "buyer_questions": {"en":"Buyer questions","fr":"Questions d'acheteur"},
    "footer_company_description": {"en":"Used vehicle sales, cars and trucks, and practical Canada-wide support from {COMPANY_NAME}.","fr":"Vente de véhicules d'occasion, voitures et camions, avec soutien partout au Canada par {COMPANY_NAME}."},
    "used_vehicles": {"en":"Used vehicles","fr":"Véhicules d'occasion"},
    "cars_trucks_suvs": {"en":"Cars, trucks & SUVs","fr":"Voitures, camionnettes et VUS"},
    "export_support": {"en":"Delivery support","fr":"Soutien à la livraison"},
    "uk_ireland_europe": {"en":"Across Canada","fr":"Partout au Canada"},

})

INTERFACE_TRANSLATION_DEFAULTS.update({
    "nav_tax": {"en":"Tax & HST","fr":"TPS/TVH"},
    "nav_location": {"en":"Our Location","fr":"Notre emplacement"},
    "nav_why_buy": {"en":"Why Buy From Us","fr":"Pourquoi nous choisir"},
    "nav_warranty": {"en":"Warranty","fr":"Garantie"},
    "nav_financing": {"en":"Financing","fr":"Financement"},
    "nav_faq": {"en":"FAQ","fr":"FAQ"},
    "nav_privacy": {"en":"Privacy Policy","fr":"Confidentialité"},
    "nav_terms": {"en":"Terms & Conditions","fr":"Conditions générales"}
})

INTERFACE_TRANSLATION_DEFAULTS.update({
    'current_inventory': {"en":"Current inventory","fr":"Inventaire actuel"},
    'used_vehicles_for_sale': {"en":"Used vehicles for sale","fr":"Véhicules d'occasion à vendre"},
    'inventory_intro': {"en":"Browse published stock with clear references, export-friendly contact options, and multilingual page foundations.","fr":"Parcourez le stock publié avec des références claires, des options de contact adaptées et des fondations de page multilingues."},
    'vehicles_listed': {"en":"vehicles listed","fr":"véhicules listés"},
    'all_categories': {"en":"All categories","fr":"Toutes les catégories"},
    'all_makes': {"en":"All makes","fr":"Toutes les marques"},
    'all_models': {"en":"All models","fr":"Tous les modèles"},
    'all_years': {"en":"All years","fr":"Toutes les années"},
    'any_status': {"en":"Any status","fr":"Tout statut"},
    'newest': {"en":"Newest","fr":"Plus récents"},
    'price_low_high': {"en":"Price low to high","fr":"Prix croissant"},
    'price_high_low': {"en":"Price high to low","fr":"Prix décroissant"},
    'year_newest': {"en":"Year newest","fr":"Année récente"},
    'apply_filters': {"en":"Apply Filters","fr":"Appliquer les filtres"},
    'reset': {"en":"Reset","fr":"Réinitialiser"},
    'no_inventory': {"en":"No published inventory is available yet.","fr":"Aucun inventaire publié n'est encore disponible."},
    'contact_stock_note': {"en":"For stock enquiries, include the CA reference and vehicle title.","fr":"Pour les demandes de stock, incluez la référence CA et le titre du véhicule."},
    'contact_export_note': {"en":"For delivery, include destination city/province and any unloading notes.","fr":"Pour la livraison, incluez la ville/province de destination et toute note de déchargement."},
    'general_enquiry': {"en":"General enquiry","fr":"Demande générale"},
    'send_message': {"en":"Send Message","fr":"Envoyer le message"},
    'next_step': {"en":"Next step","fr":"Prochaine étape"},
    'contact_us': {"en":"Contact Us","fr":"Contactez-nous"},
    'export_focused_vehicles': {"en":"Inventory highlights","fr":"Véhicules en vedette"},
    'view_matching_stock': {"en":"View Matching Stock","fr":"Voir le stock correspondant"},
    'ask_about_export': {"en":"Ask About This Vehicle","fr":"Renseignez-vous sur ce véhicule"},
    'no_matching_vehicles': {"en":"No matching vehicles is currently published. Contact us for upcoming stock.","fr":"Aucun véhicule correspondant n'est publié actuellement. Contactez-nous pour le stock à venir."},
    'header_tagline': {"en":"Quality used vehicles","fr":"Véhicules d'occasion de qualité"},
    'chat': {"en":"Chat","fr":"Clavardage"},
    'vehicles_assistant': {"en":"Vehicles assistant","fr":"Assistant véhicules"},
    'chat_test_drive_q': {"en":"Can I book a test drive?","fr":"Puis-je réserver un essai routier ?"},
    'chat_hst_q': {"en":"Is HST included in the price?","fr":"Les prix incluent-ils la TPS/TVH ?"},
    'chat_delivery_q': {"en":"Do you deliver across Canada?","fr":"Livrez-vous partout au Canada ?"},
    'chat_quote_q': {"en":"Get a quote","fr":"Obtenir un devis"},
    'chat_location_q': {"en":"Where is this vehicle located?","fr":"Où se trouve ce véhicule ?"},
    'ask_about_vehicle': {"en":"Ask about this vehicle","fr":"Renseignez-vous sur ce véhicule"},
    'leave_details': {"en":"Leave details and the team will confirm this properly.","fr":"Laissez vos coordonnées et l'équipe vous confirmera les détails."},
    'send_details': {"en":"Send details","fr":"Envoyer les détails"},
    'send': {"en":"Send","fr":"Envoyer"},
})

INTERFACE_TRANSLATION_DEFAULTS.update({"address": {"en":"Address","fr":"Adresse"}})

INTERFACE_TRANSLATION_DEFAULTS.update({
    # -- Homepage hero & CTAs --
    "home_hero_eyebrow": {"en": "{COMPANY_NAME}", "fr": "{COMPANY_NAME}"},
    "browse_inventory": {"en": "Browse Inventory", "fr": "Parcourir l'inventaire"},
    "used_cars": {"en": "Used Cars", "fr": "Voitures d'occasion"},
    "vehicle_assistant": {"en": "Vehicle assistant", "fr": "Assistant véhicules"},
    "vat_number": {"en": "HST number", "fr": "Numéro de TVH"},
    "vat_short": {"en": "HST", "fr": "TVH"},
    "quick_search_by_category": {"en": "Quick Search by Category", "fr": "Recherche rapide par catégorie"},
    "view_all_inventory": {"en": "View All Inventory", "fr": "Voir tout l'inventaire"},

    # -- Homepage featured vehicles --
    "featured_vehicles_label": {"en": "Featured Vehicles", "fr": "Véhicules en vedette"},
    "featured_vehicles_heading": {"en": "Hand-picked quality vehicles", "fr": "Véhicules de qualité sélectionnés"},

    # -- Homepage trust badges --
    "trust_carfax": {"en": "CARFAX reports available", "fr": "Rapports CARFAX disponibles"},
    "view_inspection_report": {"en": "View Inspection Report", "fr": "Voir le rapport d'inspection"},
    "trust_financing": {"en": "Financing available", "fr": "Financement disponible"},
    "trust_trade_ins": {"en": "Trade-ins welcome", "fr": "Reprises acceptées"},
    "trust_canada_delivery": {"en": "Canada-wide delivery", "fr": "Livraison partout au Canada"},

    # -- Homepage body-style section --
    "shop_by_body_style": {"en": "Shop by Body Style", "fr": "Par type de carrosserie"},
    "find_your_perfect_vehicle": {"en": "Find your perfect vehicle", "fr": "Trouvez votre véhicule idéal"},
    "sedans": {"en": "Sedans", "fr": "Berlines"},
    "suvs": {"en": "SUVs", "fr": "VUS"},
    "trucks": {"en": "Trucks", "fr": "Camionnettes"},
    "coupes": {"en": "Coupes", "fr": "Coupés"},
    "hatchbacks": {"en": "Hatchbacks", "fr": "Hayons"},
    "vans": {"en": "Vans", "fr": "Fourgonnettes"},
    "browse": {"en": "Browse", "fr": "Parcourir"},

    # -- Homepage makes section --
    "popular_makes": {"en": "Popular Makes", "fr": "Marques populaires"},
    "browse_by_make": {"en": "Browse by Make", "fr": "Parcourir par marque"},

    # -- Homepage recently added --
    "recently_added_label": {"en": "Recently Added", "fr": "Ajoutés récemment"},
    "recently_added_heading": {"en": "Fresh arrivals this week", "fr": "Nouvelles arrivées cette semaine"},

    # -- Inventory page --
    "inventory_browse_title": {"en": "Browse Inventory", "fr": "Parcourir l'inventaire"},
    "used_cars_for_sale": {"en": "Used Cars for Sale in Canada", "fr": "Voitures d'occasion à vendre au Canada"},
    "inventory_browse_subtitle": {"en": "Browse our certified inventory", "fr": "Parcourez notre inventaire certifié"},
    "vehicles_found": {"en": "vehicles found", "fr": "véhicules trouvés"},
    "search_stock_make_model": {"en": "Search make, model, stock #", "fr": "Rechercher marque, modèle, n° stock"},

    # -- Inventory filters --
    "any": {"en": "Any", "fr": "Tous"},
    "automatic": {"en": "Automatic", "fr": "Automatique"},
    "manual": {"en": "Manual", "fr": "Manuelle"},
    "odometer_lowest": {"en": "Odometer lowest", "fr": "Kilométrage le plus bas"},
    "newest_listed": {"en": "Newest listed", "fr": "Plus récents"},
    "search": {"en": "Search", "fr": "Rechercher"},
    "year_from": {"en": "Year from", "fr": "Année de"},
    "year_to": {"en": "Year to", "fr": "Année à"},
    "min_price": {"en": "Min price", "fr": "Prix min"},
    "max_price": {"en": "Max price", "fr": "Prix max"},
    "odometer_min": {"en": "Odometer min", "fr": "Kilométrage min"},
    "odometer_max": {"en": "Odometer max", "fr": "Kilométrage max"},
    "certified_pre_owned": {"en": "Certified Pre-Owned", "fr": "Certifié d'occasion"},
    "filters": {"en": "Filters", "fr": "Filtres"},
    "close": {"en": "Close", "fr": "Fermer"},
    "body_style": {"en": "Body Style", "fr": "Carrosserie"},
    "transmission": {"en": "Transmission", "fr": "Transmission"},
    "fuel_type": {"en": "Fuel Type", "fr": "Type de carburant"},
    "drivetrain": {"en": "Drivetrain", "fr": "Type de traction"},
    "from": {"en": "From", "fr": "De"},
    "to": {"en": "To", "fr": "À"},
    "price": {"en": "Price", "fr": "Prix"},
    "min": {"en": "Min", "fr": "Min"},
    "max": {"en": "Max", "fr": "Max"},
    "odometer": {"en": "Odometer", "fr": "Kilométrage"},
    "sort_by": {"en": "Sort by", "fr": "Trier par"},
    "active_filters": {"en": "Active filters", "fr": "Filtres actifs"},
    "clear_all": {"en": "Clear all", "fr": "Tout effacer"},
    "no_vehicles_found": {"en": "No vehicles found", "fr": "Aucun véhicule trouvé"},
    "no_vehicles_suggestion": {"en": "Try adjusting your filters or browse all inventory.", "fr": "Essayez d'ajuster vos filtres ou parcourez tout l'inventaire."},
    "clear_all_filters": {"en": "Clear all filters", "fr": "Effacer tous les filtres"},

    # -- Listing card (used by home + inventory) --
    "certified": {"en": "Certified", "fr": "Certifié"},
    "new_arrival": {"en": "New Arrival", "fr": "Nouvelle arrivée"},
    "save": {"en": "Save", "fr": "Économisez"},
    "vs_msrp": {"en": "vs MSRP", "fr": "vs PDSF"},
    "odo": {"en": "Odo", "fr": "Kilo"},
    "km": {"en": "km", "fr": "km"},
    "stock": {"en": "Stock", "fr": "Stock"},

    # -- Dynamic body-style filter chips (from inventory.html) --
    "body_style_sedan": {"en": "Sedan", "fr": "Berline"},
    "body_style_suv": {"en": "SUV", "fr": "VUS"},
    "body_style_truck": {"en": "Truck", "fr": "Camionnette"},
    "body_style_coupe": {"en": "Coupe", "fr": "Coupé"},
    "body_style_hatchback": {"en": "Hatchback", "fr": "Hayon"},
    "body_style_van": {"en": "Van", "fr": "Fourgonnette"},
    # -- Dynamic fuel-type filter chips (from inventory.html) --
    "fuel_gasoline": {"en": "Gasoline", "fr": "Essence"},
    "fuel_diesel": {"en": "Diesel", "fr": "Diesel"},
    "fuel_electric": {"en": "Electric", "fr": "Électrique"},
    "fuel_hybrid": {"en": "Hybrid", "fr": "Hybride"},
})

# Task 15: Trust interface strings with FR translations.
# These keys are inserted into interface_translations by the trust migration (EN-only).
# Adding them here allows init_db to backfill the FR column on existing rows.
INTERFACE_TRANSLATION_DEFAULTS.update({
    "verified_uk_company": {"en": "Verified Canadian Dealer", "fr": "Concessionnaire canadien vérifié"},
    "verification_intro": {"en": "{company_name} is an OMVIC-registered Canadian dealer. You can verify our dealer license information.", "fr": "{company_name} est un concessionnaire canadien inscrit à l'OMVIC. Vous pouvez vérifier nos informations de licence de concessionnaire."},
    "verify_companies_house": {"en": "Verify dealer license", "fr": "Vérifier la licence de concessionnaire"},
    "company_number": {"en": "Dealer license number", "fr": "Numéro de licence de concessionnaire"},
    "vat_number": {"en": "HST number", "fr": "Numéro de TVH"},
    "registered_office": {"en": "Registered office", "fr": "Siège social"},
    "sales_phone": {"en": "Sales phone", "fr": "Téléphone des ventes"},
    "sales_email": {"en": "Sales email", "fr": "Courriel des ventes"},
    "current_machine_location": {"en": "Current vehicle location", "fr": "Emplacement actuel du véhicule"},
    "inspection_or_collection": {"en": "Inspection or collection", "fr": "Inspection ou collecte"},
    "available_by_appointment": {"en": "Available by appointment", "fr": "Disponible sur rendez-vous"},
    "delivery_insured_europe": {"en": "Insured delivery available across Canada", "fr": "Livraison assurée disponible partout au Canada"},
    "listing_verification": {"en": "Listing Verification", "fr": "Vérification de l'annonce"},
    "listing_information_verified": {"en": "Availability and listing details last confirmed: {date}", "fr": "Disponibilité et détails de l'annonce confirmés le : {date}"},
    "serial_number_status": {"en": "Serial number", "fr": "Numéro de série"},
    "inspection_status": {"en": "Inspection status", "fr": "Statut d'inspection"},
    "condition_check_status": {"en": "Condition check", "fr": "Vérification de l'état"},
    "inspection_date": {"en": "Inspection date", "fr": "Date d'inspection"},
    "availability_checked_today": {"en": "Availability checked: {date}", "fr": "Disponibilité vérifiée le : {date}"},
    "inspection_not_checked": {"en": "Not checked", "fr": "Non vérifié"},
    "inspection_pending": {"en": "Inspection pending", "fr": "Inspection en attente"},
    "inspection_visually_checked": {"en": "Visually checked", "fr": "Inspection visuelle effectuée"},
    "inspection_mechanically_inspected": {"en": "Mechanically inspected", "fr": "Inspection mécanique effectuée"},
    "inspection_third_party_inspected": {"en": "Third-party inspected", "fr": "Inspection par un tiers effectuée"},
    "inspection_documentation_only": {"en": "Documentation only", "fr": "Documentation seulement"},
    "condition_not_checked": {"en": "Not checked", "fr": "Non vérifié"},
    "condition_visually_checked": {"en": "Visually checked", "fr": "Inspection visuelle effectuée"},
    "condition_mechanically_checked": {"en": "Mechanically checked", "fr": "Inspection mécanique effectuée"},
    "condition_third_party_checked": {"en": "Third-party checked", "fr": "Vérification par un tiers effectuée"},
    "condition_documentation_only": {"en": "Documentation only", "fr": "Documentation seulement"},
    "condition_pending": {"en": "Pending", "fr": "En attente"},
    "collection_available": {"en": "Collection available", "fr": "Collecte disponible"},
    "delivery_available": {"en": "Delivery available", "fr": "Livraison disponible"},
    "yes": {"en": "Yes", "fr": "Oui"},
    "no": {"en": "No", "fr": "Non"},
    "vat_short": {"en": "HST applied according to your province of registration.", "fr": "TVH appliquée selon votre province d'immatriculation."},
    "vat_cross_border_title": {"en": "HST and licensing", "fr": "TVH et immatriculation"},
    "vat_general_guidance": {"en": "HST is applied according to your province of registration. All pricing is in Canadian dollars (CAD). There are no hidden customs or import fees.", "fr": "La TVH est appliquée selon votre province d'immatriculation. Tous les prix sont en dollars canadiens (CAD). Il n'y a pas de frais de douane ou d'importation cachés."},
    "vat_business_guidance": {"en": "For business purchases, HST may be recoverable through your regular input tax credit filings. We provide a proper invoice with our HST number.", "fr": "Pour les achats d'entreprise, la TVH peut être récupérable par vos déclarations régulières de crédits de taxe sur les intrants. Nous fournissons une facture appropriée avec notre numéro de TVH."},
    "vat_private_guidance": {"en": "For private purchases, applicable HST and licensing fees will be shown clearly in the final quotation.", "fr": "Pour les achats privés, la TVH applicable et les frais d'immatriculation seront clairement indiqués dans le devis final."},
    "vat_final_quote": {"en": "The final quotation and invoice will confirm HST, licensing, and any dealer fees for the specific transaction.", "fr": "Le devis final et la facture confirmeront la TVH, l'immatriculation et les frais de concessionnaire pour la transaction spécifique."},
    "vat_disclaimer": {"en": "This information is general guidance and does not replace professional tax advice.", "fr": "Ces informations sont des conseils généraux et ne remplacent pas un avis fiscal professionnel."},
    "company_purchase": {"en": "Company purchase", "fr": "Achat d'entreprise"},
    "private_purchase": {"en": "Private purchase", "fr": "Achat privé"},
    "delivery_postcode": {"en": "Delivery postal code", "fr": "Code postal de livraison"},
    "destination_postal_code": {"en": "Destination city / postal code", "fr": "Ville / code postal de destination"},
    "country": {"en": "Country", "fr": "Pays"},
    "preferred_contact_method": {"en": "Preferred contact method", "fr": "Méthode de contact préférée"},
    "email_contact": {"en": "Email", "fr": "Courriel"},
    "telephone_contact": {"en": "Telephone", "fr": "Téléphone"},
    "written_support_languages": {"en": "Telephone support is currently available in English. Written enquiries can be handled in supported website languages using our translated enquiry system.", "fr": "Le soutien téléphonique est actuellement disponible en anglais. Les demandes écrites peuvent être traitées dans les langues du site prises en charge via notre système de demande traduit."},
    "team": {"en": "Team", "fr": "Équipe"},
    "recent_deliveries": {"en": "Recent Deliveries", "fr": "Livraisons récentes"},
    "purchase_process": {"en": "Purchase Process", "fr": "Processus d'achat"},
    "company_gallery": {"en": "Company Gallery", "fr": "Galerie de l'entreprise"},
    "machine_location_not_public": {"en": "Vehicle location available on request", "fr": "Emplacement du véhicule disponible sur demande"},
    "last_checked_inventory": {"en": "Last checked inventory", "fr": "Dernier inventaire vérifié"},
    "trust_gallery": {"en": "Real Company & Warehouse Photos", "fr": "Photos réelles de l'entreprise et de l'entrepôt"},
    "trust_gallery_intro": {"en": "Real photographs from {COMPANY_NAME} premises, storage, loading, inspection and delivery operations.", "fr": "Photographies réelles des locaux, de l'entreposage, du chargement, de l'inspection et des opérations de livraison de {COMPANY_NAME}."},
    "buyer_type": {"en": "Buyer type", "fr": "Type d'acheteur"},
    "listing_whatsapp_message": {"en": "Hello, I am interested in {LISTING_TITLE}, reference {CAS_REFERENCE}. My delivery postal code is {POSTCODE}. Please confirm the estimated delivered price.", "fr": "Bonjour, je suis intéressé par {LISTING_TITLE}, référence {CAS_REFERENCE}. Mon code postal de livraison est {POSTCODE}. Veuillez confirmer le prix livré estimé."},
    "listing_email_subject": {"en": "Enquiry: {LISTING_TITLE} — {CAS_REFERENCE}", "fr": "Demande : {LISTING_TITLE} — {CAS_REFERENCE}"},
    "listing_email_body": {"en": "Hello, I am interested in {LISTING_TITLE}, reference {CAS_REFERENCE}. Listing: {PAGE_URL}. Location: {LOCATION}. Price: {PRICE}. Delivery postal code: {POSTCODE}.", "fr": "Bonjour, je suis intéressé par {LISTING_TITLE}, référence {CAS_REFERENCE}. Annonce : {PAGE_URL}. Emplacement : {LOCATION}. Prix : {PRICE}. Code postal de livraison : {POSTCODE}."},
})

app = Flask(__name__, instance_path=str(INSTANCE_DIR), instance_relative_config=True)
app.config.update(
    SECRET_KEY=os.environ.get("SECRET_KEY", "phase-2-local-change-me"),
    DATABASE=str(DATABASE),
    UPLOAD_FOLDER=str(UPLOADS_DIR),
)


def ensure_directories():
    for folder in (
        INSTANCE_DIR,
        UPLOADS_DIR,
        SITE_UPLOADS_DIR,
        LISTING_UPLOADS_DIR,
        AI_EDIT_UPLOADS_DIR,
        GENERATED_VISUALS_UPLOADS_DIR,
    ):
        folder.mkdir(parents=True, exist_ok=True)


def now_utc():
    return datetime.utcnow().isoformat(timespec="seconds")


def get_db():
    if "db" not in g:
        g.db = sqlite3.connect(app.config["DATABASE"], timeout=10)
        g.db.row_factory = sqlite3.Row
        # WAL: lets admin reads (page loads) coexist with campaign-engine writes
        # instead of "database is locked" stutters. busy_timeout makes a writer wait
        # up to 5s for a lock before raising, instead of failing immediately.
        g.db.execute("PRAGMA journal_mode=WAL")
        g.db.execute("PRAGMA busy_timeout=5000")
    return g.db


@app.teardown_appcontext
def close_db(_error=None):
    db = g.pop("db", None)
    if db is not None:
        db.close()


INTERFACE_TRANSLATION_DEFAULTS.update({
    "status_awaiting_confirmation": {"en": "Awaiting confirmation", "fr": "En attente de confirmation"},
    "status_unavailable": {"en": "Unavailable", "fr": "Indisponible"},
    "status_archived": {"en": "Archived", "fr": "Archivé"},
})

INTERFACE_TRANSLATION_DEFAULTS.update({
    "price_plus_tax": {"en": "+ HST & licensing", "fr": "+ TPS/TVH et immatriculation"},
    "pricing_disclaimer": {"en": "All prices in CAD. HST and licensing not included.", "fr": "Tous les prix sont en CAD. TPS/TVH et immatriculation non incluses."},
})


def table_columns(table_name):
    rows = get_db().execute(f"PRAGMA table_info({table_name})").fetchall()
    return {row["name"] for row in rows}


def table_exists(table_name):
    row = get_db().execute(
        "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", (table_name,)
    ).fetchone()
    return row is not None


def add_column_if_missing(table_name, column_name, definition):
    if column_name not in table_columns(table_name):
        get_db().execute(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {definition}")


def migrate_phase1_listings_if_needed():
    if table_exists("listings") and "stock_number" not in table_columns("listings"):
        backup_name = f"phase1_listings_backup_{datetime.utcnow().strftime('%Y%m%d%H%M%S')}"
        get_db().execute(f"ALTER TABLE listings RENAME TO {backup_name}")


TURBO_BODY_TYPES = [
    (2, "Convertible"), (3, "Coupe"), (1, "Hatchback"), (12, "Minivan"),
    (15, "Pick-up Truck"), (6, "Sedan"), (14, "SUV"), (5, "Wagon"), (7, "Others"),
]


def _seed_turbo_taxonomy(db):
    """Idempotently seed turbo_makes (from makes.json, which is backslash-escaped)
    and turbo_body_types (from the fixed map)."""
    try:
        makes_path = os.path.join(BASE_DIR, "makes.json")
        if os.path.exists(makes_path):
            raw = open(makes_path, encoding="utf-8").read().strip()
            # makes.json is stored with backslash-escaped quotes (PowerShell-style).
            unescaped = raw.replace('\\"', '"').strip()
            if not unescaped.endswith("]}"):
                unescaped = unescaped + "]}"  # tolerate a truncated/paste-dropped closing
            data = json.loads(unescaped)
            makes = data.get("makes") if isinstance(data, dict) else data
            if isinstance(makes, list):
                for m in makes:
                    if isinstance(m, dict) and m.get("id") is not None:
                        db.execute("INSERT OR IGNORE INTO turbo_makes (id, name) VALUES (?, ?)",
                                   (int(m["id"]), str(m.get("name") or "").strip()))
    except Exception as e:
        logging.getLogger(__name__).warning(f"turbo_makes seed failed: {e}")
    try:
        for bid, label in TURBO_BODY_TYPES:
            db.execute("INSERT OR IGNORE INTO turbo_body_types (id, label) VALUES (?, ?)", (bid, label))
    except Exception as e:
        logging.getLogger(__name__).warning(f"turbo_body_types seed failed: {e}")
    db.commit()


TURBO_CITIES = [
    # BC
    ("BC", "Vancouver", "V6B 1A1", 49.2827, -123.1207), ("BC", "Surrey", "V3R 0Z8", 49.1913, -122.8490),
    ("BC", "Burnaby", "V5G 1H1", 49.2488, -122.9805), ("BC", "Richmond", "V6Y 1V9", 49.1666, -123.1336),
    ("BC", "Victoria", "V8V 1Z4", 48.4284, -123.3656), ("BC", "Kelowna", "V1Y 1K2", 49.8880, -119.4960),
    ("BC", "Abbotsford", "V2S 1K1", 49.0504, -122.3045), ("BC", "New Westminster", "V3M 1A1", 49.2057, -122.9110),
    # ON intentionally excluded — business does not post in Ontario.
    # AB
    ("AB", "Calgary", "T2P 1J9", 51.0447, -114.0719), ("AB", "Edmonton", "T5J 0K7", 53.5461, -113.4938),
    ("AB", "Red Deer", "T4N 0H7", 52.2681, -113.8112), ("AB", "Lethbridge", "T1J 0K9", 49.6942, -112.8328),
    ("AB", "St. Albert", "T8N 1K1", 53.6305, -113.6256), ("AB", "Medicine Hat", "T1A 0K9", 50.0406, -110.6776),
    ("AB", "Grande Prairie", "T8V 0K9", 55.1700, -118.7970), ("AB", "Airdrie", "T4B 0K9", 51.2916, -114.0140),
    # QC
    ("QC", "Montreal", "H2X 1Y4", 45.5017, -73.5673), ("QC", "Quebec City", "G1R 1T2", 46.8139, -71.2080),
    ("QC", "Laval", "H7N 1K1", 45.6066, -73.7121), ("QC", "Gatineau", "J8X 1K1", 45.4762, -75.7016),
    ("QC", "Longueuil", "J4K 1K1", 45.5372, -73.5198), ("QC", "Sherbrooke", "J1H 1K1", 45.4012, -71.8929),
    ("QC", "Saguenay", "G7H 1K1", 48.4286, -71.0657), ("QC", "Trois-Rivieres", "G8Y 1K1", 46.3432, -72.5425),
    # MB
    ("MB", "Winnipeg", "R3C 0V8", 49.8951, -97.1384), ("MB", "Brandon", "R7A 0K9", 49.8376, -99.9497),
    ("MB", "Steinbach", "R5G 0K9", 49.5259, -96.6847), ("MB", "Thompson", "R8N 0K9", 55.7435, -97.8575),
    ("MB", "Portage la Prairie", "R1N 0K9", 49.9726, -98.2925), ("MB", "Selkirk", "R1A 0K9", 50.1431, -96.8838),
    ("MB", "Winkler", "R6W 0K9", 49.1800, -97.9400), ("MB", "Morden", "R6M 0K9", 49.1878, -98.0985),
    # SK
    ("SK", "Saskatoon", "S7K 0J5", 52.1332, -106.6700), ("SK", "Regina", "S4P 0J5", 50.4452, -104.6189),
    ("SK", "Prince Albert", "S6V 0K9", 53.2030, -105.7530), ("SK", "Moose Jaw", "S6H 0K9", 50.4000, -105.5336),
    ("SK", "Yorkton", "S3N 0K9", 51.2100, -102.4700), ("SK", "Swift Current", "S9H 0K9", 50.2880, -107.7990),
    ("SK", "North Battleford", "S9A 0K9", 52.7800, -108.2900), ("SK", "Estevan", "S4A 0K9", 49.1340, -102.9780),
    # NS
    ("NS", "Halifax", "B3J 1S9", 44.6488, -63.5752), ("NS", "Sydney", "B1P 1K1", 46.1351, -60.1950),
    ("NS", "Dartmouth", "B2Y 1K1", 44.6713, -63.5772), ("NS", "Truro", "B2N 0K9", 45.3666, -63.2740),
    ("NS", "New Glasgow", "B2H 0K9", 45.5850, -62.6430), ("NS", "Bedford", "B4A 0K9", 44.7210, -63.6600),
    ("NS", "Lunenburg", "B0J 2C0", 44.3774, -64.3144), ("NS", "Yarmouth", "B5A 0K9", 43.8370, -66.1170),
    # NB
    ("NB", "Fredericton", "E3B 1K1", 45.9636, -66.6431), ("NB", "Moncton", "E1C 1K1", 46.0878, -64.7782),
    ("NB", "Saint John", "E2L 2K1", 45.2733, -66.0633), ("NB", "Miramichi", "E1V 0K9", 46.9900, -65.5000),
    ("NB", "Edmundston", "E3V 0K9", 47.3700, -68.3200), ("NB", "Bathurst", "E2A 0K9", 47.6200, -65.6500),
    ("NB", "Campbellton", "E3N 0K9", 48.0000, -66.6700), ("NB", "Dieppe", "E1A 0K9", 46.0700, -64.7600),
    # NL
    ("NL", "St. John's", "A1C 1K1", 47.5615, -52.7126), ("NL", "Corner Brook", "A2H 0K9", 48.9500, -57.9500),
    ("NL", "Gander", "A1V 0K9", 48.9600, -54.6100), ("NL", "Grand Falls-Windsor", "A2A 0K9", 48.9400, -55.6600),
    ("NL", "Conception Bay South", "A1X 0K9", 47.4900, -52.9800), ("NL", "Mount Pearl", "A1N 0K9", 47.5200, -52.7900),
    ("NL", "Paradise", "A1L 0K9", 47.5300, -52.8500), ("NL", "Marystown", "A0E 0K9", 46.9700, -55.0800),
    # PE
    ("PE", "Charlottetown", "C1A 1K1", 46.2382, -63.1311), ("PE", "Summerside", "C1N 0K9", 46.3900, -63.7900),
    ("PE", "Montague", "C0A 1R0", 46.1700, -62.6500), ("PE", "Kensington", "C0B 1M0", 46.4400, -63.6400),
    ("PE", "Souris", "C0A 2B0", 46.3500, -62.2500), ("PE", "Georgetown", "C0A 1L0", 46.2300, -62.5000),
    ("PE", "Alberton", "C0B 1B0", 46.6700, -64.0700), ("PE", "Tignish", "C0B 2B0", 46.9500, -64.0300),
    # NT
    ("NT", "Yellowknife", "X1A 2N2", 62.4540, -114.3718), ("NT", "Hay River", "X0E 0R9", 60.8200, -115.7400),
    ("NT", "Inuvik", "X0E 0T0", 68.3600, -133.7300), ("NT", "Fort Smith", "X0E 0P0", 60.0000, -111.8900),
    ("NT", "Norman Wells", "X0E 0V0", 65.2800, -126.5000), ("NT", "Behchoko", "X0E 0Y0", 62.2600, -115.9100),
    ("NT", "Fort Simpson", "X0E 0N0", 61.7600, -121.2300), ("NT", "Fort Providence", "X0E 0T0", 61.3500, -117.6500),
    # NU
    ("NU", "Iqaluit", "X0A 0H0", 63.7467, -68.5170), ("NU", "Rankin Inlet", "X0C 0G0", 62.8100, -92.0800),
    ("NU", "Cambridge Bay", "X0B 0C0", 69.1140, -105.0590), ("NU", "Kugluktuk", "X0B 0E0", 67.8200, -115.1000),
    ("NU", "Cape Dorset", "X0A 0C0", 64.2300, -76.5200), ("NU", "Pangnirtung", "X0A 0R0", 66.1400, -65.7100),
    ("NU", "Pond Inlet", "X0A 0S0", 72.7000, -78.0000), ("NU", "Baker Lake", "X0C 0A0", 64.3000, -96.0000),
    # YT
    ("YT", "Whitehorse", "Y1A 2C6", 60.7212, -135.0568), ("YT", "Dawson City", "Y0B 1G0", 64.0600, -139.4300),
    ("YT", "Watson Lake", "Y0A 1C0", 60.0700, -128.7000), ("YT", "Haines Junction", "Y0B 1L0", 60.7500, -137.5000),
    ("YT", "Carmacks", "Y0B 1C0", 62.1000, -134.1600), ("YT", "Faro", "Y0B 1H0", 62.2300, -133.3400),
    ("YT", "Teslin", "Y0A 1B0", 60.1700, -132.7500), ("YT", "Mayo", "Y0B 1M0", 63.6000, -135.8700),
]


def _seed_turbo_cities(db):
    """Idempotently seed turbo_cities with top 8 Canadian cities per province/territory."""
    try:
        count = db.execute("SELECT COUNT(*) FROM turbo_cities").fetchone()[0]
        if count > 0:
            return  # already seeded
        for prov, city, postal, lat, lng in TURBO_CITIES:
            db.execute(
                "INSERT INTO turbo_cities (province, city, postal_code, latitude, longitude) VALUES (?,?,?,?,?)",
                (prov, city, postal, lat, lng),
            )
        db.commit()
    except Exception as e:
        logging.getLogger(__name__).warning(f"turbo_cities seed failed: {e}")


def migrate_phase25_columns():
    if table_exists("site_settings"):
        add_column_if_missing("site_settings", "enable_call_button", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "enable_whatsapp_button", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "enable_email_button", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "enable_sticky_contact_form", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "base_currency", "TEXT DEFAULT 'CAD'")
        add_column_if_missing("site_settings", "enable_currency_conversion", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "openai_api_key_encrypted", "TEXT")
        add_column_if_missing("site_settings", "openai_logo_file_id", "TEXT")
        add_column_if_missing("site_settings", "openai_image_model", "TEXT DEFAULT 'gpt-5'")
        add_column_if_missing("site_settings", "openai_image_edit_prompt", "TEXT")
        add_column_if_missing("site_settings", "openai_angle_generation_prompt", "TEXT")
        add_column_if_missing("site_settings", "fireworks_api_key_encrypted", "TEXT")
        add_column_if_missing("site_settings", "fireworks_model", f"TEXT DEFAULT '{DEFAULT_FIREWORKS_MODEL}'")
        add_column_if_missing("site_settings", "fireworks_custom_model", "TEXT")
        add_column_if_missing("site_settings", "fireworks_image_review_model", f"TEXT DEFAULT '{DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL}'")
        add_column_if_missing("site_settings", "openai_model", "TEXT DEFAULT 'gpt-5.2'")
        add_column_if_missing("site_settings", "ai_provider", "TEXT DEFAULT 'fireworks'")
        add_column_if_missing("site_settings", "ai_generation_enabled", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "public_stock_prefix", "TEXT DEFAULT 'CA'")
        add_column_if_missing("site_settings", "show_internal_stock_publicly", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "chatbot_enabled", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "chatbot_provider", "TEXT DEFAULT 'fireworks'")
        add_column_if_missing("site_settings", "chatbot_model", f"TEXT DEFAULT '{DEFAULT_FIREWORKS_MODEL}'")
        add_column_if_missing("site_settings", "chatbot_custom_model", "TEXT")
        add_column_if_missing("site_settings", "chatbot_greeting", "TEXT")
        add_column_if_missing("site_settings", "chatbot_fallback_message", "TEXT")
        add_column_if_missing("site_settings", "chatbot_lead_capture_enabled", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "chatbot_show_desktop", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "chatbot_show_mobile", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "chatbot_system_prompt", "TEXT")
        try:
            row = get_db().execute("SELECT chatbot_system_prompt, company_name, free_delivery_promo_enabled FROM site_settings WHERE id=1").fetchone()
            if row and row["chatbot_system_prompt"] is None:
                seeded = default_chatbot_system_prompt(row["company_name"] or "Canada Auto Sales", free_delivery_promo_enabled=bool(row["free_delivery_promo_enabled"]))
                get_db().execute("UPDATE site_settings SET chatbot_system_prompt = ? WHERE id = 1", (seeded,))
                get_db().commit()
        except Exception:
            pass
        add_column_if_missing("site_settings", "logo_max_height", "INTEGER DEFAULT 44")
        add_column_if_missing("site_settings", "logo_max_width", "INTEGER DEFAULT 180")
        add_column_if_missing("site_settings", "smtp_enabled", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "smtp_host", "TEXT")
        add_column_if_missing("site_settings", "smtp_encryption", "TEXT DEFAULT 'tls'")
        add_column_if_missing("site_settings", "smtp_port", "INTEGER DEFAULT 587")
        add_column_if_missing("site_settings", "smtp_authentication", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "smtp_username", "TEXT")
        add_column_if_missing("site_settings", "smtp_password_encrypted", "TEXT")
        add_column_if_missing("site_settings", "smtp_from_email", "TEXT")
        add_column_if_missing("site_settings", "smtp_from_name", "TEXT")
        add_column_if_missing("site_settings", "smtp_recipient_email", "TEXT")
        add_column_if_missing("site_settings", "translation_model", f"TEXT DEFAULT '{DEFAULT_TRANSLATION_MODEL}'")
        add_column_if_missing("site_settings", "content_model", f"TEXT DEFAULT '{DEFAULT_CONTENT_MODEL}'")
        add_column_if_missing("site_settings", "auto_language_detection", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "browser_language_fallback", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "remember_language_choice", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "analytics_enabled", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "analytics_reliable_since", "TEXT")
        add_column_if_missing("site_settings", "analytics_show_full_ip", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "analytics_include_zero_seconds", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "chat_sessions_purged_v2", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "free_delivery_promo_enabled", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "free_delivery_promo_text", "TEXT DEFAULT 'Free Canada-wide delivery this {month}'")
        add_column_if_missing("site_settings", "free_delivery_promo_placement", "TEXT DEFAULT 'both'")
        add_column_if_missing("site_settings", "free_delivery_promo_duration", "INTEGER DEFAULT 6")
        add_column_if_missing("site_settings", "free_delivery_promo_theme", "TEXT DEFAULT 'amber'")
        add_column_if_missing("site_settings", "free_delivery_promo_anim_entrance", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "free_delivery_promo_anim_shimmer", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "free_delivery_promo_anim_pulse", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "free_delivery_promo_anim_slide_text", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "free_delivery_promo_anim_countdown", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "free_delivery_promo_end_date", "TEXT DEFAULT ''")
        add_column_if_missing("site_settings", "popular_makes_count", "INTEGER DEFAULT 8")
        add_column_if_missing("site_settings", "featured_carousel_count", "INTEGER DEFAULT 8")
        add_column_if_missing("site_settings", "carfax_button_enabled", "INTEGER DEFAULT 1")
        # -- Mechanical inspection reports (AI-generated, shop-approved) --
        add_column_if_missing("site_settings", "inspection_button_enabled", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "shop_name", "TEXT")
        add_column_if_missing("site_settings", "shop_phone", "TEXT")
        add_column_if_missing("site_settings", "shop_email", "TEXT")
        add_column_if_missing("site_settings", "shop_address", "TEXT")
        add_column_if_missing("site_settings", "shop_license", "TEXT")
        add_column_if_missing("site_settings", "shop_logo_path", "TEXT")
        add_column_if_missing("site_settings", "shop_portal_token", "TEXT")
        try:
            row = get_db().execute("SELECT shop_portal_token FROM site_settings WHERE id=1").fetchone()
            if row and not (row["shop_portal_token"] or "").strip():
                get_db().execute("UPDATE site_settings SET shop_portal_token = ? WHERE id = 1",
                                 (uuid.uuid4().hex,))
                get_db().commit()
        except Exception:
            pass
        add_column_if_missing("site_settings", "savings_banner_enabled", "INTEGER DEFAULT 1")
        add_column_if_missing("site_settings", "phone2", "TEXT")
        add_column_if_missing("site_settings", "address_street", "TEXT")
        add_column_if_missing("site_settings", "address_city", "TEXT")
        add_column_if_missing("site_settings", "address_province", "TEXT")
        add_column_if_missing("site_settings", "address_postal", "TEXT")
        try:
            row = get_db().execute("SELECT address, address_street FROM site_settings WHERE id=1").fetchone()
            if row and (row["address"] or "").strip() and row["address_street"] is None:
                get_db().execute("UPDATE site_settings SET address_street = ? WHERE id = 1", (row["address"],))
                get_db().commit()
        except Exception:
            pass
        add_column_if_missing("listings", "carfax_accident_notes", "TEXT")
        add_column_if_missing("redirects", "notes", "TEXT")
        add_column_if_missing("campaign_links", "duplicate", "INTEGER DEFAULT 0")
        add_column_if_missing("campaign_links", "reply_subject", "TEXT")
        add_column_if_missing("campaign_links", "scheduled_at", "TEXT")
        add_column_if_missing("campaign_links", "send_error", "TEXT")
        add_column_if_missing("campaign_links", "gen_status", "TEXT")
        add_column_if_missing("campaign_links", "gen_job", "TEXT")
        add_column_if_missing("site_settings", "reply_gen_active_job", "TEXT")
        add_column_if_missing("site_settings", "reply_gen_started_at", "TEXT")
        add_column_if_missing("site_settings", "reply_gen_progress_at", "TEXT")
        add_column_if_missing("site_settings", "zai_api_key_encrypted", "TEXT")
        add_column_if_missing("site_settings", "zai_model", "TEXT")
        try:
            row = get_db().execute("SELECT zai_model FROM site_settings WHERE id=1").fetchone()
            if row and not (row["zai_model"] or "").strip():
                get_db().execute("UPDATE site_settings SET zai_model = 'glm-5.2' WHERE id = 1")
                get_db().commit()
        except Exception:
            pass
        add_column_if_missing("site_settings", "ai_provider_chatbot", "TEXT")
        add_column_if_missing("site_settings", "ai_provider_lead_reply", "TEXT")
        add_column_if_missing("site_settings", "ai_provider_descriptions", "TEXT")
        add_column_if_missing("site_settings", "ai_provider_pages", "TEXT")
        add_column_if_missing("site_settings", "ai_provider_inspection", "TEXT")
        _seed_turbo_taxonomy(get_db())
        add_column_if_missing("turbo_session_profiles", "powershell_encrypted", "TEXT")
        add_column_if_missing("turbo_session_profiles", "proxy_encrypted", "TEXT")
        add_column_if_missing("turbo_session_profiles", "email", "TEXT")
        add_column_if_missing("turbo_session_profiles", "flagged", "INTEGER DEFAULT 0")
        add_column_if_missing("turbo_session_profiles", "flagged_reason", "TEXT")
        add_column_if_missing("turbo_session_profiles", "refresh_cookies_encrypted", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "delete_error", "TEXT")
        # Campaign engine hardening (BUG #4/#5 + improvements).
        add_column_if_missing("turbo_session_profiles", "paused_until", "TEXT")
        add_column_if_missing("turbo_session_profiles", "paused_reason", "TEXT")
        add_column_if_missing("turbo_campaigns", "last_error", "TEXT")
        add_column_if_missing("turbo_campaigns", "last_error_at", "TEXT")
        add_column_if_missing("turbo_campaigns", "last_heartbeat", "TEXT")
        add_column_if_missing("turbo_campaigns", "always_no_vin", "INTEGER DEFAULT 0")
        # Unified post tracking: turbo_campaign_posts is the single source of truth
        # for every post (campaign / manual single-send / external listing found via
        # "View current listings"), plus its live-check state.
        add_column_if_missing("turbo_campaign_posts", "source", "TEXT DEFAULT 'campaign'")
        add_column_if_missing("turbo_campaign_posts", "price_used", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "mileage_used", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "vin_mode", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "http_status", "INTEGER")
        add_column_if_missing("turbo_campaign_posts", "at_request_id", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "live_status", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "live_checked_at", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "live_http", "INTEGER")
        add_column_if_missing("turbo_campaign_posts", "live_check_method", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "live_error", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "external_title", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "external_price", "TEXT")
        add_column_if_missing("turbo_campaign_posts", "external_json", "TEXT")
        get_db().execute(
            "CREATE TABLE IF NOT EXISTS turbo_campaign_log ("
            "id INTEGER PRIMARY KEY AUTOINCREMENT, campaign_id INTEGER, batch_number INTEGER, "
            "profile_id INTEGER, listing_id INTEGER, phase TEXT, level TEXT, message TEXT, logged_at TEXT)"
        )
        # Cached AI listing descriptions (T1.3): re-posting the same car across batches
        # reuses the description instead of a fresh Fireworks call (~20-30s) each time.
        # Keyed by (listing_id, input_hash) so a changed year/make/model/trim/odo regenerates.
        get_db().execute(
            "CREATE TABLE IF NOT EXISTS turbo_descriptions ("
            "listing_id INTEGER, input_hash TEXT, description TEXT, generated_at TEXT, "
            "PRIMARY KEY (listing_id))"
        )
        # VIN cooldown / escalation: a VIN whose listing AutoTrader marks INACTIVE is
        # parked (posted WITHOUT the VIN) for VIN_COOLDOWN_DAYS, then retried with the
        # VIN. A second INACTIVE → permanent exclude (always posted without VIN).
        get_db().execute(
            "CREATE TABLE IF NOT EXISTS turbo_vin_cooldown ("
            "vin TEXT PRIMARY KEY, inactive_count INTEGER DEFAULT 0, "
            "last_inactive_at TEXT, excluded INTEGER DEFAULT 0, reason TEXT)"
        )
        # Defensive uniqueness so a publish race can never double-insert the same
        # (campaign, batch, vehicle, profile) SUCCESSFUL post (BUG #7). Partial
        # index: failed rows are excluded so a vehicle can be retried in-batch.
        get_db().execute(
            "CREATE UNIQUE INDEX IF NOT EXISTS idx_turbo_posts_unique "
            "ON turbo_campaign_posts(campaign_id, batch_number, listing_id, profile_id) "
            "WHERE post_status='success'"
        )
        _seed_turbo_cities(get_db())
        add_column_if_missing("site_settings", "imap_host", "TEXT")
        add_column_if_missing("site_settings", "imap_port", "INTEGER")
        add_column_if_missing("site_settings", "imap_encryption", "TEXT DEFAULT 'ssl'")
        add_column_if_missing("site_settings", "imap_username", "TEXT")
        add_column_if_missing("site_settings", "imap_password_encrypted", "TEXT")
        add_column_if_missing("site_settings", "inbox_master_prompt", "TEXT")
        add_column_if_missing("site_settings", "inbox_autonomous_enabled", "INTEGER DEFAULT 0")
        add_column_if_missing("site_settings", "inbox_autonomous_interval", "INTEGER DEFAULT 30")
        add_column_if_missing("site_settings", "inbox_auto_send", "INTEGER DEFAULT 0")
        add_column_if_missing("inbox_messages", "archived", "INTEGER DEFAULT 0")
        add_column_if_missing("inbox_messages", "received_at_iso", "TEXT")
        # Reply Composer (inbox): one saved draft per lead — input thread + comments +
        # the AI-generated reply, so the admin can come back to a lead's draft.
        get_db().execute(
            "CREATE TABLE IF NOT EXISTS inbox_composer_drafts ("
            "lead_id INTEGER PRIMARY KEY, input_text TEXT, comments TEXT, "
            "output_text TEXT, updated_at TEXT)"
        )
        add_column_if_missing("campaign_links", "sms_token", "TEXT")
        add_column_if_missing("campaign_links", "text_message", "TEXT")
        add_column_if_missing("campaign_links", "texted_at", "TEXT")
        add_column_if_missing("campaign_links", "sms_visited", "INTEGER DEFAULT 0")
        add_column_if_missing("campaign_links", "sms_visit_count", "INTEGER DEFAULT 0")
        add_column_if_missing("campaign_links", "sms_first_visit_at", "TEXT")
        add_column_if_missing("campaign_links", "sms_last_visit_at", "TEXT")
        # Title → stock# reference map: when a pasted lead has no CAS/VIN, its (cleaned)
        # title is looked up here to auto-assign the car. Admin maintains it on the
        # lead-campaigns page; one row per "Title $price|CAS-####" line.
        get_db().execute(
            "CREATE TABLE IF NOT EXISTS lead_title_map ("
            "id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT, title_key TEXT, "
            "stock_number TEXT, created_at TEXT, UNIQUE(title_key, stock_number))"
        )
        try:
            get_db().execute(
                "UPDATE campaign_links SET sms_token = lower(hex(randomblob(16))) WHERE sms_token IS NULL"
            )
            get_db().commit()
        except Exception:
            pass
        add_column_if_missing("site_settings", "lead_reply_master_prompt", "TEXT")
        try:
            row = get_db().execute("SELECT lead_reply_master_prompt FROM site_settings WHERE id=1").fetchone()
            if row and not (row["lead_reply_master_prompt"] or "").strip():
                get_db().execute(
                    "UPDATE site_settings SET lead_reply_master_prompt = ? WHERE id = 1",
                    (
                        "You are a Canadian used car dealer writing a reply to a lead who enquired about a vehicle. "
                        "Use the lead's name and reference the specific vehicle (year, make, model, price). Answer their "
                        "question, invite a test drive or a quick call, and mention Canada-wide delivery and financing "
                        "options if relevant. Keep it concise (120-180 words), warm, professional, Canadian English, no "
                        "pressure tactics. Sign off as the dealership.",
                    ),
                )
                get_db().commit()
        except Exception:
            pass
        add_column_if_missing("site_settings", "show_dealer_license_publicly", "INTEGER DEFAULT 0")
        try:
            row = get_db().execute("SELECT companies_house_url FROM site_settings WHERE id=1").fetchone()
            if row and ("omvic.ca" not in (row["companies_house_url"] or "")):
                get_db().execute("UPDATE site_settings SET companies_house_url = ? WHERE id = 1",
                                 ("https://www.omvic.ca/dealer-search/dealership-profile/?entry-id=5323a642-0d1b-f011-998a-000d3ae85379",))
                get_db().commit()
        except Exception:
            pass
        # Regenerate slugs that contain the VIN or have a doubled year (VIN must never appear in public URLs).
        try:
            all_rows = get_db().execute("SELECT * FROM listings WHERE slug IS NOT NULL AND slug != ''").fetchall()
            changed = 0
            for row in all_rows:
                new_slug = public_slug_for_listing(row)
                if new_slug and new_slug != row["slug"]:
                    get_db().execute("UPDATE listings SET slug = ?, updated_at = ? WHERE id = ?",
                                     (new_slug, now_utc(), row["id"]))
                    changed += 1
            if changed:
                get_db().commit()
        except Exception:
            pass
        # Recalculate savings = msrp - price (comma-stripped) for listings where msrp > price.
        # The imported savings values were stale/wrong; recalculate from actual price + msrp.
        try:
            rows = get_db().execute("SELECT id, price, msrp, savings FROM listings WHERE msrp IS NOT NULL AND msrp > 0").fetchall()
            recalculated = 0
            for row in rows:
                price_num = float(str(row["price"]).replace(",", "")) if row["price"] else 0
                msrp_num = float(row["msrp"]) if row["msrp"] else 0
                if msrp_num > price_num:
                    calc_savings = msrp_num - price_num
                    existing = float(row["savings"]) if row["savings"] is not None else 0
                    if abs(existing - calc_savings) > 0.01:
                        get_db().execute("UPDATE listings SET savings = ? WHERE id = ?", (calc_savings, row["id"]))
                        recalculated += 1
                else:
                    if row["savings"] is not None and float(row["savings"]) > 0:
                        get_db().execute("UPDATE listings SET savings = 0 WHERE id = ?", (row["id"],))
                        recalculated += 1
            if recalculated:
                get_db().commit()
        except Exception:
            pass
        # Normalize body_style values to match the filter labels (SUV, Truck, Van, etc.)
        # The imported values were source-specific (Sport Utility, Short Bed, Mini-van Passenger, etc.)
        body_style_map = {
            "Sport Utility": "SUV",
            "Short Bed": "Truck",
            "Double Cab": "Truck",
            "Standard Bed": "Truck",
            "Crew Cab": "Truck",
            "Long Bed": "Truck",
            "Extended Cab": "Truck",
            "Mini-van, Passenger": "Van",
            "Mini-van": "Van",
            "Passenger Van": "Van",
        }
        try:
            changed = False
            for old_val, new_val in body_style_map.items():
                cur = get_db().execute("UPDATE listings SET body_style = ? WHERE body_style = ?", (new_val, old_val))
                if cur.rowcount > 0:
                    changed = True
            if changed:
                get_db().commit()
        except Exception:
            pass
        if table_exists("analytics_sessions"):
            add_column_if_missing("analytics_sessions", "ip_masked", "TEXT")
            add_column_if_missing("analytics_sessions", "ip_full", "TEXT")
            add_column_if_missing("analytics_sessions", "ip_source", "TEXT")
        if table_exists("analytics_events"):
            add_column_if_missing("analytics_events", "ip_masked", "TEXT")
            add_column_if_missing("analytics_events", "ip_full", "TEXT")
            add_column_if_missing("analytics_events", "ip_source", "TEXT")
        if table_exists("security_events"):
            add_column_if_missing("security_events", "source_ip_full", "TEXT")
        get_db().execute("UPDATE site_settings SET logo_max_height = 44 WHERE logo_max_height IS NULL OR logo_max_height = '' OR logo_max_height <= 0")
        get_db().execute("UPDATE site_settings SET logo_max_width = 180 WHERE logo_max_width IS NULL OR logo_max_width = '' OR logo_max_width <= 0")
        get_db().execute("UPDATE site_settings SET base_currency = 'CAD' WHERE base_currency IS NULL OR base_currency = '' OR base_currency NOT IN ('CAD','USD')")
        get_db().execute("UPDATE site_settings SET openai_image_model = 'gpt-image-1' WHERE openai_image_model IS NULL OR openai_image_model = '' OR openai_image_model NOT IN ('gpt-image-1','gpt-image-1-mini','dall-e-3','dall-e-2')")
        get_db().execute("UPDATE site_settings SET fireworks_model = ? WHERE fireworks_model IS NULL OR fireworks_model = ''", (DEFAULT_FIREWORKS_MODEL,))
        get_db().execute(
            "UPDATE site_settings SET fireworks_image_review_model = ? WHERE fireworks_image_review_model IS NULL OR fireworks_image_review_model = ''",
            (DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL,),
        )
        get_db().execute("UPDATE site_settings SET openai_model = 'gpt-5.2' WHERE openai_model IS NULL OR openai_model = ''")
        get_db().execute("UPDATE site_settings SET ai_provider = 'fireworks' WHERE ai_provider IS NULL OR ai_provider = ''")
        get_db().execute("UPDATE site_settings SET ai_generation_enabled = 0 WHERE ai_generation_enabled IS NULL")
        get_db().execute("UPDATE site_settings SET public_stock_prefix = 'CA' WHERE public_stock_prefix IS NULL OR public_stock_prefix = ''")
        get_db().execute("UPDATE site_settings SET show_internal_stock_publicly = 0 WHERE show_internal_stock_publicly IS NULL")
        get_db().execute("UPDATE site_settings SET chatbot_provider = 'fireworks' WHERE chatbot_provider IS NULL OR chatbot_provider = ''")
        get_db().execute("UPDATE site_settings SET chatbot_model = ? WHERE chatbot_model IS NULL OR chatbot_model = ''", (DEFAULT_FIREWORKS_MODEL,))
        get_db().execute("UPDATE site_settings SET chatbot_greeting = ? WHERE chatbot_greeting IS NULL OR chatbot_greeting = ''", (default_chatbot_greeting(),))
        get_db().execute("UPDATE site_settings SET chatbot_fallback_message = ? WHERE chatbot_fallback_message IS NULL OR chatbot_fallback_message = ''", (default_chatbot_fallback_message(),))
        get_db().execute("UPDATE site_settings SET chatbot_lead_capture_enabled = 1 WHERE chatbot_lead_capture_enabled IS NULL")
        get_db().execute("UPDATE site_settings SET chatbot_show_desktop = 1 WHERE chatbot_show_desktop IS NULL")
        get_db().execute("UPDATE site_settings SET chatbot_show_mobile = 1 WHERE chatbot_show_mobile IS NULL")
        get_db().execute("UPDATE site_settings SET smtp_encryption = 'tls' WHERE smtp_encryption IS NULL OR smtp_encryption = ''")
        get_db().execute("UPDATE site_settings SET smtp_port = 587 WHERE smtp_port IS NULL OR smtp_port = '' OR smtp_port <= 0")
        get_db().execute("UPDATE site_settings SET smtp_authentication = 1 WHERE smtp_authentication IS NULL")
        get_db().execute("UPDATE site_settings SET smtp_from_email = email WHERE (smtp_from_email IS NULL OR smtp_from_email = '') AND email IS NOT NULL AND email != ''")
        get_db().execute("UPDATE site_settings SET smtp_from_name = company_name WHERE (smtp_from_name IS NULL OR smtp_from_name = '') AND company_name IS NOT NULL AND company_name != ''")
        get_db().execute("UPDATE site_settings SET smtp_recipient_email = email WHERE (smtp_recipient_email IS NULL OR smtp_recipient_email = '') AND email IS NOT NULL AND email != ''")
        get_db().execute("UPDATE site_settings SET translation_model = ? WHERE translation_model IS NULL OR translation_model = ''", (DEFAULT_TRANSLATION_MODEL,))
        get_db().execute("UPDATE site_settings SET content_model = ? WHERE content_model IS NULL OR content_model = ''", (DEFAULT_CONTENT_MODEL,))
        get_db().execute("UPDATE site_settings SET auto_language_detection = 1 WHERE auto_language_detection IS NULL")
        get_db().execute("UPDATE site_settings SET browser_language_fallback = 1 WHERE browser_language_fallback IS NULL")
        get_db().execute("UPDATE site_settings SET remember_language_choice = 1 WHERE remember_language_choice IS NULL")
        get_db().execute("UPDATE site_settings SET analytics_enabled = 1 WHERE analytics_enabled IS NULL")
        get_db().execute("UPDATE site_settings SET analytics_show_full_ip = 0 WHERE analytics_show_full_ip IS NULL")
        get_db().execute("UPDATE site_settings SET analytics_include_zero_seconds = 1 WHERE analytics_include_zero_seconds IS NULL")
        get_db().execute("UPDATE site_settings SET analytics_reliable_since = ? WHERE analytics_reliable_since IS NULL OR analytics_reliable_since = ''", (now_utc(),))
        if table_exists("security_rules"):
            for rule_type, rule_value, reason in (
                ("approved_host", "canadaautosales.ca", "Primary production domain"),
                ("approved_host", "www.canadaautosales.ca", "WWW production domain"),
                ("blocked_domain", "paginademanuseamento.com", "Unauthorized proxy/clone"),
                ("blocked_ip", "216.252.233.168", "Unauthorized proxy/clone server"),
            ):
                get_db().execute(
                    "INSERT OR IGNORE INTO security_rules(rule_type,rule_value,reason,is_enabled,created_at,updated_at) VALUES(?,?,?,?,?,?)",
                    (rule_type, rule_value, reason, 1, now_utc(), now_utc()),
                )
        get_db().execute("UPDATE site_settings SET chatbot_model = ? WHERE chatbot_model IS NULL OR chatbot_model = '' OR chatbot_model IN ('accounts/fireworks/models/kimi-k2p5','accounts/fireworks/models/kimi-k2p6')", (DEFAULT_CHATBOT_MODEL,))
        get_db().execute(
            "UPDATE site_settings SET openai_image_edit_prompt = ? WHERE openai_image_edit_prompt IS NULL OR openai_image_edit_prompt = ''",
            (default_ai_image_edit_prompt(),),
        )
        get_db().execute(
            "UPDATE site_settings SET openai_angle_generation_prompt = ? WHERE openai_angle_generation_prompt IS NULL OR openai_angle_generation_prompt = ''",
            (default_ai_angle_generation_prompt(),),
        )
    add_column_if_missing("site_settings", "pages_master_prompt", "TEXT")
    add_column_if_missing("site_settings", "pages_generation_enabled", "INTEGER DEFAULT 0")
    add_column_if_missing("site_settings", "pages_generation_interval_hours", "INTEGER DEFAULT 24")
    add_column_if_missing("site_settings", "pages_generation_last_run", "TEXT")
    add_column_if_missing("site_pages", "custom_prompt", "TEXT")
    if table_exists("listings"):
        add_column_if_missing("listings", "hours", "TEXT")
        add_column_if_missing("listings", "source_metadata_json", "TEXT")
        add_column_if_missing("listings", "public_stock_number", "TEXT")
        add_column_if_missing("listings", "is_deleted", "INTEGER DEFAULT 0")
        add_column_if_missing("listings", "carfax_url", "TEXT")
        add_column_if_missing("listings", "carfax_enabled", "INTEGER DEFAULT 1")
        get_db().execute("UPDATE listings SET is_deleted = 0 WHERE is_deleted IS NULL")
        prefix_row = get_db().execute("SELECT public_stock_prefix FROM site_settings WHERE id = 1").fetchone()
        public_prefix = (prefix_row["public_stock_prefix"] if prefix_row else "CAS") or "CAS"
        get_db().execute(
            "UPDATE listings SET public_stock_number = ? || '-' || (1000 + id) WHERE (public_stock_number IS NULL OR public_stock_number = '')",
            (public_prefix.strip().upper(),),
        )
    if table_exists("leads"):
        add_column_if_missing("leads", "smtp_status", "TEXT")
        add_column_if_missing("leads", "smtp_message", "TEXT")
        add_column_if_missing("leads", "smtp_sent_at", "TEXT")
        add_column_if_missing("leads", "visitor_id", "TEXT")
        add_column_if_missing("leads", "analytics_session_token", "TEXT")
        add_column_if_missing("leads", "landing_page", "TEXT")
        add_column_if_missing("leads", "first_referrer", "TEXT")
        add_column_if_missing("leads", "first_visit_at", "TEXT")
        add_column_if_missing("leads", "views_before_lead", "INTEGER DEFAULT 0")
        add_column_if_missing("leads", "engaged_seconds_before_lead", "INTEGER DEFAULT 0")
        add_column_if_missing("leads", "attribution_country", "TEXT")
        add_column_if_missing("leads", "attribution_language", "TEXT")
        add_column_if_missing("leads", "attribution_device", "TEXT")
        add_column_if_missing("leads", "analytics_user_agent", "TEXT")
        add_column_if_missing("leads", "analytics_ip_hash", "TEXT")

    get_db().execute("""
        CREATE TABLE IF NOT EXISTS interface_translations (
            translation_key TEXT PRIMARY KEY,
            context TEXT,
            en TEXT,
            it TEXT, de TEXT, nl TEXT, no TEXT, da TEXT, sv TEXT, pl TEXT,
            updated_at TEXT
        )
    """)
    # Keep interface columns aligned with every configured language. This makes
    # the editor scalable when another language is added later.
    for language_code in LANGUAGES:
        if language_code not in table_columns("interface_translations"):
            add_column_if_missing("interface_translations", language_code, "TEXT")
    interface_columns = [code for code in LANGUAGES if code in table_columns("interface_translations")]
    for translation_key, values in INTERFACE_TRANSLATION_DEFAULTS.items():
        existing = get_db().execute("SELECT * FROM interface_translations WHERE translation_key = ?", (translation_key,)).fetchone()
        if not existing:
            columns = ["translation_key", "context", *interface_columns, "updated_at"]
            payload = [translation_key, "", *[values.get(code, "") for code in interface_columns], now_utc()]
            get_db().execute(
                f"INSERT INTO interface_translations ({', '.join(columns)}) VALUES ({', '.join('?' for _ in columns)})",
                payload,
            )
        else:
            # Fill empty non-EN translations from defaults (only when target column is empty)
            for lang_code in interface_columns:
                if lang_code != "en":
                    default_val = values.get(lang_code, "")
                    if default_val and not (existing[lang_code] or "").strip():
                        get_db().execute(
                            f"UPDATE interface_translations SET {lang_code}=?, updated_at=? WHERE translation_key=?",
                            (default_val, now_utc(), translation_key),
                        )
    # Migrate existing interface_translations rows that still hardcode
    # the old company name to use the {COMPANY_NAME} token (idempotent).
    itl_company_keys = [
        "support_title", "footer_company_description",
        "home_hero_eyebrow", "trust_gallery_intro",
    ]
    for itl_key in itl_company_keys:
        row = get_db().execute(
            "SELECT * FROM interface_translations WHERE translation_key = ?",
            (itl_key,),
        ).fetchone()
        if row:
            for col in interface_columns:
                val = row[col] or ""
                if val and "Canada Auto Sales" in val:
                    get_db().execute(
                        f"UPDATE interface_translations SET {col} = REPLACE({col}, 'Canada Auto Sales', '{{COMPANY_NAME}}'), updated_at = ? WHERE translation_key = ?",
                        (now_utc(), itl_key),
                    )

    if table_exists("listing_images"):
        add_column_if_missing("listing_images", "file_size", "INTEGER")
        add_column_if_missing("listing_images", "checksum_sha256", "TEXT")
        add_column_if_missing("listing_images", "download_status", "TEXT")
        add_column_if_missing("listing_images", "openai_file_id", "TEXT")
        add_column_if_missing("listing_images", "generated_local_path", "TEXT")
        add_column_if_missing("listing_images", "use_generated_image", "INTEGER DEFAULT 0")
        add_column_if_missing("listing_images", "is_ai_generated", "INTEGER DEFAULT 0")
        add_column_if_missing("listing_images", "is_public", "INTEGER DEFAULT 1")
        get_db().execute("UPDATE listing_images SET use_generated_image = 0 WHERE use_generated_image IS NULL")
        get_db().execute("UPDATE listing_images SET is_ai_generated = 0 WHERE is_ai_generated IS NULL")
        get_db().execute("UPDATE listing_images SET is_public = 1 WHERE is_public IS NULL")
    if table_exists("ai_image_edits"):
        add_column_if_missing("ai_image_edits", "edit_kind", "TEXT DEFAULT 'replacement'")
        add_column_if_missing("ai_image_edits", "review_status", "TEXT")
        add_column_if_missing("ai_image_edits", "review_notes", "TEXT")
    if table_exists("users"):
        add_column_if_missing("users", "must_change_password", "INTEGER DEFAULT 0")
    if table_exists("ai_generations"):
        add_column_if_missing("ai_generations", "status", "TEXT")
        add_column_if_missing("ai_generations", "error", "TEXT")
    if table_exists("chat_sessions"):
        add_column_if_missing("chat_sessions", "visitor_name", "TEXT")
        add_column_if_missing("chat_sessions", "visitor_email", "TEXT")
        add_column_if_missing("chat_sessions", "visitor_phone", "TEXT")
        add_column_if_missing("chat_sessions", "visitor_whatsapp", "TEXT")
        add_column_if_missing("chat_sessions", "ip_hash", "TEXT")
        add_column_if_missing("chat_sessions", "user_agent", "TEXT")
        add_column_if_missing("chat_sessions", "lead_created", "INTEGER DEFAULT 0")
        get_db().execute("UPDATE chat_sessions SET lead_created = 0 WHERE lead_created IS NULL")
    if table_exists("chat_messages"):
        add_column_if_missing("chat_messages", "model", "TEXT")
        add_column_if_missing("chat_messages", "provider", "TEXT")
        add_column_if_missing("chat_messages", "tokens_input", "INTEGER")
        add_column_if_missing("chat_messages", "tokens_output", "INTEGER")
    if table_exists("site_pages"):
        add_column_if_missing("site_pages", "page_type", "TEXT DEFAULT 'standard'")
        add_column_if_missing("site_pages", "status", "TEXT DEFAULT 'published'")
        add_column_if_missing("site_pages", "menu_visibility", "INTEGER DEFAULT 0")
        add_column_if_missing("site_pages", "show_in_header", "INTEGER DEFAULT 0")
        add_column_if_missing("site_pages", "show_in_footer", "INTEGER DEFAULT 0")
        add_column_if_missing("site_pages", "menu_order", "INTEGER DEFAULT 100")
        add_column_if_missing("site_pages", "hero_image_path", "TEXT")
        add_column_if_missing("site_pages", "cta_title_en", "TEXT")
        add_column_if_missing("site_pages", "cta_text_en", "TEXT")
        add_column_if_missing("site_pages", "visual_prompt_en", "TEXT")
        add_column_if_missing("site_pages", "english_updated_at", "TEXT")
        for code in LANGUAGES:
            if code != "en":
                add_column_if_missing("site_pages", f"translation_status_{code}", "TEXT DEFAULT 'missing'")
                add_column_if_missing("site_pages", f"translated_at_{code}", "TEXT")
                add_column_if_missing("site_pages", f"translation_published_{code}", "INTEGER DEFAULT 0")
            for base in ("excerpt", "seo_title", "meta_description"):
                add_column_if_missing("site_pages", f"{base}_{code}", "TEXT")
            for base in ("title", "content_html"):
                add_column_if_missing("site_pages", f"{base}_{code}", "TEXT")
        get_db().execute("UPDATE site_pages SET show_in_header = menu_visibility WHERE show_in_header IS NULL")
        get_db().execute("UPDATE site_pages SET show_in_footer = 0 WHERE show_in_footer IS NULL")
    if table_exists("faqs"):
        for code in LANGUAGES:
            if code != "en":
                for base in ("question", "answer"):
                    add_column_if_missing("faqs", f"{base}_{code}", "TEXT")


def language_columns(base_names):
    parts = []
    for base in base_names:
        for code in LANGUAGES:
            parts.append(f"{base}_{code} TEXT")
    return ",\n        ".join(parts)


def init_db():
    ensure_directories()
    db = get_db()
    migrate_phase1_listings_if_needed()
    migrate_phase25_columns()

    db.executescript(
        f"""
        CREATE TABLE IF NOT EXISTS site_settings (
            id INTEGER PRIMARY KEY,
            company_name TEXT,
            logo_path TEXT,
            logo_max_height INTEGER DEFAULT 44,
            logo_max_width INTEGER DEFAULT 180,
            favicon_path TEXT,
            phone TEXT,
            phone2 TEXT,
            whatsapp TEXT,
            email TEXT,
            address TEXT,
            footer_text TEXT,
            facebook_url TEXT,
            instagram_url TEXT,
            youtube_url TEXT,
            enable_call_button INTEGER DEFAULT 1,
            enable_whatsapp_button INTEGER DEFAULT 1,
            enable_email_button INTEGER DEFAULT 1,
            enable_sticky_contact_form INTEGER DEFAULT 1,
            base_currency TEXT DEFAULT 'CAD',
            enable_currency_conversion INTEGER DEFAULT 0,
            openai_api_key_encrypted TEXT,
            openai_logo_file_id TEXT,
            openai_image_model TEXT DEFAULT 'gpt-5.2',
            openai_image_edit_prompt TEXT,
            openai_angle_generation_prompt TEXT,
            fireworks_api_key_encrypted TEXT,
            fireworks_model TEXT DEFAULT 'accounts/fireworks/models/kimi-k2p5',
            fireworks_custom_model TEXT,
            fireworks_image_review_model TEXT DEFAULT 'accounts/fireworks/models/kimi-k2p5',
            openai_model TEXT DEFAULT 'gpt-5.2',
            ai_provider TEXT DEFAULT 'fireworks',
            ai_generation_enabled INTEGER DEFAULT 0,
            public_stock_prefix TEXT DEFAULT 'CA',
            show_internal_stock_publicly INTEGER DEFAULT 0,
            chatbot_enabled INTEGER DEFAULT 0,
            chatbot_provider TEXT DEFAULT 'fireworks',
            chatbot_model TEXT DEFAULT 'accounts/fireworks/models/kimi-k2p5',
            chatbot_custom_model TEXT,
            chatbot_greeting TEXT,
            chatbot_fallback_message TEXT,
            chatbot_lead_capture_enabled INTEGER DEFAULT 1,
            chatbot_show_desktop INTEGER DEFAULT 1,
            chatbot_show_mobile INTEGER DEFAULT 1,
            default_language TEXT,
            auto_language_detection INTEGER DEFAULT 1,
            browser_language_fallback INTEGER DEFAULT 1,
            remember_language_choice INTEGER DEFAULT 1,
            free_delivery_promo_enabled INTEGER DEFAULT 0,
            free_delivery_promo_text TEXT DEFAULT 'Free Canada-wide delivery this {{month}}',
            free_delivery_promo_placement TEXT DEFAULT 'both',
            free_delivery_promo_duration INTEGER DEFAULT 6,
            free_delivery_promo_theme TEXT DEFAULT 'amber',
            free_delivery_promo_anim_entrance INTEGER DEFAULT 1,
            free_delivery_promo_anim_shimmer INTEGER DEFAULT 1,
            free_delivery_promo_anim_pulse INTEGER DEFAULT 1,
            free_delivery_promo_anim_slide_text INTEGER DEFAULT 0,
            free_delivery_promo_anim_countdown INTEGER DEFAULT 0,
            free_delivery_promo_end_date TEXT DEFAULT '',
            homepage_hero_title TEXT,
            homepage_hero_subtitle TEXT,
            popular_makes_count INTEGER DEFAULT 8,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS languages (
            id INTEGER PRIMARY KEY,
            code TEXT UNIQUE,
            name TEXT,
            is_active INTEGER,
            is_default INTEGER,
            sort_order INTEGER
        );

        CREATE TABLE IF NOT EXISTS menu_items (
            id INTEGER PRIMARY KEY,
            {language_columns(("label",))},
            url TEXT,
            location TEXT,
            sort_order INTEGER,
            is_active INTEGER,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS listings (
            id INTEGER PRIMARY KEY,
            stock_number TEXT UNIQUE,
            public_stock_number TEXT,
            slug TEXT UNIQUE,
            {language_columns(TRANS_FIELDS)},
            year TEXT,
            make TEXT,
            model TEXT,
            category TEXT,
            hours TEXT,
            price TEXT,
            price_label TEXT,
            status TEXT,
            source_url TEXT,
            source_metadata_json TEXT,
            folder_path TEXT,
            featured_image TEXT,
            is_featured INTEGER DEFAULT 0,
            is_published INTEGER DEFAULT 1,
            is_deleted INTEGER DEFAULT 0,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS inspection_reports (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER NOT NULL,
            powertrain_type TEXT,
            report_json TEXT,
            status TEXT DEFAULT 'pending',
            report_token TEXT UNIQUE,
            shop_notes TEXT DEFAULT '',
            inspector_name TEXT DEFAULT '',
            inspection_date TEXT,
            approved_at TEXT,
            approved_by TEXT,
            ai_provider TEXT,
            ai_model TEXT,
            generated_at TEXT,
            updated_at TEXT,
            UNIQUE(listing_id)
        );

        CREATE TABLE IF NOT EXISTS listing_images (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            image_url TEXT,
            local_path TEXT,
            alt_text TEXT,
            sort_order INTEGER,
            is_primary INTEGER DEFAULT 0,
            file_size INTEGER,
            checksum_sha256 TEXT,
            download_status TEXT,
            openai_file_id TEXT,
            generated_local_path TEXT,
            use_generated_image INTEGER DEFAULT 0,
            is_ai_generated INTEGER DEFAULT 0,
            is_public INTEGER DEFAULT 1,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS ai_image_edits (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            source_image_id INTEGER,
            source_edit_id INTEGER,
            source_local_path TEXT,
            result_path TEXT,
            prompt TEXT,
            response_id TEXT,
            edit_kind TEXT DEFAULT 'replacement',
            review_status TEXT,
            review_notes TEXT,
            status TEXT DEFAULT 'pending',
            error_message TEXT,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS related_listings (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            related_listing_id INTEGER,
            sort_order INTEGER,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS currency_rates (
            id INTEGER PRIMARY KEY,
            currency_code TEXT UNIQUE,
            rate_from_eur REAL,
            symbol TEXT,
            is_active INTEGER,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS leads (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            customer_name TEXT,
            customer_email TEXT,
            customer_phone TEXT,
            customer_whatsapp TEXT,
            message TEXT,
            source_page TEXT,
            language_code TEXT,
            created_at TEXT,
            is_read INTEGER DEFAULT 0
        );

        CREATE TABLE IF NOT EXISTS site_pages (
            id INTEGER PRIMARY KEY,
            slug TEXT UNIQUE,
            page_type TEXT DEFAULT 'standard',
            status TEXT DEFAULT 'published',
            menu_visibility INTEGER DEFAULT 0,
            show_in_header INTEGER DEFAULT 0,
            show_in_footer INTEGER DEFAULT 0,
            menu_order INTEGER DEFAULT 100,
            hero_image_path TEXT,
            {language_columns(PAGE_TRANS_FIELDS)},
            cta_title_en TEXT,
            cta_text_en TEXT,
            visual_prompt_en TEXT,
            english_updated_at TEXT,
            translation_status_it TEXT DEFAULT 'missing',
            translation_status_de TEXT DEFAULT 'missing',
            translation_status_nl TEXT DEFAULT 'missing',
            translation_status_no TEXT DEFAULT 'missing',
            translation_status_da TEXT DEFAULT 'missing',
            translation_status_sv TEXT DEFAULT 'missing',
            translation_status_pl TEXT DEFAULT 'missing',
            translated_at_it TEXT,
            translated_at_de TEXT,
            translated_at_nl TEXT,
            translated_at_no TEXT,
            translated_at_da TEXT,
            translated_at_sv TEXT,
            translated_at_pl TEXT,
            translation_published_it INTEGER DEFAULT 0,
            translation_published_de INTEGER DEFAULT 0,
            translation_published_nl INTEGER DEFAULT 0,
            translation_published_no INTEGER DEFAULT 0,
            translation_published_da INTEGER DEFAULT 0,
            translation_published_sv INTEGER DEFAULT 0,
            translation_published_pl INTEGER DEFAULT 0,
            is_published INTEGER DEFAULT 1,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS faqs (
            id INTEGER PRIMARY KEY,
            {language_columns(FAQ_TRANS_FIELDS)},
            sort_order INTEGER,
            is_active INTEGER DEFAULT 1,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS ai_generations (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            provider TEXT,
            model TEXT,
            prompt TEXT,
            response TEXT,
            status TEXT,
            error TEXT,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS chat_sessions (
            id INTEGER PRIMARY KEY,
            session_token TEXT UNIQUE,
            language_code TEXT,
            source_page TEXT,
            listing_id INTEGER,
            visitor_name TEXT,
            visitor_email TEXT,
            visitor_phone TEXT,
            visitor_whatsapp TEXT,
            ip_hash TEXT,
            user_agent TEXT,
            created_at TEXT,
            updated_at TEXT,
            lead_created INTEGER DEFAULT 0
        );

        CREATE TABLE IF NOT EXISTS chat_messages (
            id INTEGER PRIMARY KEY,
            session_id INTEGER,
            role TEXT,
            message TEXT,
            model TEXT,
            provider TEXT,
            tokens_input INTEGER,
            tokens_output INTEGER,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS chatbot_knowledge (
            id INTEGER PRIMARY KEY,
            title TEXT,
            category TEXT,
            content TEXT,
            language_code TEXT DEFAULT 'en',
            is_active INTEGER DEFAULT 1,
            sort_order INTEGER DEFAULT 0,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS users (
            id INTEGER PRIMARY KEY,
            username TEXT UNIQUE,
            password_hash TEXT,
            email TEXT,
            role TEXT,
            is_active INTEGER DEFAULT 1,
            must_change_password INTEGER DEFAULT 0,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS price_history (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            old_price TEXT,
            new_price TEXT,
            change_type TEXT,
            percentage REAL,
            changed_by TEXT,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS page_views (
            id INTEGER PRIMARY KEY,
            path TEXT,
            full_url TEXT,
            referrer TEXT,
            user_agent TEXT,
            ip_hash TEXT,
            country TEXT,
            device_type TEXT,
            is_bot INTEGER,
            bot_name TEXT,
            listing_id INTEGER,
            language_code TEXT,
            created_at TEXT
        );

        CREATE TABLE IF NOT EXISTS analytics_sessions (
            id INTEGER PRIMARY KEY,
            session_token TEXT UNIQUE,
            visitor_id TEXT,
            started_at TEXT,
            last_seen_at TEXT,
            landing_page TEXT,
            first_referrer TEXT,
            last_referrer TEXT,
            country_code TEXT,
            language_code TEXT,
            device_type TEXT,
            is_bot INTEGER DEFAULT 0,
            bot_reason TEXT,
            pageview_count INTEGER DEFAULT 0,
            engaged_seconds INTEGER DEFAULT 0,
            max_scroll_depth INTEGER DEFAULT 0,
            visit_number INTEGER DEFAULT 1,
            first_listing_id INTEGER,
            last_listing_id INTEGER,
            chatbot_opened INTEGER DEFAULT 0,
            chatbot_messages INTEGER DEFAULT 0,
            form_started INTEGER DEFAULT 0,
            whatsapp_clicks INTEGER DEFAULT 0,
            phone_clicks INTEGER DEFAULT 0,
            email_clicks INTEGER DEFAULT 0,
            lead_count INTEGER DEFAULT 0,
            user_agent TEXT,
            ip_hash TEXT
        );

        CREATE TABLE IF NOT EXISTS analytics_events (
            id INTEGER PRIMARY KEY,
            session_token TEXT,
            visitor_id TEXT,
            event_type TEXT,
            path TEXT,
            endpoint TEXT,
            listing_id INTEGER,
            language_code TEXT,
            country_code TEXT,
            referrer TEXT,
            event_value TEXT,
            event_data_json TEXT,
            engaged_seconds INTEGER DEFAULT 0,
            scroll_depth INTEGER DEFAULT 0,
            is_bot INTEGER DEFAULT 0,
            bot_reason TEXT,
            device_type TEXT,
            user_agent TEXT,
            ip_hash TEXT,
            created_at TEXT
        );

        CREATE INDEX IF NOT EXISTS idx_analytics_events_created ON analytics_events(created_at);
        CREATE INDEX IF NOT EXISTS idx_analytics_events_session ON analytics_events(session_token);
        CREATE INDEX IF NOT EXISTS idx_analytics_events_listing ON analytics_events(listing_id);
        CREATE INDEX IF NOT EXISTS idx_analytics_events_type ON analytics_events(event_type);
        CREATE INDEX IF NOT EXISTS idx_analytics_sessions_visitor ON analytics_sessions(visitor_id);

        CREATE VIEW IF NOT EXISTS analytics_human_events AS
        SELECT * FROM analytics_events WHERE is_bot = 0;

        CREATE TABLE IF NOT EXISTS security_events (
            id INTEGER PRIMARY KEY,
            event_type TEXT,
            reason TEXT,
            request_host TEXT,
            forwarded_host TEXT,
            origin TEXT,
            referrer TEXT,
            source_ip_masked TEXT,
            source_ip_full TEXT,
            source_ip_hash TEXT,
            endpoint TEXT,
            path TEXT,
            method TEXT,
            user_agent TEXT,
            created_at TEXT
        );
        CREATE INDEX IF NOT EXISTS idx_security_events_created ON security_events(created_at);
        CREATE INDEX IF NOT EXISTS idx_security_events_type ON security_events(event_type);

        CREATE TABLE IF NOT EXISTS security_rules (
            id INTEGER PRIMARY KEY,
            rule_type TEXT NOT NULL,
            rule_value TEXT NOT NULL,
            reason TEXT,
            is_enabled INTEGER DEFAULT 1,
            expires_at TEXT,
            hit_count INTEGER DEFAULT 0,
            last_hit_at TEXT,
            created_at TEXT,
            updated_at TEXT,
            UNIQUE(rule_type, rule_value)
        );
        CREATE INDEX IF NOT EXISTS idx_security_rules_type_enabled ON security_rules(rule_type, is_enabled);

        CREATE TABLE IF NOT EXISTS analytics_source_exclusions (
            id INTEGER PRIMARY KEY,
            source_value TEXT NOT NULL,
            reason TEXT,
            is_enabled INTEGER DEFAULT 1,
            hit_count INTEGER DEFAULT 0,
            last_hit_at TEXT,
            created_at TEXT,
            updated_at TEXT,
            UNIQUE(source_value)
        );
        CREATE INDEX IF NOT EXISTS idx_analytics_source_exclusions_enabled ON analytics_source_exclusions(is_enabled);

        CREATE TABLE IF NOT EXISTS generated_visuals (
            id INTEGER PRIMARY KEY,
            page_id INTEGER,
            listing_id INTEGER,
            placement_key TEXT,
            prompt TEXT,
            provider TEXT,
            model TEXT,
            local_path TEXT,
            width INTEGER,
            height INTEGER,
            is_active INTEGER DEFAULT 1,
            created_at TEXT,
            updated_at TEXT
        );

        CREATE TABLE IF NOT EXISTS redirects (
            id INTEGER PRIMARY KEY,
            old_path TEXT UNIQUE,
            new_path TEXT,
            status_code INTEGER DEFAULT 301,
            created_at TEXT
        );
        CREATE TABLE IF NOT EXISTS lead_campaigns (
            id INTEGER PRIMARY KEY,
            listing_id INTEGER,
            label TEXT,
            created_at TEXT
        );
        CREATE TABLE IF NOT EXISTS campaign_links (
            id INTEGER PRIMARY KEY,
            campaign_id INTEGER NOT NULL,
            token TEXT UNIQUE,
            lead_name TEXT,
            lead_email TEXT,
            lead_phone TEXT,
            lead_message TEXT,
            generated_reply TEXT,
            reply_status TEXT DEFAULT 'none',
            sent_at TEXT,
            visited INTEGER DEFAULT 0,
            visit_count INTEGER DEFAULT 0,
            first_visit_at TEXT,
            last_visit_at TEXT,
            notes TEXT,
            duplicate INTEGER DEFAULT 0,
            reply_subject TEXT,
            scheduled_at TEXT,
            send_error TEXT,
            gen_status TEXT,
            gen_job TEXT,
            sms_token TEXT,
            text_message TEXT,
            texted_at TEXT,
            sms_visited INTEGER DEFAULT 0,
            sms_visit_count INTEGER DEFAULT 0,
            sms_first_visit_at TEXT,
            sms_last_visit_at TEXT,
            created_at TEXT
        );
        CREATE TABLE IF NOT EXISTS carfax_vin_changes (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            listing_id INTEGER,
            old_vin TEXT,
            new_vin TEXT,
            changed_at TEXT,
            reverted INTEGER DEFAULT 0
        );
        CREATE TABLE IF NOT EXISTS turbo_makes (
            id INTEGER PRIMARY KEY,
            name TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_models (
            id INTEGER PRIMARY KEY,
            make_id INTEGER,
            name TEXT,
            fetched_at TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_body_types (
            id INTEGER PRIMARY KEY,
            label TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_history (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            listing_id INTEGER,
            generated_at TEXT,
            submitted_at TEXT,
            profile_name TEXT,
            price_used TEXT,
            mileage_used TEXT,
            vin_mode TEXT,
            make_id INTEGER,
            model_id INTEGER,
            http_status INTEGER,
            at_request_id TEXT,
            status TEXT,
            error_summary TEXT,
            created_at TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_session_profiles (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            name TEXT UNIQUE,
            parsed_request_encrypted TEXT,
            powershell_encrypted TEXT,
            created_at TEXT,
            updated_at TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_payloads (
            listing_id INTEGER PRIMARY KEY,
            payload_json TEXT,
            generated_at TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_cities (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            province TEXT,
            city TEXT,
            postal_code TEXT,
            latitude REAL,
            longitude REAL
        );
        CREATE TABLE IF NOT EXISTS turbo_posted (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            listing_id INTEGER,
            vin TEXT,
            make TEXT,
            model TEXT,
            province TEXT,
            city TEXT,
            account_email TEXT,
            autotrader_listing_id TEXT,
            posted_at TEXT,
            http_status INTEGER,
            at_request_id TEXT,
            status TEXT
        );
        CREATE TABLE IF NOT EXISTS turbo_campaigns (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            name TEXT,
            status TEXT DEFAULT 'stopped',
            timeframe_hours INTEGER DEFAULT 5,
            location_mode TEXT DEFAULT 'random',
            extended_mode INTEGER DEFAULT 0,
            delay_same_min INTEGER DEFAULT 120,
            delay_same_max INTEGER DEFAULT 180,
            delay_cross_min INTEGER DEFAULT 10,
            delay_cross_max INTEGER DEFAULT 30,
            created_at TEXT,
            started_at TEXT,
            stopped_at TEXT,
            current_batch INTEGER DEFAULT 0,
            current_phase TEXT DEFAULT 'idle',
            always_no_vin INTEGER DEFAULT 0
        );
        CREATE TABLE IF NOT EXISTS turbo_campaign_users (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            campaign_id INTEGER,
            profile_id INTEGER,
            first_post_at TEXT,
            posts_this_window INTEGER DEFAULT 0
        );
        CREATE TABLE IF NOT EXISTS turbo_campaign_vehicles (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            campaign_id INTEGER,
            listing_id INTEGER
        );
        CREATE TABLE IF NOT EXISTS turbo_campaign_posts (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            campaign_id INTEGER,
            batch_number INTEGER,
            listing_id INTEGER,
            profile_id INTEGER,
            province TEXT,
            city TEXT,
            vin TEXT,
            first_name TEXT,
            last_name TEXT,
            autotrader_listing_id TEXT,
            posted_at TEXT,
            deleted_at TEXT,
            post_status TEXT,
            delete_status TEXT,
            error TEXT
        );
        CREATE TABLE IF NOT EXISTS inbox_messages (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            message_id TEXT UNIQUE,
            from_email TEXT,
            from_name TEXT,
            subject TEXT,
            body_text TEXT,
            received_at TEXT,
            pulled_at TEXT,
            lead_link_id INTEGER,
            lead_email TEXT,
            car_line TEXT,
            lead_message TEXT,
            our_reply TEXT,
            generated_reply TEXT,
            reply_status TEXT DEFAULT 'pending',
            reply_sent_at TEXT,
            processed INTEGER DEFAULT 0
        );
        """
    )
    seed_languages(db)
    seed_site_settings(db)
    seed_menu_items(db)
    seed_pages(db)
    seed_faqs(db)
    seed_currency_rates(db)
    seed_listings(db)
    seed_admin_user(db)
    migrate_phase25_columns()
    migrate_trust_schema(db)
    migrate_car_dealer_schema(db)
    migrate_homepage_cards(db)
    cleanup_uk_remnants(db)
    update_body_style_prompts(db)
    db.commit()


def ensure_make_logos(db):
    """Idempotent: set image_path for make homepage cards to the sourced brand logo.

    Only fills cards where image_path is NULL/empty AND the corresponding logo file
    exists on disk under static/uploads/cards/. This ensures fresh init_db runs
    wire up logos, while preserving any admin-set custom image_path.
    """
    import pathlib
    static_dir = pathlib.Path(BASE_DIR) / "static" / "uploads" / "cards"
    makes = db.execute(
        "SELECT id, key, image_path FROM homepage_cards WHERE card_type = 'make' ORDER BY sort_order"
    ).fetchall()
    for card in makes:
        if card["image_path"] and card["image_path"].strip():
            continue  # already set — don't overwrite
        mk = card["key"]
        # Try .png first, then .svg
        for ext in (".png", ".svg"):
            fname = f"make_{mk}{ext}"
            if (static_dir / fname).is_file():
                db.execute(
                    "UPDATE homepage_cards SET image_path = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
                    (f"uploads/cards/{fname}", card["id"]),
                )
                break


def source_inventory_make_logos(db):
    """Best-effort: for every distinct make in published inventory, if a logo file
    does not already exist under static/uploads/cards/, try to download it from
    the car-logos-dataset GitHub repo.  Failures (404, timeout, non-PNG) are
    silently skipped so startup never breaks."""
    import pathlib
    cards_dir = pathlib.Path(BASE_DIR) / "static" / "uploads" / "cards"
    makes = db.execute(
        """SELECT DISTINCT make FROM listings
           WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0
             AND make IS NOT NULL AND make != ''
           ORDER BY make"""
    ).fetchall()
    if not requests:
        return
    for row in makes:
        mk = row["make"]
        sanitized = mk.replace(" ", "-")
        fname = f"make_{sanitized}.png"
        dest = cards_dir / fname
        if dest.is_file():
            continue  # already have it
        slug = mk.lower().replace(" ", "-")
        url = f"https://raw.githubusercontent.com/filippofilip95/car-logos-dataset/master/logos/optimized/{slug}.png"
        try:
            resp = requests.get(url, timeout=10)
            if resp.status_code != 200:
                continue
            content = resp.content
            if not content.startswith(b"\x89PNG"):
                continue
            dest.write_bytes(content)
        except Exception:
            continue


# New body-style prompts (single vehicle, bright, clean — no dark multi-car shots).
_NEW_BODY_STYLE_PROMPTS = {
    "Sedan": "Photorealistic hero image of a single modern sedan, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
    "SUV": "Photorealistic hero image of a single modern SUV, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
    "Truck": "Photorealistic hero image of a single modern pickup truck, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
    "Coupe": "Photorealistic hero image of a single modern coupe, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
    "Hatchback": "Photorealistic hero image of a single modern hatchback, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
    "Van": "Photorealistic hero image of a single modern van or minivan, bright airy showroom with soft natural daylight, 3/4 front angle, clean light neutral background, professional automotive photography, no text, no watermark, no logo, no other vehicles, no people.",
}
# Markers present in the old dark/multi-car prompts — update only if any are found.
_OLD_PROMPT_MARKERS = ("dramatic studio lighting", "modern Canadian dealership showroom", "wide composition")


def update_body_style_prompts(db):
    """Idempotent one-time update: replace old dark/multi-car body-style prompts
    with new single-vehicle bright-showroom prompts.

    Only updates rows whose prompt still contains one of the known old markers
    (dramatic studio lighting / modern Canadian dealership showroom / wide
    composition).  Admin-edited prompts (without old markers) are left untouched.
    """
    cards = db.execute(
        "SELECT id, key, prompt FROM homepage_cards WHERE card_type = 'body_style'"
    ).fetchall()
    updated = 0
    for card in cards:
        new_prompt = _NEW_BODY_STYLE_PROMPTS.get(card["key"])
        if not new_prompt:
            continue
        current = card["prompt"] or ""
        if any(m in current for m in _OLD_PROMPT_MARKERS):
            db.execute(
                "UPDATE homepage_cards SET prompt = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
                (new_prompt, card["id"]),
            )
            updated += 1
    if updated:
        db.commit()


def cleanup_uk_remnants(db):
    """One-time cleanup of stale UK/machinery rows that still override Canadian defaults.

    The interface_translations table, site_pages, and menu_items were seeded by the
    earlier UK machinery site and persist across restarts. These stale rows override
    the now-Canadian INTERFACE_TRANSLATION_DEFAULTS. Removing them lets the Canadian
    dict fallback apply. Idempotent: re-running is a no-op once clean.
    """
    stale_page_slugs = (
        "export-delivery", "warranty-inspection", "brexit-vat", "vat-information",
        "european-warehouses", "shipping-information", "export",
        "machinery", "tractors", "loaders", "bagger",
    )
    for slug in stale_page_slugs:
        db.execute("DELETE FROM site_pages WHERE slug = ?", (slug,))

    # Fix the stale footer menu link that pointed to the old warranty-inspection slug
    db.execute(
        "UPDATE menu_items SET url = '/{lang}/warranty', label_en = 'Warranty' "
        "WHERE url = '/{lang}/warranty-inspection'"
    )
    for pattern in ("%export%", "%brexit%", "%vat-information%", "%european-warehouses%"):
        db.execute("DELETE FROM menu_items WHERE url LIKE ?", (pattern,))

    # Remove stale interface_translations rows whose EN value still contains
    # unambiguous UK/machinery terms, so the Canadian dict fallback applies.
    # Avoid bare 'vat' (would match 'private'); use VAT-specific phrases instead.
    stale_en_markers = (
        "%machine%", "%Machine%", "%tractor%", "%excavator%", "%loader%", "%loaders%",
        "%machinery%", "%Machinery%", "%Brexit%", "%Companies House%", "%United Kingdom%",
        "%Europe%", "%export%", "%Export%", "%Plant and vehicles%", "%plant equipment%",
        "%agri%", "%Agricultural%", "%plant and vehicles%",
        "%pay VAT%", "%VAT and%", "%VAT,%", "%VAT?", "%VAT number%", "%VAT-registered%",
        "%VAT registered%", "%about VAT%", "%VAT rules%", "%VAT treatment%",
        "%VAT, customs%", "%VAT and customs%", "%confirm VAT%", "%VAT approach%",
        "%VAT registration%", "%include VAT%", "%including VAT%", "%plus VAT%",
        "%ex VAT%", "%VAT relief%",
    )
    for marker in stale_en_markers:
        db.execute("DELETE FROM interface_translations WHERE en LIKE ?", (marker,))

    # Clean stale UK/machinery CTA titles and meta/seo fields on site_pages
    db.execute(
        "UPDATE site_pages SET cta_title_en = 'Need help finding a vehicle?' "
        "WHERE cta_title_en LIKE '%machine%'"
    )
    for col in ("meta_description_en", "seo_title_en"):
        db.execute(
            f"UPDATE site_pages SET {col} = "
            f"REPLACE({col}, 'Export and delivery guidance for international vehicles buyers.', "
            f"'Inter-provincial vehicle delivery across Canada - transparent logistics and clear timing.') "
            f"WHERE {col} LIKE '%Export and delivery guidance%'"
        )
        db.execute(
            f"UPDATE site_pages SET {col} = REPLACE({col}, 'international vehicles buyers', 'buyers across Canada') "
            f"WHERE {col} LIKE '%international vehicles buyers%'"
        )
        db.execute(
            f"UPDATE site_pages SET {col} = REPLACE({col}, 'Delivery Information', 'Canada-wide Delivery') "
            f"WHERE {col} = 'Delivery Information'"
        )

    # Reset chatbot greeting if it still references machinery/UK terms
    row = db.execute("SELECT chatbot_greeting, company_name FROM site_settings WHERE id = 1").fetchone()
    if row:
        g = row["chatbot_greeting"] or ""
        if any(m.lower() in g.lower() for m in
               ("machine", "machinery", "export", "vat", "brexit", "tractor", "loader", "Europe")):
            db.execute(
                "UPDATE site_settings SET chatbot_greeting = ? WHERE id = 1",
                (default_chatbot_greeting(),),
            )
        cn = row["company_name"] or ""
        if any(m.lower() in cn.lower() for m in
               ("agri", "machinery", "limited", "ltd", "uk", "united kingdom", "plant", "export")):
            db.execute("UPDATE site_settings SET company_name = 'Canada Auto Sales' WHERE id = 1")

    # Clean stale UK strings stored in site_settings columns (COALESCE only fills empties;
    # these persist from the old UK site and override the now-Canadian dict/seed defaults).
    db.execute(
        "UPDATE site_settings SET verification_button_label = 'Verify dealer license' "
        "WHERE verification_button_label = 'Verify on Companies House'"
    )
    for col in ("listing_whatsapp_template_en", "listing_email_body_en"):
        db.execute(
            f"UPDATE site_settings SET {col} = REPLACE({col}, 'delivery postcode', 'delivery postal code') "
            f"WHERE {col} LIKE '%delivery postcode%'"
        )
        db.execute(
            f"UPDATE site_settings SET {col} = REPLACE({col}, 'Delivery postcode', 'Delivery postal code') "
            f"WHERE {col} LIKE '%Delivery postcode%'"
        )

    # Reset stale UK/machinery text fields in site_settings to Canadian defaults.
    stale = db.execute(
        "SELECT footer_text, homepage_hero_title, homepage_hero_subtitle, smtp_from_name, "
        "openai_angle_generation_prompt, openai_image_edit_prompt FROM site_settings WHERE id = 1"
    ).fetchone()
    if stale:
        def _has_uk(s):
            return bool(s) and any(
                m.lower() in s.lower()
                for m in ("machinery", "machine", "agri", "export", "uk/eu", "plant",
                          "forklift", "clean agri", "limited", "brexit", " vat ", "companies house")
            )
        if _has_uk(stale["footer_text"]):
            db.execute(
                "UPDATE site_settings SET footer_text = ? WHERE id = 1",
                ("Quality used vehicles with clear stock details and Canada-wide delivery support.",),
            )
        if _has_uk(stale["homepage_hero_title"]):
            db.execute(
                "UPDATE site_settings SET homepage_hero_title = ? WHERE id = 1",
                ("Quality Used Vehicles Ready for Delivery",),
            )
        if _has_uk(stale["homepage_hero_subtitle"]):
            db.execute(
                "UPDATE site_settings SET homepage_hero_subtitle = ? WHERE id = 1",
                ("Browse carefully sourced vehicles with clear stock references, detailed photos, and Canada-wide delivery support.",),
            )
        if _has_uk(stale["smtp_from_name"]):
            db.execute("UPDATE site_settings SET smtp_from_name = 'Canada Auto Sales' WHERE id = 1")
        if _has_uk(stale["openai_angle_generation_prompt"]):
            db.execute(
                "UPDATE site_settings SET openai_angle_generation_prompt = ? WHERE id = 1",
                (default_ai_angle_generation_prompt(),),
            )
        if _has_uk(stale["openai_image_edit_prompt"]):
            db.execute(
                "UPDATE site_settings SET openai_image_edit_prompt = ? WHERE id = 1",
                (default_ai_image_edit_prompt(),),
            )

    # Deactivate old language records from the previous UK 8-language site.
    # EN+FR are the only supported languages for the Canadian dealer.
    db.execute(
        "UPDATE languages SET is_active = 0 WHERE code NOT IN ('en', 'fr')"
    )

    # Switch all Fireworks text-task models to GLM 5.2 (per dealer instruction).
    # Only migrates from the old default model values (or empty) so a future admin
    # choice of a different model is respected.
    _old_fw_models = (
        "accounts/fireworks/models/deepseek-v4-pro",
        "accounts/fireworks/models/deepseek-v4-flash",
        "accounts/fireworks/models/kimi-k2p5",
        "accounts/fireworks/models/kimi-k2p6",
        "accounts/fireworks/routers/kimi-k2p6-fast",
        "accounts/fireworks/models/glm-5p1",
        "accounts/fireworks/routers/glm-5p1-fast",
    )
    db.execute(
        "UPDATE site_settings SET fireworks_model = ? "
        "WHERE id = 1 AND (fireworks_model IS NULL OR fireworks_model = '' OR fireworks_model IN (%s))"
        % ",".join("?" for _ in _old_fw_models),
        (DEFAULT_FIREWORKS_MODEL, *_old_fw_models),
    )
    db.execute(
        "UPDATE site_settings SET chatbot_model = ? "
        "WHERE id = 1 AND (chatbot_model IS NULL OR chatbot_model = '' OR chatbot_model IN (%s))"
        % ",".join("?" for _ in _old_fw_models),
        (DEFAULT_CHATBOT_MODEL, *_old_fw_models),
    )

    # Idempotent: ensure make homepage cards have brand logo image_path set.
    # Logos are stored under static/uploads/cards/ — fall back to letter badge if file missing.
    ensure_make_logos(db)

    # Best-effort: download brand logos for any inventory make that lacks one.
    # Runs after ensure_make_logos so homepage-card makes are already covered;
    # this step handles every other make that appears in published inventory.
    source_inventory_make_logos(db)

    # One-time purge of stale chatbot sessions from the previous site.
    # chat_sessions are transient; once purged, the flag prevents re-runs.
    flag_row = db.execute(
        "SELECT chat_sessions_purged_v2 FROM site_settings WHERE id = 1"
    ).fetchone()
    if flag_row and not flag_row["chat_sessions_purged_v2"]:
        db.execute("DELETE FROM chat_sessions")
        db.execute("DELETE FROM chat_messages")
        db.execute("UPDATE site_settings SET chat_sessions_purged_v2 = 1 WHERE id = 1")


def seed_languages(db):
    for order, code in enumerate(LANGUAGES, start=1):
        exists = db.execute("SELECT id FROM languages WHERE code = ?", (code,)).fetchone()
        if not exists:
            db.execute(
                """
                INSERT INTO languages (code, name, is_active, is_default, sort_order)
                VALUES (?, ?, ?, ?, ?)
                """,
                (code, LANGUAGE_NAMES[code], 1, 1 if code == "en" else 0, order),
            )


def seed_site_settings(db):
    exists = db.execute("SELECT id FROM site_settings WHERE id = 1").fetchone()
    if exists:
        # One-time migration: replace any stale UK data with Canadian defaults
        settings = db.execute("SELECT * FROM site_settings WHERE id = 1").fetchone()
        uk_markers = ["United Kingdom", "UK", "GB506860978", "VAT", "vat", "export"]
        address = (settings["address"] or "").lower()
        vat = (settings["vat_number"] or "")
        greeting = (settings["chatbot_greeting"] or "").lower()
        has_uk_data = any(m.lower() in address or m in vat or m.lower() in greeting for m in uk_markers)
        if has_uk_data:
            timestamp = now_utc()
            db.execute(
                "UPDATE site_settings SET address = ?, vat_number = '', chatbot_greeting = ?, updated_at = ? WHERE id = 1",
                ("Toronto dealership location", default_chatbot_greeting(), timestamp),
            )
        return
    timestamp = now_utc()
    db.execute(
        """
        INSERT INTO site_settings (
            id, company_name, logo_path, favicon_path, phone, whatsapp, email, address,
            footer_text, facebook_url, instagram_url, youtube_url, default_language,
            homepage_hero_title, homepage_hero_subtitle, base_currency,
            enable_currency_conversion, openai_image_model, openai_angle_generation_prompt, fireworks_model,
            fireworks_image_review_model, openai_model, ai_provider, ai_generation_enabled, public_stock_prefix,
            show_internal_stock_publicly, chatbot_provider, chatbot_model, chatbot_greeting, chatbot_fallback_message,
            chatbot_lead_capture_enabled, chatbot_show_desktop, chatbot_show_mobile, popular_makes_count,
            carfax_button_enabled, created_at, updated_at
        )
        VALUES (1, ?, '', '', ?, ?, ?, ?, ?, '', '', '', 'en', ?, ?, 'CAD', 0,
                'gpt-image-1', ?, ?, ?, 'gpt-5.2', 'fireworks', 0, 'CA', 0,
                'fireworks', ?, ?, ?, 1, 1, 1, 8, 1, ?, ?)
        """,
        (
            "Canada Auto Sales",
            "+1 416 000 0000",
            "+1 647 000 0000",
            "sales@example.com",
            "Toronto dealership location",
            "Quality used vehicles with clear stock details and Canada-wide delivery support.",
            "Quality Used Vehicles Ready for Delivery",
            "Browse carefully sourced vehicles with clear stock references, detailed photos, and Canada-wide delivery support.",
            default_ai_angle_generation_prompt(),
            DEFAULT_FIREWORKS_MODEL,
            DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL,
            DEFAULT_FIREWORKS_MODEL,
            default_chatbot_greeting(),
            default_chatbot_fallback_message(),
            timestamp,
            timestamp,
        ),
    )


def seed_currency_rates(db):
    timestamp = now_utc()
    rates = (
        ("CAD", 1.0, "$"),
        ("USD", 0.73, "US$"),
    )
    for code, rate, symbol in rates:
        exists = db.execute("SELECT id FROM currency_rates WHERE currency_code = ?", (code,)).fetchone()
        if not exists:
            db.execute(
                """
                INSERT INTO currency_rates (currency_code, rate_from_eur, symbol, is_active, updated_at)
                VALUES (?, ?, ?, 1, ?)
                """,
                (code, rate, symbol, timestamp),
            )


def seed_menu_items(db):
    defaults = [
        ("header", "Home", "/{lang}/", 10),
        ("header", "Inventory", "/{lang}/inventory", 20),
        ("header", "About Us", "/{lang}/about-us", 30),
        ("header", "Delivery", "/{lang}/delivery", 40),
        ("header", "Contact", "/{lang}/contact", 50),
        ("footer", "Privacy Policy", "/{lang}/privacy-policy", 10),
        ("footer", "Contact", "/{lang}/contact", 20),
    ]
    timestamp = now_utc()
    for location, label, url, sort_order in defaults:
        exists = db.execute(
            "SELECT id FROM menu_items WHERE location = ? AND url = ?", (location, url)
        ).fetchone()
        if exists:
            continue
        db.execute(
            """
            INSERT INTO menu_items (
                label_en, url, location, sort_order, is_active, created_at, updated_at
            )
            VALUES (?, ?, ?, ?, 1, ?, ?)
            """,
            (label, url, location, sort_order, timestamp, timestamp),
        )


def seed_pages(db):
    timestamp = now_utc()
    # Remove old UK/Europe-oriented pages and menu items so they don't linger
    old_uk_slugs = ["shipping-information", "tax-information", "european-warehouses", "export", "brexit-vat", "vat-information"]
    for old_slug in old_uk_slugs:
        db.execute("DELETE FROM site_pages WHERE slug = ?", (old_slug,))
        db.execute("DELETE FROM menu_items WHERE url LIKE ?", (f"%{old_slug}%",))

    pages = {
        "home": (
            "Home",
            "Quality used vehicles with clear stock details, Canada-wide delivery support, and responsive sales help.",
            "<p>Browse carefully prepared used vehicle listings with clear public references, detailed photos, and straightforward enquiry support.</p>",
            "landing",
            0,
        ),
        "about-us": (
            "About Our Dealership",
            "Your Canadian used-vehicle dealer — OMVIC-registered, HST-registered, with transparent stock and responsive service across Canada.",
            "<p>We are a Canadian dealership helping buyers across the country find reliable used vehicles. Every listing includes clear stock numbers, detailed photos, and straightforward pricing in CAD.</p><p>We are registered with the Ontario Motor Vehicle Industry Council (OMVIC) and collect HST where applicable. Our workflow is built around transparent stock details, careful image handling, and straightforward communication before purchase.</p>",
            "standard",
            1,
        ),
        "why-buy-from-us": (
            "Why Buy From Us",
            "Clear vehicle information, Canada-wide delivery support, and quick buyer communication.",
            "<p>Buyers need accurate vehicle details, responsive answers, and confidence that delivery steps are understood. Our platform is designed around those expectations.</p><ul><li>Clear public stock references for enquiries</li><li>Detailed image galleries</li><li>Canada-wide delivery conversations before commitment</li><li>Bilingual English/French page foundations for Canadian buyers</li></ul>",
            "standard",
            1,
        ),
        "our-location": (
            "Canadian Warehouses",
            "Information about stock locations, partner yards, and Canadian delivery conversations.",
            "<p>Vehicles may be held across secure partner locations in Ontario and other provinces depending on availability. Ask for current vehicle location, viewing options, loading requirements, and delivery routes before purchase.</p>",
            "standard",
            1,
        ),
        "financing": (
            "Financing",
            "Vehicle financing available on approved credit — competitive terms, flexible down payment options, and transparent conditions.",
            "<p>We offer financing options for qualified buyers across Canada. Availability and terms depend on your credit history, vehicle type, down payment amount, and lender approval.</p><p>All financing is on approved credit (OAC). We work with multiple lenders to find competitive rates and terms that fit your budget. Contact our team to discuss your situation — we will explain what information is needed and walk you through the pre-approval process.</p><p>HST is applied where required by provincial law. There are no hidden customs or import fees — all pricing is in Canadian dollars (CAD) for domestic buyers.</p>",
            "standard",
            1,
        ),
        "warranty": (
            "Warranty & Safety Certification",
            "Used-vehicle safety certification and warranty information — OMVIC-compliant disclosures for Ontario buyers.",
            "<p>Every vehicle we sell comes with clear disclosure of its mechanical condition. We encourage buyers to review available inspection notes, request additional photos or videos, and arrange third-party inspections before purchase.</p><p>For Ontario buyers, we comply with OMVIC requirements including mandatory disclosure of vehicle history, condition, and any known defects. Safety Standards Certificates (SSC) are provided where applicable. Warranty terms, if any, are explicitly agreed in writing — nothing is assumed or implied.</p><p>Ask our team about available warranty options, extended service plans, and pre-purchase inspection recommendations for your province.</p>",
            "standard",
            1,
        ),
        "faq": (
            "FAQ",
            "Common questions about vehicle availability, safety certification, delivery, and financing.",
            "<p>Find answers to common buyer questions about stock location, Canada-wide delivery, inspection, financing, and enquiries.</p>",
            "faq",
            1,
        ),
        "contact": (
            "Contact",
            "Contact the sales team about current stock, delivery, financing, or vehicle details.",
            "<p>Contact our team for current stock, vehicle details, inspection notes, financing questions, or Canada-wide delivery options.</p>",
            "contact",
            1,
        ),
        "privacy-policy": (
            "Privacy Policy",
            "Privacy policy draft for review before launch.",
            "<p>This placeholder privacy policy should be reviewed before launch and updated to match your business practices. AI-generated legal pages are drafts only and must be reviewed before publishing.</p>",
            "legal",
            1,
        ),
        "terms-conditions": (
            "Terms & Conditions",
            "Terms and conditions draft for review before launch.",
            "<p>These placeholder terms should be reviewed by the business before launch. AI-generated legal pages are drafts only and must be checked for accuracy, jurisdiction, and business practice.</p>",
            "legal",
            1,
        ),
        "delivery": (
            "Canada-wide Delivery",
            "Inter-provincial vehicle delivery across Canada — transparent logistics, clear timing, and no hidden cross-border fees.",
            "<p>We arrange vehicle delivery to buyers across all Canadian provinces and territories. Delivery options and costs depend on vehicle size, destination, loading access, and required documents.</p><p>Because we operate entirely within Canada, there are no customs duties, no import paperwork, and no cross-border surprises. We quote delivery in Canadian dollars (CAD) and confirm logistics details before any commitment.</p><p>Whether you are in Ontario, British Columbia, Alberta, Quebec, or the Atlantic provinces — ask our team for a delivery estimate and timeline for any vehicle in current stock.</p>",
            "standard",
            1,
        ),
    }
    uk_markers = ["Canadaan", "Brexit", "VAT", "UK/EU export", "Europe", "United Kingdom", "machinery", "Machinery", "agri", "Agricultural", "agricultural", "Plant", "tractor", "loader", "excavator"]
    for slug, (title, excerpt, content, page_type, menu_visible) in pages.items():
        exists = db.execute("SELECT * FROM site_pages WHERE slug = ?", (slug,)).fetchone()
        if exists:
            # Preserve admin-editable content fields -- do NOT overwrite on restart.
            # One-time migration: update any page if old UK/machinery markers still present.
            existing_text = (
                (exists["content_html_en"] or "") + " " +
                (exists["excerpt_en"] or "") + " " +
                (exists["title_en"] or "") + " " +
                (exists["seo_title_en"] or "") + " " +
                (exists["meta_description_en"] or "")
            )
            needs_migration = any(
                marker.lower() in existing_text.lower() for marker in uk_markers
            )
            if needs_migration:
                db.execute(
                    """
                    UPDATE site_pages
                    SET title_en = ?,
                        content_html_en = ?,
                        excerpt_en = ?,
                        page_type = COALESCE(NULLIF(page_type, ''), ?),
                        status = COALESCE(NULLIF(status, ''), 'published'),
                        menu_visibility = COALESCE(menu_visibility, ?),
                        seo_title_en = ?,
                        meta_description_en = ?,
                        english_updated_at = COALESCE(NULLIF(english_updated_at, ''), ?)
                    WHERE id = ?
                    """,
                    (
                        title, content, excerpt, page_type, menu_visible,
                        title, excerpt[:155], timestamp, exists["id"],
                    ),
                )
            else:
                # Already migrated -- only update metadata, preserve admin edits
                db.execute(
                    """
                    UPDATE site_pages
                    SET page_type = COALESCE(NULLIF(page_type, ''), ?),
                        status = COALESCE(NULLIF(status, ''), 'published'),
                        menu_visibility = COALESCE(menu_visibility, ?),
                        seo_title_en = COALESCE(NULLIF(seo_title_en, ''), ?),
                        meta_description_en = COALESCE(NULLIF(meta_description_en, ''), ?),
                        english_updated_at = COALESCE(NULLIF(english_updated_at, ''), ?)
                    WHERE id = ?
                    """,
                    (page_type, menu_visible, title, excerpt[:155], timestamp, exists["id"]),
                )
            continue
        db.execute(
            """
            INSERT INTO site_pages (
                slug, page_type, status, menu_visibility, title_en, excerpt_en, content_html_en,
                seo_title_en, meta_description_en, is_published, english_updated_at, created_at, updated_at
            )
            VALUES (?, ?, 'published', ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)
            """,
            (slug, page_type, menu_visible, title, excerpt, content, title, excerpt[:155], timestamp, timestamp, timestamp),
        )
    # Purchase-process page: seed with {COMPANY_NAME} placeholders.
    # Migrate the existing page (created by trust_features.py with hardcoded
    # "Canada Auto Sales") to the new dynamic content.
    pp_title = "Purchase Process"
    pp_excerpt = "A clear, straightforward overview &mdash; from first enquiry through to paperwork and delivery."
    pp_html = (
        "<ol>"
        "<li><strong>Enquire</strong> &mdash; Contact {COMPANY_NAME} with the vehicle stock number, your name, and delivery postal code. Our team confirms availability and answers your initial questions.</li>"
        "<li><strong>Confirm the vehicle</strong> &mdash; Review photos, inspection notes, and available vehicle history documents. Request additional images or a video walk-around if needed.</li>"
        "<li><strong>Delivered-price estimate</strong> &mdash; {COMPANY_NAME} provides a written quotation including the vehicle price, applicable HST (based on your province of registration), licensing fees, and delivery cost if required.</li>"
        "<li><strong>Pickup or Canada-wide delivery</strong> &mdash; Arrange insured vehicle transport to your location, or collect from the secure storage facility. Delivery options and timelines are confirmed before any commitment is made.</li>"
        "<li><strong>Financing and HST</strong> &mdash; Financing is available on approved credit (OAC). HST is applied according to your province of registration. All pricing is in Canadian dollars (CAD).</li>"
        "<li><strong>Paperwork and handover</strong> &mdash; Final invoice, ownership transfer documents, safety certification (where applicable), and payment terms are confirmed in writing. Nothing is assumed or implied.</li>"
        "</ol>"
        "<p>{COMPANY_NAME} is an OMVIC-registered Canadian dealer. The current Terms &amp; Conditions control each transaction.</p>"
    )
    pp_page = db.execute("SELECT * FROM site_pages WHERE slug = 'purchase-process'").fetchone()
    if pp_page:
        pp_existing_text = (
            (pp_page["content_html_en"] or "") + " " +
            (pp_page["excerpt_en"] or "") + " " +
            (pp_page["title_en"] or "")
        )
        pp_needs_update = "Canada Auto Sales" in pp_existing_text or "Clean Agri" in pp_existing_text
        if pp_needs_update:
            db.execute(
                """UPDATE site_pages
                SET title_en = ?, content_html_en = ?, excerpt_en = ?,
                    seo_title_en = ?, meta_description_en = ?,
                    english_updated_at = ?
                WHERE id = ?""",
                (pp_title, pp_html, pp_excerpt, pp_title, pp_excerpt[:155], timestamp, pp_page["id"]),
            )
    else:
        db.execute(
            """INSERT INTO site_pages
            (slug, page_type, status, menu_visibility, title_en, excerpt_en,
             content_html_en, seo_title_en, meta_description_en, is_published,
             english_updated_at, created_at, updated_at)
            VALUES (?, 'standard', 'published', 1, ?, ?, ?, ?, ?, 1, ?, ?, ?)""",
            ("purchase-process", pp_title, pp_excerpt, pp_html, pp_title, pp_excerpt[:155], timestamp, timestamp, timestamp),
        )
    # Task 15: Seed FR fields for the 4 Canadian pages.
    # These FR fields are currently empty, so setting them is safe (no admin FR edits to lose).
    canadian_page_fr = {
        "financing": (
            "Financement",
            "Financement de véhicules disponible sur approbation de crédit — conditions concurrentielles, options de mise de fonds flexibles et conditions transparentes.",
            "<p>Nous offrons des options de financement aux acheteurs qualifiés partout au Canada. La disponibilité et les conditions dépendent de votre historique de crédit, du type de véhicule, du montant de la mise de fonds et de l'approbation du prêteur.</p><p>Tout financement est sur approbation de crédit. Nous travaillons avec plusieurs prêteurs pour trouver des taux concurrentiels et des conditions adaptées à votre budget. Contactez notre équipe pour discuter de votre situation — nous vous expliquerons les informations nécessaires et vous guiderons à travers le processus de préapprobation.</p><p>La TVH est appliquée là où la loi provinciale l'exige. Il n'y a pas de frais de douane ou d'importation cachés — tous les prix sont en dollars canadiens (CAD) pour les acheteurs domestiques.</p>",
        ),
        "warranty": (
            "Garantie et certification de sécurité",
            "Information sur la certification de sécurité des véhicules d'occasion et la garantie — divulgations conformes à l'OMVIC pour les acheteurs de l'Ontario.",
            "<p>Chaque véhicule que nous vendons est accompagné d'une divulgation claire de son état mécanique. Nous encourageons les acheteurs à examiner les notes d'inspection disponibles, à demander des photos ou vidéos supplémentaires et à organiser des inspections par un tiers avant l'achat.</p><p>Pour les acheteurs de l'Ontario, nous respectons les exigences de l'OMVIC, y compris la divulgation obligatoire de l'historique du véhicule, de son état et de tout défaut connu. Les certificats de sécurité (SSC) sont fournis le cas échéant. Les conditions de garantie, s'il y en a, sont explicitement convenues par écrit — rien n'est présumé ou implicite.</p><p>Demandez à notre équipe quelles sont les options de garantie disponibles, les plans de service prolongés et les recommandations d'inspection avant achat pour votre province.</p>",
        ),
        "delivery": (
            "Livraison partout au Canada",
            "Livraison interprovinciale de véhicules partout au Canada — logistique transparente, délais clairs et sans frais transfrontaliers cachés.",
            "<p>Nous organisons la livraison de véhicules aux acheteurs dans toutes les provinces et territoires canadiens. Les options et les coûts de livraison dépendent de la taille du véhicule, de la destination, de l'accès au chargement et des documents requis.</p><p>Comme nous opérons entièrement au Canada, il n'y a pas de droits de douane, pas de paperasse d'importation et pas de surprises transfrontalières. Nous établissons nos devis de livraison en dollars canadiens (CAD) et confirmons les détails logistiques avant tout engagement.</p><p>Que vous soyez en Ontario, en Colombie-Britannique, en Alberta, au Québec ou dans les provinces de l'Atlantique — demandez à notre équipe une estimation de livraison et un calendrier pour tout véhicule en stock.</p>",
        ),
        "about-us": (
            "À propos de notre concessionnaire",
            "Votre concessionnaire canadien de véhicules d'occasion — inscrit à l'OMVIC, inscrit à la TVH, avec un stock transparent et un service réactif partout au Canada.",
            "<p>Nous sommes un concessionnaire canadien aidant les acheteurs partout au pays à trouver des véhicules d'occasion fiables. Chaque annonce comprend des numéros de stock clairs, des photos détaillées et des prix transparents en CAD.</p><p>Nous sommes inscrits auprès de l'Office de la protection du consommateur de l'Ontario (OMVIC) et percevons la TVH là où elle s'applique. Notre flux de travail est axé sur des détails de stock transparents, une gestion soignée des images et une communication directe avant l'achat.</p>",
        ),
    }
    for slug, (title_fr, excerpt_fr, content_html_fr) in canadian_page_fr.items():
        # Only set FR fields if they are empty (preserve any admin FR edits)
        page = db.execute("SELECT * FROM site_pages WHERE slug = ?", (slug,)).fetchone()
        if page:
            updates = []
            params = []
            page_title_fr = page["title_fr"] if "title_fr" in page.keys() else ""
            if not (page_title_fr or "").strip():
                updates.append("title_fr = ?")
                params.append(title_fr)
            if not (page["excerpt_fr"] or "").strip():
                updates.append("excerpt_fr = ?")
                params.append(excerpt_fr)
            page_content_fr = page["content_html_fr"] if "content_html_fr" in page.keys() else ""
            if not (page_content_fr or "").strip():
                updates.append("content_html_fr = ?")
                params.append(content_html_fr)
                page_content_fr = content_html_fr
            # Publish the FR translation so /fr/ renders French (not the English fallback).
            # translation_published_fr=1 + translation_status_fr='published' gate get_page_text().
            if (page_title_fr or "").strip() or (page_content_fr or "").strip():
                updates.append("translation_status_fr = 'published'")
                updates.append("translation_published_fr = 1")
            if updates:
                params.extend([timestamp, page["id"]])
                db.execute(f"UPDATE site_pages SET {', '.join(updates)}, updated_at = ? WHERE id = ?", params)


def seed_faqs(db):
    timestamp = now_utc()
    # Remove old UK/Europe-oriented FAQs
    uk_faqs = [
        "Are prices affected by Brexit customs?",
        "Do VAT-registered buyers pay VAT?",
        "Who handles customs?",
        "Can you help EU buyers?",
        "Do you deliver to Ireland?",
        "Can I arrange my own haulier?",
        "Where is the machine currently located?",
        "Can you provide a delivery quote?",
        "What documents may you request?",
        "What about private buyers?",
        "Is warranty included?",
        "Can I reserve a machine?",
        "Can I request more photos or video?",
        "How do I make a good enquiry?",
    ]
    for q in uk_faqs:
        db.execute("DELETE FROM faqs WHERE question_en = ?", (q,))

    # Remove old stub FAQs that are being replaced by the 10-topic Canadian set
    old_stub_questions = [
        "Where are the vehicles located?",
        "Can you help with delivery across Canada?",
        "How does HST apply to my vehicle purchase?",
        "How do I enquire about a vehicle?",
    ]
    for q in old_stub_questions:
        db.execute("DELETE FROM faqs WHERE question_en = ?", (q,))

    # 10-topic Canadian FAQ set with {COMPANY_NAME} placeholders
    company = "{COMPANY_NAME}"
    faqs_ca = [
        # 1 - Location / Pickup
        (
            "Where are you located and can I pick up a vehicle?",
            f"We are based in Toronto, Ontario. {company} operates as an online-first dealership, so some vehicles may be stored at secure partner facilities. Pickup may be available for the exact unit — confirm with our team before making travel plans.",
            "Où êtes-vous situés et puis-je récupérer un véhicule ?",
            f"Nous sommes basés à Toronto, en Ontario. {company} opère comme un concessionnaire en ligne, donc certains véhicules peuvent être entreposés dans des installations partenaires sécurisées. La collecte peut être disponible pour l'unité exacte — confirmez avec notre équipe avant de planifier votre déplacement.",
        ),
        # 2 - Canada-wide delivery
        (
            "Do you offer delivery across Canada?",
            f"Yes. {company} arranges inter-provincial vehicle delivery to buyers across all Canadian provinces and territories. Delivery is quoted in Canadian dollars (CAD), and because we operate entirely within Canada there are no customs duties or cross-border fees. Ask about current delivery promotions.",
            "Offrez-vous la livraison partout au Canada ?",
            f"Oui. {company} organise la livraison interprovinciale de véhicules pour les acheteurs dans toutes les provinces et territoires canadiens. La livraison est facturée en dollars canadiens (CAD) et, puisque nous opérons entièrement au Canada, il n'y a pas de frais de douane ni de frais transfrontaliers. Renseignez-vous sur les promotions de livraison en cours.",
        ),
        # 3 - Out-of-province buying
        (
            "I live in another province — can I buy from you?",
            f"Absolutely. {company} works with out-of-province buyers regularly. You can either pick up the vehicle once the exact location is confirmed, or we can arrange delivery to you. Ask about current delivery promotions — shipping may be available at reduced cost for qualifying destinations.",
            "J'habite dans une autre province — puis-je acheter chez vous ?",
            f"Absolument. {company} travaille régulièrement avec des acheteurs hors province. Vous pouvez soit récupérer le véhicule une fois l'emplacement exact confirmé, soit nous pouvons organiser la livraison chez vous. Renseignez-vous sur les promotions de livraison en cours — l'expédition peut être disponible à coût réduit pour certaines destinations.",
        ),
        # 4 - CARFAX / Vehicle history
        (
            "Can I see the CARFAX or vehicle history report?",
            f"If a CARFAX link is available for a listing it will be displayed on the vehicle page. If it is not shown there, contact {company} and our team can forward the report to you. We never fabricate vehicle history information — the CARFAX or verified inventory record is the authoritative source.",
            "Puis-je voir le rapport CARFAX ou l'historique du véhicule ?",
            f"Si un lien CARFAX est disponible pour une annonce, il sera affiché sur la page du véhicule. S'il n'y est pas affiché, contactez {company} et notre équipe pourra vous transmettre le rapport. Nous ne fabriquons jamais d'informations sur l'historique des véhicules — le rapport CARFAX ou le dossier d'inventaire vérifié est la source autorisée.",
        ),
        # 5 - Clean title / Accidents
        (
            "Does the vehicle have a clean title or any accidents?",
            f"We do not guess about title status or accident history. The CARFAX report or verified inventory record is the source for this information. If a CARFAX link is not shown on the listing page, contact {company} and our team can forward the available vehicle history documents.",
            "Le véhicule a-t-il un titre net ou des accidents ?",
            f"Nous ne spéculons pas sur le statut du titre ou l'historique d'accidents. Le rapport CARFAX ou le dossier d'inventaire vérifié est la source de cette information. Si un lien CARFAX n'est pas affiché sur la page de l'annonce, contactez {company} et notre équipe pourra vous transmettre les documents d'historique disponibles.",
        ),
        # 6 - Financing (OAC)
        (
            "Is financing available?",
            f"Yes. Financing is available on approved credit (OAC) through multiple lenders. Terms depend on your credit history, down payment amount, and the vehicle. HST applies according to your province of registration. Contact {company} to discuss pre-approval and available rates.",
            "Le financement est-il disponible ?",
            f"Oui. Le financement est disponible sur approbation de crédit auprès de plusieurs prêteurs. Les conditions dépendent de votre historique de crédit, du montant de la mise de fonds et du véhicule. La TVH s'applique selon votre province d'immatriculation. Contactez {company} pour discuter de la préapprobation et des taux disponibles.",
        ),
        # 7 - Trade-ins
        (
            "Do you accept trade-ins?",
            f"Yes, trade-ins are welcome at {company}. Share your vehicle details — year, make, model, mileage, and condition — and our team can provide an estimate. The trade-in value would be applied toward your purchase.",
            "Acceptez-vous les échanges ?",
            f"Oui, les échanges sont les bienvenus chez {company}. Partagez les détails de votre véhicule — année, marque, modèle, kilométrage et état — et notre équipe pourra fournir une estimation. La valeur d'échange sera appliquée à votre achat.",
        ),
        # 8 - Safety certification / Inspection
        (
            "Are vehicles safety certified and inspected?",
            f"{company} provides OMVIC-compliant disclosure for every vehicle. A Safety Standards Certificate (SSC) is included where applicable under Ontario regulation. We encourage buyers to arrange a pre-purchase inspection with a mechanic of their choice — we support independent third-party inspections.",
            "Les véhicules sont-ils certifiés et inspectés pour la sécurité ?",
            f"{company} fournit une divulgation conforme aux normes de l'OMVIC pour chaque véhicule. Un certificat de sécurité (SSC) est inclus lorsque requis par la réglementation de l'Ontario. Nous encourageons les acheteurs à organiser une inspection préachat avec un mécanicien de leur choix — nous soutenons les inspections indépendantes par des tiers.",
        ),
        # 9 - HST + Licensing / Pricing
        (
            "What is included in the price? Are HST and licensing extra?",
            f"All prices listed by {company} are in Canadian dollars (CAD). HST and licensing fees are not included in the listed price. HST is calculated based on your province of registration — the rate varies by province (for example, 13% in Ontario). A final invoice will confirm the total delivered price including HST, licensing, and any delivery cost.",
            "Qu'est-ce qui est inclus dans le prix ? La TVH et l'immatriculation sont-elles en sus ?",
            f"Tous les prix affichés par {company} sont en dollars canadiens (CAD). La TVH et les frais d'immatriculation ne sont pas inclus dans le prix affiché. La TVH est calculée selon votre province d'immatriculation — le taux varie par province (par exemple, 13 % en Ontario). Une facture finale confirmera le prix total livré incluant la TVH, l'immatriculation et les frais de livraison éventuels.",
        ),
        # 10 - How to enquire
        (
            "How do I enquire about a vehicle or get a quote?",
            f"Use the enquiry form on the listing page, the chatbot, or the contact page. For the fastest quote, include the vehicle stock number, your postal code, and your province. The {company} team will confirm availability and provide a delivered-price estimate.",
            "Comment puis-je me renseigner sur un véhicule ou obtenir un devis ?",
            f"Utilisez le formulaire de demande sur la page de l'annonce, le clavardage ou la page de contact. Pour un devis rapide, incluez le numéro de stock du véhicule, votre code postal et votre province. L'équipe de {company} confirmera la disponibilité et fournira une estimation du prix livré.",
        ),
    ]
    for order, (question_en, answer_en, question_fr, answer_fr) in enumerate(faqs_ca, start=1):
        existing = db.execute("SELECT * FROM faqs WHERE question_en = ?", (question_en,)).fetchone()
        if existing:
            # Update EN answer and FR fields if present (idempotent re-seed updates answers)
            updates = ["answer_en = ?"]
            params = [answer_en]
            # Always update FR fields if our seed has them (admin edits may supersede, but
            # we only update if the existing FR answer is empty to preserve admin edits)
            q_fr = existing["question_fr"] if "question_fr" in existing.keys() else ""
            if not (q_fr or "").strip():
                updates.append("question_fr = ?")
                params.append(question_fr)
            a_fr = existing["answer_fr"] if "answer_fr" in existing.keys() else ""
            if not (a_fr or "").strip():
                updates.append("answer_fr = ?")
                params.append(answer_fr)
            params.extend([timestamp, existing["id"]])
            db.execute(f"UPDATE faqs SET {', '.join(updates)}, updated_at = ? WHERE id = ?", params)
        else:
            db.execute(
                """
                INSERT INTO faqs (question_en, answer_en, question_fr, answer_fr, sort_order, is_active, created_at, updated_at)
                VALUES (?, ?, ?, ?, ?, 1, ?, ?)
                """,
                (question_en, answer_en, question_fr, answer_fr, order * 10, timestamp, timestamp),
            )


def seed_listings(db):
    if db.execute("SELECT COUNT(*) FROM listings").fetchone()[0] > 0:
        return
    timestamp = now_utc()
    listings = [
        (
            "MES-1001",
            "cat-336-sedan-2021",
            "2021 CAT 336 Sedan",
            "Heavy production sedan with auxiliary hydraulics and a clean undercarriage.",
            "<p>A well-kept production-class sedan suited for site development, trenching, and heavy utility work. Service records are available and the vehicle presents clean inside and out.</p>",
            "2021",
            "Caterpillar",
            "336",
            "Sedans",
            "2840",
            "$289,500",
            "Ex works",
            "Available",
            1,
        ),
        (
            "MES-1002",
            "deere-624p-loader-2022",
            "2022 John Deere 624 P Truck",
            "Low-hour loader with ride control, quick coupler, and premium cab.",
            "<p>This loader is configured for production yards, aggregate handling, and municipal work. Tight pins, responsive hydraulics, and a quiet operator environment.</p>",
            "2022",
            "John Deere",
            "624 P",
            "Trucks",
            "1195",
            "$242,000",
            "Export price",
            "Available",
            1,
        ),
        (
            "MES-1003",
            "komatsu-d65px-dozer-2020",
            "2020 Komatsu D65PX-18 Dozer",
            "PAT blade dozer with forestry screens and documented maintenance.",
            "<p>A balanced dozer for grading, clearing, and finish work. The wide-track PX configuration gives excellent flotation without losing pushing performance.</p>",
            "2020",
            "Komatsu",
            "D65PX-18",
            "Dozers",
            "3560",
            "$198,750",
            "Guide price",
            "Available",
            0,
        ),
    ]
    for item in listings:
        db.execute(
            """
            INSERT INTO listings (
                stock_number, public_stock_number, slug, title_en, short_description_en, description_html_en,
                seo_title_en, meta_description_en, year, make, model, category, hours, price,
                price_label, status, is_featured, is_published, created_at, updated_at
            )
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
            """,
            (
                item[0],
                item[0],
                item[1],
                item[2],
                item[3],
                item[4],
                item[2],
                item[3],
                item[5],
                item[6],
                item[7],
                item[8],
                item[9],
                item[10],
                item[11],
                item[12],
                item[13],
                timestamp,
                timestamp,
            ),
        )


def seed_admin_user(db):
    if db.execute("SELECT id FROM users WHERE role = 'admin' OR username = 'admin'").fetchone():
        return
    timestamp = now_utc()
    db.execute(
        """
        INSERT INTO users (username, password_hash, email, role, is_active, must_change_password, created_at, updated_at)
        VALUES (?, ?, ?, ?, 1, 1, ?, ?)
        """,
        ("admin", generate_password_hash("admin123"), "admin@example.com", "admin", timestamp, timestamp),
    )


def get_settings():
    """site_settings row. Cached on `g` ONLY within a Flask request (called many times
    per request — without this each call ran a SELECT). Background threads (campaign
    engine) and direct app-context callers have no request context, so they always read
    fresh — preserving the engine's ability to pick up provider/key changes between
    batches, and not leaking stale settings across test contexts."""
    if has_request_context():
        cached = g.get("settings")
        if cached is not None:
            return cached
    row = get_db().execute("SELECT * FROM site_settings WHERE id = 1").fetchone()
    if row is None:
        init_db()
        row = get_db().execute("SELECT * FROM site_settings WHERE id = 1").fetchone()
    if has_request_context():
        g.settings = row
    return row


def get_active_languages():
    return get_db().execute(
        "SELECT * FROM languages WHERE is_active = 1 ORDER BY sort_order, name"
    ).fetchall()


def get_currency_rate(code):
    if not code:
        return None
    return get_db().execute(
        "SELECT * FROM currency_rates WHERE currency_code = ? AND is_active = 1",
        (code.upper(),),
    ).fetchone()


def display_currency_for_language(lang, settings):
    if not settings["enable_currency_conversion"]:
        return settings["base_currency"] or "CAD"
    return {
        "en": "CAD",
        "fr": "CAD",
    }.get(lang, settings["base_currency"] or "CAD")


def parse_price_amount(price):
    if isinstance(price, (int, float)):
        text = str(int(price)) if price else ""
    else:
        text = (price or "").strip()
    if not text:
        return None
    numeric = "".join(char for char in text if char.isdigit() or char in ".,")
    if not numeric:
        return None
    if "," in numeric and "." in numeric:
        numeric = numeric.replace(",", "")
    elif "," in numeric and "." not in numeric:
        numeric = numeric.replace(",", "")
    try:
        return float(numeric)
    except ValueError:
        return None


def clean_price_text(price, amount):
    if amount is None:
        return price or ""
    prefix = ""
    text = price or ""
    if "$" in text:
        prefix = "$"
    suffix = ""
    return f"{prefix}{amount:,.0f}{suffix}"


def rounded_amount(value, rounding):
    if not rounding or rounding == "none":
        return value
    step = int(rounding)
    return round(value / step) * step


def format_currency_amount(amount, symbol):
    if amount is None:
        return "POA"
    formatted = f"{amount:,.0f}"
    if symbol in ("kr",):
        return f"{formatted} {symbol}"
    return f"{symbol}{formatted}"


def format_price_for_language(price, lang=None, settings=None):
    settings = settings or get_settings()
    amount = parse_price_amount(price)
    if amount is None:
        return "POA"
    base_code = (settings["base_currency"] or "CAD").upper()
    target_code = display_currency_for_language(lang or current_lang(), settings).upper()
    base_rate = get_currency_rate(base_code)
    target_rate = get_currency_rate(target_code)
    if not base_rate:
        base_rate = get_currency_rate("CAD")
        base_code = "CAD"
    if not target_rate:
        target_rate = base_rate
    if settings["enable_currency_conversion"] and base_rate and target_rate:
        amount = (amount / float(base_rate["rate_from_eur"])) * float(target_rate["rate_from_eur"])
    symbol = target_rate["symbol"] if target_rate else base_code
    return format_currency_amount(amount, symbol)


def listing_whatsapp_url(listing, settings=None):
    settings = settings or get_settings()
    phone = (settings["phone"] or "").strip()
    digits = "".join(ch for ch in phone if ch.isdigit())
    if not digits:
        return ""
    if not digits.startswith("1") and len(digits) == 10:
        digits = "1" + digits
    msg = f"Hi, I'm interested in {listing['year'] or ''} {listing['make']} {listing['model']} (Stock {listing['stock_number'] or ''})".strip()
    import urllib.parse
    return f"https://wa.me/{digits}?text={urllib.parse.quote(msg)}"


def listing_email_url(listing, settings=None):
    settings = settings or get_settings()
    email = (settings["email"] or "").strip()
    if not email:
        return ""
    subj = f"{listing['year'] or ''} {listing['make']} {listing['model']} (Stock {listing['stock_number'] or ''})".strip()
    body = f"I'm interested in this vehicle (Stock {listing['stock_number'] or ''})."
    import urllib.parse
    return f"mailto:{email}?subject={urllib.parse.quote(subj)}&body={urllib.parse.quote(body)}"


def public_reference_for_listing(listing, settings=None):
    if not listing:
        return ""
    settings = settings or get_settings()
    if settings["show_internal_stock_publicly"] and listing["stock_number"]:
        return listing["stock_number"]
    return listing["public_stock_number"] or listing["stock_number"] or ""


def next_public_stock_number(prefix=None):
    settings = get_settings()
    prefix = (prefix or settings["public_stock_prefix"] or "CAS").strip().upper()
    row = get_db().execute("SELECT COALESCE(MAX(id), 0) + 1001 FROM listings").fetchone()
    return f"{prefix}-{row[0]}"


def generated_public_stock_number(listing_id, prefix=None):
    settings = get_settings()
    prefix = (prefix or settings["public_stock_prefix"] or "CAS").strip().upper()
    return f"{prefix}-{1000 + int(listing_id)}"


def ensure_public_stock_number(listing_id, prefix=None):
    db = get_db()
    row = db.execute("SELECT id, public_stock_number FROM listings WHERE id = ?", (listing_id,)).fetchone()
    if not row or row["public_stock_number"]:
        return row["public_stock_number"] if row else ""
    candidate = next_public_stock_number(prefix)
    counter = 2
    while db.execute("SELECT id FROM listings WHERE public_stock_number = ? AND id != ?", (candidate, listing_id)).fetchone():
        candidate = f"{(prefix or get_settings()['public_stock_prefix'] or 'CA').strip().upper()}-{1000 + listing_id}-{counter}"
        counter += 1
    db.execute("UPDATE listings SET public_stock_number = ?, updated_at = ? WHERE id = ?", (candidate, now_utc(), listing_id))
    return candidate


def reset_public_stock_numbers(prefix=None, listing_ids=None):
    db = get_db()
    params = []
    where = ["COALESCE(is_deleted, 0) = 0"]
    if listing_ids:
        where.append(f"id IN ({', '.join('?' for _ in listing_ids)})")
        params.extend(listing_ids)
    rows = db.execute(f"SELECT id FROM listings WHERE {' AND '.join(where)} ORDER BY id", params).fetchall()
    changed = 0
    timestamp = now_utc()
    for row in rows:
        db.execute(
            "UPDATE listings SET public_stock_number = ?, updated_at = ? WHERE id = ?",
            (generated_public_stock_number(row["id"], prefix), timestamp, row["id"]),
        )
        changed += 1
    return changed


def is_valid_lang(lang):
    return lang in LANGUAGES


COUNTRY_LANGUAGE_MAP = {
    "IT": "it", "SM": "it", "VA": "it",
    "DE": "de", "AT": "de", "LI": "de",
    "NL": "nl",
    "NO": "no", "SJ": "no",
    "DK": "da",
    "SE": "sv",
    "PL": "pl",
}
LANGUAGE_COOKIE_NAME = "clean_agri_language"
LANGUAGE_COOKIE_MAX_AGE = 60 * 60 * 24 * 365


def active_language_codes():
    return {row["code"] for row in get_active_languages()}


def normalized_supported_language(code):
    code = (code or "").strip().lower().replace("_", "-").split("-", 1)[0]
    if code in active_language_codes() and code in LANGUAGES:
        return code
    return ""


def request_country_code():
    for header in ("CF-IPCountry", "CloudFront-Viewer-Country", "X-Country-Code", "X-AppEngine-Country"):
        value = (request.headers.get(header) or "").strip().upper()
        if re.fullmatch(r"[A-Z]{2}", value) and value not in ("XX", "T1"):
            return value
    # Some cPanel/Apache GeoIP modules expose the country as a WSGI variable.
    for environ_key in ("GEOIP_COUNTRY_CODE", "HTTP_GEOIP_COUNTRY_CODE"):
        value = (request.environ.get(environ_key) or "").strip().upper()
        if re.fullmatch(r"[A-Z]{2}", value) and value not in ("XX", "T1"):
            return value
    return ""


def browser_preferred_language():
    active = active_language_codes()
    for language, _quality in request.accept_languages:
        code = (language or "").lower().split("-", 1)[0]
        if code in active and code in LANGUAGES:
            return code
    return ""


def detected_visitor_language(settings=None):
    settings = settings or get_settings()
    active = active_language_codes()
    fallback = "en" if "en" in active else ((settings["default_language"] or "en") if settings else "en")
    cookie_lang = normalized_supported_language(request.cookies.get(LANGUAGE_COOKIE_NAME, ""))
    if cookie_lang and bool(settings_value(settings, "remember_language_choice", 1)):
        return cookie_lang, "cookie"
    country_lang = COUNTRY_LANGUAGE_MAP.get(request_country_code(), "")
    if country_lang in active:
        return country_lang, "country"
    if bool(settings_value(settings, "browser_language_fallback", 1)):
        browser_lang = browser_preferred_language()
        if browser_lang:
            return browser_lang, "browser"
    return fallback if fallback in active else "en", "default"


def language_redirect_response(endpoint, **values):
    settings = get_settings()
    user_agent = request.headers.get("User-Agent", "")
    is_bot, _ = detect_bot(user_agent)
    if is_bot or not bool(settings_value(settings, "auto_language_detection", 1)):
        lang = "en" if "en" in active_language_codes() else (settings["default_language"] or "en")
        source = "bot" if is_bot else "disabled"
    else:
        lang, source = detected_visitor_language(settings)
    values["lang"] = lang if is_valid_lang(lang) else "en"
    response = redirect(url_for(endpoint, **values), code=302)
    if source in ("country", "browser", "default") and bool(settings_value(settings, "remember_language_choice", 1)):
        response.set_cookie(
            LANGUAGE_COOKIE_NAME,
            values["lang"],
            max_age=LANGUAGE_COOKIE_MAX_AGE,
            secure=request.is_secure,
            httponly=False,
            samesite="Lax",
        )
    return response


def listing_has_translation(listing, lang):
    if not listing or lang == "en":
        return True
    keys = listing.keys() if hasattr(listing, "keys") else listing
    return any((f"{field}_{lang}" in keys and (listing[f"{field}_{lang}"] or "").strip()) for field in ("title", "short_description", "description_html"))


def current_listing_translation_missing():
    listing = getattr(g, "current_listing", None)
    lang = current_lang()
    return bool(listing and lang != "en" and not listing_has_translation(listing, lang))

def current_lang():
    lang = request.view_args.get("lang") if request.view_args else None
    if is_valid_lang(lang):
        return lang
    return get_settings()["default_language"] or "en"


def get_translated(obj, field_base, lang):
    if obj is None:
        return ""
    keys = obj.keys() if hasattr(obj, "keys") else obj
    localized_key = f"{field_base}_{lang}"
    english_key = f"{field_base}_en"
    if localized_key in keys and obj[localized_key]:
        return obj[localized_key]
    if english_key in keys and obj[english_key]:
        return obj[english_key]
    return ""


def page_translation_is_public(page, lang):
    if lang == "en":
        return True
    key = f"translation_published_{lang}"
    status_key = f"translation_status_{lang}"
    keys = page.keys() if hasattr(page, "keys") else page
    return key in keys and status_key in keys and bool(page[key]) and page[status_key] in ("approved", "published")


def get_page_text(page, field_base, lang):
    if lang != "en" and not page_translation_is_public(page, lang):
        lang = "en"
    text = get_translated(page, field_base, lang)
    if text and "{COMPANY_NAME}" in text:
        try:
            text = text.replace("{COMPANY_NAME}", get_settings()["company_name"])
        except Exception:
            pass
    return text


def absolute_url_for(endpoint, **values):
    return url_for(endpoint, _external=True, **values)


def canonical_url():
    listing = getattr(g, "current_listing", None)
    if listing and current_listing_translation_missing():
        return url_for("listing_detail", lang="en", slug=listing["slug"], _external=True)
    return request.base_url


def hreflang_urls(endpoint=None, **values):
    endpoint = endpoint or request.endpoint
    links = []
    listing = getattr(g, "current_listing", None)
    for language in get_active_languages():
        code = language["code"]
        if listing and code != "en" and not listing_has_translation(listing, code):
            continue
        next_values = dict(request.view_args or {})
        next_values["lang"] = code
        next_values.update(values)
        try:
            links.append((code, url_for(endpoint, _external=True, **next_values)))
        except Exception:
            continue
    return links


def menu_interface_key(label="", url="", slug=""):
    value = (slug or "").strip().lower().replace("_", "-")
    if not value and url:
        clean_url = str(url).split("?")[0].rstrip("/")
        value = clean_url.rsplit("/", 1)[-1].lower().replace("_", "-")
    label_key = re.sub(r"[^a-z0-9]+", " ", (label or "").lower()).strip()
    url_value = (url or "").lower()
    if value == "home" or label_key == "home" or url_value.rstrip("/").endswith("/home"):
        return "nav_home"
    if value == "inventory" or label_key == "inventory" or "/inventory" in url_value:
        return "nav_inventory"
    if value in ("about-us", "about") or label_key == "about us":
        return "nav_about_us"
    if value in ("delivery", "export-and-delivery") or label_key in ("export delivery", "export and delivery"):
        return "nav_delivery"
    if value in ("shipping", "shipping-information") or label_key in ("shipping", "shipping information"):
        return "nav_shipping"
    if value == "contact" or label_key == "contact":
        return "nav_contact"
    footer_map = {
        "tax-information": "nav_tax",
        "our-location": "nav_location",
        "why-buy-from-us": "nav_why_buy",
        "warranty": "nav_warranty",
        "financing": "nav_financing",
        "faq": "nav_faq",
        "privacy-policy": "nav_privacy",
        "terms-conditions": "nav_terms",
    }
    if value in footer_map:
        return footer_map[value]
    return ""


def menu_items(location, lang):
    rows = get_db().execute(
        """
        SELECT * FROM menu_items
        WHERE location = ? AND is_active = 1
        ORDER BY sort_order, id
        """,
        (location,),
    ).fetchall()
    items = []
    seen_urls = set()
    for row in rows:
        url = (row["url"] or "#").replace("{lang}", lang)
        if url in seen_urls:
            continue
        seen_urls.add(url)
        english_label = get_translated(row, "label", "en")
        interface_key = menu_interface_key(english_label, url)
        label = ui_text(interface_key, lang) if interface_key else get_translated(row, "label", lang)
        items.append({"label": label, "url": url})
    page_field = "show_in_header" if location == "header" else "show_in_footer"
    if table_exists("site_pages") and page_field in table_columns("site_pages"):
        page_rows = get_db().execute(
            f"""
            SELECT * FROM site_pages
            WHERE is_published = 1
              AND status IN ('approved','published')
              AND {page_field} = 1
            ORDER BY menu_order, title_en
            """
        ).fetchall()
        for page in page_rows:
            url = url_for("home", lang=lang) if page["slug"] == "home" else url_for("public_page", lang=lang, page_slug=page["slug"])
            if url in seen_urls:
                continue
            seen_urls.add(url)
            interface_key = menu_interface_key(page["title_en"] or "", url, page["slug"] or "")
            label = ui_text(interface_key, lang) if interface_key else get_page_text(page, "title", lang)
            items.append({"label": label, "url": url})
    return items


def localized_url(lang):
    path = request.path.strip("/")
    parts = path.split("/") if path else []
    if parts and parts[0] in LANGUAGES:
        parts[0] = lang
        return "/" + "/".join(parts)
    return f"/{lang}/" + "/".join(parts) if parts else f"/{lang}/"


def current_user():
    user_id = session.get("user_id")
    if not user_id:
        return None
    return get_db().execute("SELECT * FROM users WHERE id = ?", (user_id,)).fetchone()


def encryption_ready():
    return bool(get_encryption_secret() and Fernet is not None)


def encryption_error_message():
    if Fernet is None:
        return "Install cryptography before saving an OpenAI API key."
    if not get_encryption_secret():
        return "Set OPENAI_SETTINGS_SECRET in the hosting environment, or allow the local instance secret file to be created."
    return ""


def get_encryption_secret():
    env_secret = os.environ.get("OPENAI_SETTINGS_SECRET", "").strip()
    if env_secret:
        return env_secret
    if LOCAL_OPENAI_SECRET_FILE.exists():
        return LOCAL_OPENAI_SECRET_FILE.read_text(encoding="utf-8").strip()
    if Fernet is None:
        return ""
    ensure_directories()
    secret = Fernet.generate_key().decode()
    LOCAL_OPENAI_SECRET_FILE.write_text(secret, encoding="utf-8")
    return secret


def get_fernet():
    secret = get_encryption_secret()
    if not secret or Fernet is None:
        return None
    try:
        return Fernet(secret.encode())
    except (ValueError, TypeError):
        try:
            return Fernet(base64.urlsafe_b64encode(secret.encode().ljust(32, b"0")[:32]))
        except (ValueError, TypeError):
            return None


def encrypt_secret(value):
    fernet = get_fernet()
    if not fernet:
        raise RuntimeError(encryption_error_message() or "OpenAI API key encryption is not configured.")
    return fernet.encrypt(value.encode()).decode()


def decrypt_secret(value):
    if not value:
        return ""
    fernet = get_fernet()
    if not fernet:
        return ""
    try:
        return fernet.decrypt(value.encode()).decode()
    except InvalidToken:
        return ""


def mask_secret(value):
    if not value:
        return ""
    if len(value) <= 10:
        return "*" * len(value)
    return f"{value[:6]}...{value[-4:]}"



def settings_value(settings, field_name, default=None):
    if not settings:
        return default
    try:
        if field_name in settings.keys():
            value = settings[field_name]
            return default if value is None else value
    except Exception:
        pass
    return default


def smtp_password(settings=None):
    settings = settings or get_settings()
    return os.environ.get("SMTP_PASSWORD", "").strip() or decrypt_secret(settings_value(settings, "smtp_password_encrypted", "") or "")


def smtp_status(settings=None):
    settings = settings or get_settings()
    password_saved = bool(settings_value(settings, "smtp_password_encrypted", "")) or bool(os.environ.get("SMTP_PASSWORD", "").strip())
    return {
        "enabled": bool(settings_value(settings, "smtp_enabled", 0)),
        "host": settings_value(settings, "smtp_host", ""),
        "port": settings_value(settings, "smtp_port", 587),
        "encryption": (settings_value(settings, "smtp_encryption", "tls") or "tls").lower(),
        "authentication": bool(settings_value(settings, "smtp_authentication", 1)),
        "username": settings_value(settings, "smtp_username", ""),
        "password_configured": password_saved,
        "from_email": settings_value(settings, "smtp_from_email", "") or settings_value(settings, "email", ""),
        "recipient_email": settings_value(settings, "smtp_recipient_email", "") or settings_value(settings, "email", ""),
        "encryption_ready": encryption_ready(),
        "secret_source": "SMTP_PASSWORD environment variable" if os.environ.get("SMTP_PASSWORD", "").strip() else ("encrypted database value" if password_saved else "not configured"),
    }


def smtp_send_message(settings, subject, text_body, html_body=None, reply_to=None, recipient=None):
    if not bool(settings_value(settings, "smtp_enabled", 0)):
        return False, "SMTP sending is disabled."
    host = (settings_value(settings, "smtp_host", "") or "").strip()
    if not host:
        return False, "SMTP Host is missing."
    try:
        port = int(settings_value(settings, "smtp_port", 587) or 587)
    except (TypeError, ValueError):
        port = 587
    encryption = ((settings_value(settings, "smtp_encryption", "tls") or "tls").strip().lower())
    if encryption not in ("none", "ssl", "tls"):
        encryption = "tls"
    username = (settings_value(settings, "smtp_username", "") or "").strip()
    password = smtp_password(settings)
    auth_enabled = bool(settings_value(settings, "smtp_authentication", 1))
    from_email = (settings_value(settings, "smtp_from_email", "") or settings_value(settings, "email", "") or username).strip()
    from_name = (settings_value(settings, "smtp_from_name", "") or settings_value(settings, "company_name", "") or "Website").strip()
    to_email = (recipient or settings_value(settings, "smtp_recipient_email", "") or settings_value(settings, "email", "")).strip()
    if not from_email:
        return False, "From Email is missing."
    if not to_email:
        return False, "Recipient Email is missing."
    msg = EmailMessage()
    msg["Subject"] = subject
    msg["From"] = f"{from_name} <{from_email}>" if from_name else from_email
    msg["To"] = to_email
    if reply_to:
        msg["Reply-To"] = reply_to
    msg.set_content(text_body or "")
    if html_body:
        msg.add_alternative(html_body, subtype="html")
    try:
        context = ssl.create_default_context()
        if encryption == "ssl":
            server = smtplib.SMTP_SSL(host, port, timeout=25, context=context)
        else:
            server = smtplib.SMTP(host, port, timeout=25)
        try:
            server.ehlo()
            if encryption == "tls":
                server.starttls(context=context)
                server.ehlo()
            if auth_enabled:
                if not username:
                    return False, "Authentication is enabled but SMTP Username is missing."
                server.login(username, password)
            server.send_message(msg)
        finally:
            try:
                server.quit()
            except Exception:
                server.close()
        return True, "Email sent successfully."
    except Exception as error:
        return False, str(error)


def lead_notification_payload(lead, listing=None):
    listing_title = listing["title_en"] if listing and "title_en" in listing.keys() else ""
    listing_ref = ""
    if listing:
        listing_ref = listing["public_stock_number"] or listing["stock_number"] or ""
    subject_bits = ["New website enquiry"]
    if listing_ref:
        subject_bits.append(listing_ref)
    if listing_title:
        subject_bits.append(listing_title[:70])
    subject = " - ".join(subject_bits)
    rows = [
        ("Name", lead["customer_name"] or ""),
        ("Email", lead["customer_email"] or ""),
        ("Phone", lead["customer_phone"] or ""),
        ("WhatsApp", lead["customer_whatsapp"] or ""),
        ("Language", lead["language_code"] or ""),
        ("Source page", lead["source_page"] or ""),
        ("Listing reference", listing_ref),
        ("Listing title", listing_title),
        ("Submitted", lead["created_at"] or ""),
        ("Message", lead["message"] or ""),
    ]
    text = "New website enquiry\n\n" + "\n".join(f"{label}: {value}" for label, value in rows if value)
    html_rows = "".join(
        f"<tr><th style='text-align:left;padding:6px 10px;border-bottom:1px solid #ddd'>{escape(label)}</th>"
        f"<td style='padding:6px 10px;border-bottom:1px solid #ddd'>{escape(str(value)).replace(chr(10), '<br>')}</td></tr>"
        for label, value in rows if value
    )
    html = f"<h2>New website enquiry</h2><table cellspacing='0' cellpadding='0' style='border-collapse:collapse'>{html_rows}</table>"
    reply_to = lead["customer_email"] if lead["customer_email"] else None
    return subject, text, html, reply_to


def send_lead_notification_email(lead_id):
    settings = get_settings()
    db = get_db()
    if not bool(settings_value(settings, "smtp_enabled", 0)):
        message = "SMTP disabled"
        try:
            db.execute(
                "UPDATE leads SET smtp_status = ?, smtp_message = ?, smtp_sent_at = NULL WHERE id = ?",
                ("disabled", message, lead_id),
            )
            db.commit()
        except Exception:
            pass
        return False, message

    lead = db.execute("SELECT * FROM leads WHERE id = ?", (lead_id,)).fetchone()
    if not lead:
        return False, "Lead not found"

    listing = None
    if lead["listing_id"]:
        listing = db.execute("SELECT * FROM listings WHERE id = ?", (lead["listing_id"],)).fetchone()

    subject, text, html, reply_to = lead_notification_payload(lead, listing)

    recipient = (
        settings_value(settings, "smtp_recipient_email", "")
        or settings_value(settings, "email", "")
        or settings_value(settings, "smtp_username", "")
    )

    ok, message = smtp_send_message(settings, subject, text, html, reply_to=reply_to, recipient=recipient)
    try:
        db.execute(
            "UPDATE leads SET smtp_status = ?, smtp_message = ?, smtp_sent_at = ? WHERE id = ?",
            ("sent" if ok else "failed", message[:1000], now_utc() if ok else None, lead_id),
        )
        db.commit()
    except Exception as error:
        app.logger.warning("Could not record SMTP status for lead %s: %s", lead_id, error)

    if not ok:
        app.logger.warning("SMTP lead notification failed for lead %s: %s", lead_id, message)
    return ok, message


def openai_key_status(settings=None):
    settings = settings or get_settings()
    encrypted = settings["openai_api_key_encrypted"] if "openai_api_key_encrypted" in settings.keys() else ""
    key = decrypt_secret(encrypted)
    env_secret = bool(os.environ.get("OPENAI_SETTINGS_SECRET", "").strip())
    return {
        "configured": bool(key),
        "masked": mask_secret(key),
        "encryption_ready": encryption_ready(),
        "package_available": OpenAI is not None or importlib.util.find_spec("openai") is not None,
        "secret_source": "environment" if env_secret else "local instance file",
        "message": encryption_error_message(),
    }


def decrypt_setting_key(settings, field_name):
    return decrypt_secret(settings[field_name] or "") if field_name in settings.keys() else ""


def fireworks_api_key(settings=None):
    settings = settings or get_settings()
    return os.environ.get("FIREWORKS_API_KEY", "").strip() or decrypt_setting_key(settings, "fireworks_api_key_encrypted")


def openai_content_api_key(settings=None):
    settings = settings or get_settings()
    return os.environ.get("OPENAI_API_KEY", "").strip() or decrypt_setting_key(settings, "openai_api_key_encrypted")


def zai_api_key(settings=None):
    settings = settings or get_settings()
    return os.environ.get("ZAI_API_KEY", "").strip() or decrypt_setting_key(settings, "zai_api_key_encrypted")


ZAI_ANTHROPIC_BASE = "https://api.z.ai/api/anthropic"
ZAI_OPENAI_CHAT_URL = "https://api.z.ai/api/paas/v4/chat/completions"
AI_FEATURES = ("chatbot", "lead_reply", "descriptions", "pages", "inspection")


def provider_for(settings, feature):
    """Which provider handles this feature? Per-feature override, else global, else fireworks."""
    if feature in AI_FEATURES and "ai_provider_" + feature in settings.keys():
        per = (settings["ai_provider_" + feature] or "").strip().lower()
        if per in ("fireworks", "openai", "zai"):
            return per
    provider = (settings["ai_provider"] or "").strip().lower()
    return provider if provider in ("fireworks", "openai", "zai") else "fireworks"


def provider_api_key(settings, provider):
    if provider == "openai":
        return openai_content_api_key(settings)
    if provider == "zai":
        return zai_api_key(settings)
    return fireworks_api_key(settings)


def ai_provider_status(settings=None):
    settings = settings or get_settings()
    provider = (settings["ai_provider"] or "fireworks").lower()
    fireworks_key = fireworks_api_key(settings)
    openai_key = openai_content_api_key(settings)
    zai_key = zai_api_key(settings)
    return {
        "enabled": bool(settings["ai_generation_enabled"]),
        "provider": provider,
        "fireworks_configured": bool(fireworks_key),
        "openai_configured": bool(openai_key),
        "zai_configured": bool(zai_key),
        "zai_model": settings["zai_model"] or "glm-5.2",
        "active_configured": bool(fireworks_key if provider == "fireworks" else (openai_key if provider == "openai" else zai_key)),
        "fireworks_model": settings["fireworks_custom_model"] or settings["fireworks_model"] or DEFAULT_FIREWORKS_MODEL,
        "openai_model": settings["openai_model"] or "gpt-5.2",
    }


def price_sort_expression():
    return "CAST(REPLACE(REPLACE(COALESCE(price, ''), '$', ''), ',', '') AS REAL)"


def listing_filter_options(published_only=True):
    base_where = "COALESCE(is_deleted, 0) = 0 AND "
    if published_only:
        base_where += "is_published = 1 AND "
    model_where = (
        f"{base_where}model IS NOT NULL AND model != '' "
        "AND model NOT GLOB 'ST[0-9][0-9][0-9][0-9][0-9]' "
        "AND model NOT GLOB '(ST[0-9][0-9][0-9][0-9][0-9])'"
    )
    db = get_db()
    return {
        "categories": db.execute(f"SELECT DISTINCT category FROM listings WHERE {base_where}category IS NOT NULL AND category != '' ORDER BY category").fetchall(),
        "makes": db.execute(f"SELECT make, COUNT(*) AS count FROM listings WHERE {base_where}make IS NOT NULL AND make != '' GROUP BY make ORDER BY make").fetchall(),
        "models": db.execute(f"SELECT DISTINCT model FROM listings WHERE {model_where} ORDER BY model").fetchall(),
        "years": db.execute(f"SELECT CAST(year AS INTEGER) AS year, COUNT(*) AS count FROM listings WHERE {base_where}year IS NOT NULL AND year != '' GROUP BY year ORDER BY year DESC").fetchall(),
        "statuses": db.execute(f"SELECT DISTINCT status FROM listings WHERE {base_where}status IS NOT NULL AND status != '' ORDER BY status").fetchall(),
    }


def build_listing_query(published_only=True):
    filters = {
        "q": request.args.get("q", "").strip(),
        "category": request.args.get("category", "").strip(),
        "make": request.args.get("make", "").strip(),
        "model": request.args.get("model", "").strip(),
        "year": request.args.get("year", "").strip(),
        "status": request.args.get("status", "").strip(),
        "price_min": request.args.get("price_min", "").strip(),
        "price_max": request.args.get("price_max", "").strip(),
        "sort": request.args.get("sort", "newest").strip() or "newest",
        "published": request.args.get("published", "").strip(),
        "missing_images": request.args.get("missing_images", "").strip(),
        "missing_category": request.args.get("missing_category", "").strip(),
        "missing_description": request.args.get("missing_description", "").strip(),
        "archived": request.args.get("archived", "").strip(),
        "body_style": request.args.get("body_style", "").strip(),
        "transmission": request.args.get("transmission", "").strip(),
        "fuel_type": request.args.get("fuel_type", "").strip(),
        "drive_line": request.args.get("drive_line", "").strip(),
        "year_from": request.args.get("year_from", "").strip(),
        "year_to": request.args.get("year_to", "").strip(),
        "odometer_min": request.args.get("odometer_min", "").strip(),
        "odometer_max": request.args.get("odometer_max", "").strip(),
        "certified": request.args.get("certified", "").strip(),
    }
    where = []
    params = []
    if published_only:
        where.append("is_published = 1")
        where.append("COALESCE(is_deleted, 0) = 0")
    elif filters["archived"] == "1":
        where.append("COALESCE(is_deleted, 0) = 1")
    elif filters["published"] in ("0", "1"):
        where.append("is_published = ?")
        params.append(int(filters["published"]))
        where.append("COALESCE(is_deleted, 0) = 0")
    else:
        where.append("COALESCE(is_deleted, 0) = 0")
    if filters["q"]:
        like = f"%{filters['q']}%"
        where.append(
            "(title_en LIKE ? OR stock_number LIKE ? OR make LIKE ? OR model LIKE ? OR category LIKE ? OR hours LIKE ?)"
        )
        params.extend([like, like, like, like, like, like])
    if filters["category"]:
        where.append("category = ?")
        params.append(filters["category"])
    if filters["make"]:
        where.append("LOWER(make) = LOWER(?)")
        params.append(filters["make"])
    if filters["model"]:
        where.append("LOWER(model) = LOWER(?)")
        params.append(filters["model"])
    if filters["year"]:
        where.append("year = ?")
        params.append(filters["year"])
    if filters["status"]:
        where.append("status = ?")
        params.append(filters["status"])
    price_expr = price_sort_expression()
    if filters["price_min"]:
        where.append(f"{price_expr} >= ?")
        params.append(filters["price_min"])
    if filters["price_max"]:
        where.append(f"{price_expr} <= ?")
        params.append(filters["price_max"])
    if filters["missing_category"]:
        where.append("(category IS NULL OR category = '' OR category = 'Other')")
    if filters["missing_description"]:
        where.append("(description_html_en IS NULL OR description_html_en = '')")
    if filters["missing_images"]:
        where.append("id NOT IN (SELECT DISTINCT listing_id FROM listing_images)")
    if filters["body_style"]:
        where.append("LOWER(body_style) = LOWER(?)")
        params.append(filters["body_style"])
    if filters["transmission"]:
        where.append("LOWER(transmission) = LOWER(?)")
        params.append(filters["transmission"])
    if filters["fuel_type"]:
        where.append("LOWER(fuel_type) = LOWER(?)")
        params.append(filters["fuel_type"])
    if filters["drive_line"]:
        where.append("LOWER(drive_line) = LOWER(?)")
        params.append(filters["drive_line"])
    if filters["year_from"]:
        try:
            yf = int(filters["year_from"])
        except (ValueError, TypeError):
            yf = None
        if yf is not None:
            where.append("CAST(year AS INTEGER) >= ?")
            params.append(yf)
    if filters["year_to"]:
        try:
            yt = int(filters["year_to"])
        except (ValueError, TypeError):
            yt = None
        if yt is not None:
            where.append("CAST(year AS INTEGER) <= ?")
            params.append(yt)
    if filters["odometer_min"]:
        try:
            omi = int(filters["odometer_min"])
        except (ValueError, TypeError):
            omi = None
        if omi is not None:
            where.append("CAST(odometer AS INTEGER) >= ?")
            params.append(omi)
    if filters["odometer_max"]:
        try:
            oma = int(filters["odometer_max"])
        except (ValueError, TypeError):
            oma = None
        if oma is not None:
            where.append("CAST(odometer AS INTEGER) <= ?")
            params.append(oma)
    if filters["certified"]:
        where.append("certified = 1")

    sort_sql = {
        "price_low": f"{price_expr} ASC, is_featured DESC, updated_at DESC",
        "price_high": f"{price_expr} DESC, is_featured DESC, updated_at DESC",
        "year_newest": "CAST(COALESCE(year, '0') AS INTEGER) DESC, is_featured DESC, updated_at DESC",
        "newest": "is_featured DESC, updated_at DESC, id DESC",
    }.get(filters["sort"], "is_featured DESC, updated_at DESC, id DESC")
    where_sql = " WHERE " + " AND ".join(where) if where else ""
    return f"SELECT * FROM listings{where_sql} ORDER BY {sort_sql}", params, filters


def remove_filter_param(name):
    """Return a copy of request.args with `name` removed, for filter-chip close links."""
    result = {}
    for key, values in request.args.lists():
        if key != name:
            if len(values) == 1:
                result[key] = values[0]
            else:
                result[key] = values
    return result



def interface_translation_has_content(row, lang):
    return bool(row and lang in row.keys() and (row[lang] or "").strip())


def ai_interface_translation_prompt(key, english_text, target_language):
    return f"""
Translate one website interface string from English to {target_language}.
Return strict JSON only with key: translation.
Rules:
- Use natural, concise UI language suitable for a professional vehicles dealership.
- Preserve placeholders exactly, including braces such as {{title}}, {{reference}}, and {{value}}.
- Preserve company names, model names, CA references, numbers, URLs and abbreviations.
- Do not add punctuation or marketing claims that are not present in English.

Interface key: {key}
English source: {english_text}
"""


def translate_interface_with_ai(key, lang, force=False):
    if lang == "en" or not is_valid_lang(lang):
        raise RuntimeError("Choose a supported non-English language.")
    db = get_db()
    row = db.execute("SELECT * FROM interface_translations WHERE translation_key = ?", (key,)).fetchone()
    if not row:
        raise RuntimeError("Interface string not found.")
    if not force and interface_translation_has_content(row, lang):
        return "skipped"
    english = (row["en"] or "").strip()
    if not english:
        raise RuntimeError("English source is empty.")
    settings = get_settings()
    prompt = ai_interface_translation_prompt(key, english, LANGUAGE_NAMES.get(lang, lang))
    data, raw_response, model, provider, _attempts_used = request_translation_json(prompt, settings, max_tokens=1400, attempts=3)
    translated = (data.get("translation") or "").strip()
    if not translated:
        raise RuntimeError("Translation model returned empty text.")
    db.execute(f"UPDATE interface_translations SET {lang} = ?, updated_at = ? WHERE translation_key = ?", (translated, now_utc(), key))
    db.execute("""
        INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
        VALUES (NULL, ?, ?, ?, ?, 'success', '', ?)
    """, (provider, model, prompt, raw_response, now_utc()))
    return "translated"


def ui_text(key, lang="en", **kwargs):
    lang = lang if lang in LANGUAGES else "en"
    value = ""
    try:
        row = get_db().execute("SELECT * FROM interface_translations WHERE translation_key = ?", (key,)).fetchone()
        if row:
            value = row[lang] or row["en"] or ""
    except Exception:
        pass
    if not value:
        values = INTERFACE_TRANSLATION_DEFAULTS.get(key, {})
        value = values.get(lang) or values.get("en") or key.replace("_", " ").title()
    try:
        result = value.format(**kwargs)
    except Exception:
        result = value
    if "{COMPANY_NAME}" in result:
        try:
            result = result.replace("{COMPANY_NAME}", get_settings()["company_name"])
        except Exception:
            pass
    return result


def display_status(value, lang="en"):
    key = "status_" + slugify(value or "available").replace("-", "_")
    if key in INTERFACE_TRANSLATION_DEFAULTS:
        return ui_text(key, lang)
    if (value or "").lower() in ("available", "reserved", "sold"):
        return ui_text((value or "available").lower(), lang)
    return value or ui_text("available", lang)


def display_category(value, lang="en"):
    if not value:
        return ui_text("vehicles", lang)
    key = "category_" + slugify(value).replace("-", "_")
    return ui_text(key, lang) if key in INTERFACE_TRANSLATION_DEFAULTS else value


def public_price_label(listing, lang="en"):
    price = (listing["price"] or "").strip() if listing else ""
    formatted = format_price_for_language(price, lang, get_settings()) if price else ""
    if formatted and any(ch.isdigit() for ch in formatted):
        return ""
    label = (listing["price_label"] or "").strip() if listing else ""
    return label or ui_text("price_on_request", lang)

def public_price_display(listing, lang="en"):
    if not listing:
        return ui_text("price_on_request", lang)
    amount = parse_price_amount(listing["price"] or "")
    if amount is not None:
        return format_price_for_language(listing["price"], lang, get_settings())
    label = (listing["price_label"] or "").strip()
    return label or ui_text("price_on_request", lang)

PROMO_THEME_PRESETS = ("amber", "red", "midnight", "emerald", "ocean", "carbon", "white")


def promo_countdown_label(end_date_value):
    """Return a short urgency label for the promo banner, or "" when not applicable.

    end_date_value is a plain YYYY-MM-DD string. Days remaining is computed
    against the local calendar date (not UTC) so the label matches what the
    visitor expects mid-evening.
    """
    raw = (end_date_value or "").strip()
    if not raw:
        return ""
    try:
        end_date = datetime.strptime(raw[:10], "%Y-%m-%d").date()
    except ValueError:
        return ""
    days = (end_date - datetime.now().date()).days
    if days < 0:
        return ""
    if days == 0:
        return "Ends today"
    if days == 1:
        return "Ends tomorrow"
    return f"Ends in {days} days"


@app.context_processor
def inject_public_context():
    lang = current_lang() if request.endpoint else "en"
    settings = get_settings()
    promo_enabled = bool(int(settings_value(settings, "free_delivery_promo_enabled", 0)))
    raw_promo_text = settings_value(settings, "free_delivery_promo_text", "Free Canada-wide delivery this {month}") or "Free Canada-wide delivery this {month}"
    promo_text = raw_promo_text.replace("{month}", datetime.now().strftime("%B"))
    promo_placement = settings_value(settings, "free_delivery_promo_placement", "both") or "both"
    try:
        promo_duration = int(settings_value(settings, "free_delivery_promo_duration", 6) or 0)
    except (TypeError, ValueError):
        promo_duration = 6
    promo_duration = max(0, min(120, promo_duration))
    promo_theme = (settings_value(settings, "free_delivery_promo_theme", "amber") or "amber").strip().lower()
    if promo_theme not in PROMO_THEME_PRESETS:
        promo_theme = "amber"
    promo_countdown_text = ""
    if int(settings_value(settings, "free_delivery_promo_anim_countdown", 0) or 0):
        promo_countdown_text = promo_countdown_label(settings_value(settings, "free_delivery_promo_end_date", "") or "")
    return {
        "settings": settings,
        "active_languages": get_active_languages(),
        "promo_enabled": promo_enabled,
        "promo_text": promo_text,
        "promo_placement": promo_placement,
        "promo_duration": promo_duration,
        "promo_theme": promo_theme,
        "promo_countdown_text": promo_countdown_text,
        "promo_anim_entrance": bool(int(settings_value(settings, "free_delivery_promo_anim_entrance", 1) or 0)),
        "promo_anim_shimmer": bool(int(settings_value(settings, "free_delivery_promo_anim_shimmer", 1) or 0)),
        "promo_anim_pulse": bool(int(settings_value(settings, "free_delivery_promo_anim_pulse", 1) or 0)),
        "promo_anim_slide_text": bool(int(settings_value(settings, "free_delivery_promo_anim_slide_text", 0) or 0)),
        "current_lang": lang,
        "header_menu": menu_items("header", lang),
        "footer_menu": menu_items("footer", lang),
        "get_translated": get_translated,
        "localized_url": localized_url,
        "admin_user": current_user(),
        "admin_url_prefix": ADMIN_URL_PREFIX,
        "get_listing_primary_image": get_listing_primary_image,
        "get_image_display_path": get_image_display_path,
        "get_original_image_display_path": get_original_image_display_path,
        "format_price_for_language": format_price_for_language,
        "at_listing_url": turbo_listing.at_listing_url,
        "language_display_code": language_display_code,
        "t": ui_text,
        "display_status": display_status,
        "display_category": display_category,
        "public_price_label": public_price_label,
        "public_price_display": public_price_display,
        "public_reference_for_listing": public_reference_for_listing,
        "public_title_for_listing": public_title_for_listing,
        "public_text_for_listing": public_text_for_listing,
        "get_page_text": get_page_text,
        "canonical_url": canonical_url,
        "hreflang_urls": hreflang_urls,
        "listing_translation_missing": current_listing_translation_missing,
        "chatbot_public_enabled": chatbot_is_publicly_enabled(settings) if "chatbot_enabled" in settings.keys() else False,
        "listing_whatsapp_url": listing_whatsapp_url,
        "listing_email_url": listing_email_url,
        "remove_param": remove_filter_param,
    }


BOT_PATTERNS = (
    "bot", "crawl", "crawler", "spider", "scrape", "scan", "headless",
    "puppeteer", "selenium", "phantom", "curl", "wget", "python-requests",
    "python-httpx", "go-http", "java/", "okhttp", "ahrefs", "semrush",
    "majestic", "moz.com", "rogerbot", "googlebot", "bingbot", "slurp",
    "duckduckbot", "baiduspider", "yandexbot", "facebookexternalhit",
    "twitterbot", "linkedinbot", "discordbot", "whatsapp", "bingpreview",
)

ANALYTICS_EXCLUDED_PREFIXES = (ADMIN_URL_PREFIX, "/static/", "/media/", "/analytics/")
ANALYTICS_EXCLUDED_PATHS = {"/favicon.ico", "/robots.txt", "/sitemap.xml"}
ANALYTICS_ASSET_EXTENSIONS = {
    ".webp", ".jpg", ".jpeg", ".png", ".gif", ".svg", ".ico",
    ".css", ".js", ".map", ".woff", ".woff2", ".ttf", ".xml", ".txt",
}


def is_bot_user_agent(user_agent):
    lower = (user_agent or "").strip().lower()
    if not lower:
        return True, "empty-user-agent"
    for pattern in BOT_PATTERNS:
        if pattern in lower:
            return True, pattern
    return False, ""


def detect_bot(user_agent):
    return is_bot_user_agent(user_agent)


def detect_device(user_agent, is_bot=False):
    if is_bot:
        return "bot"
    lower = (user_agent or "").lower()
    if "ipad" in lower or "tablet" in lower:
        return "tablet"
    if "mobile" in lower or "iphone" in lower or "android" in lower:
        return "mobile"
    return "desktop"


def _valid_ip(value):
    try:
        return str(ipaddress.ip_address((value or "").strip()))
    except (ValueError, TypeError):
        return ""


def _private_or_proxy_ip(value):
    try:
        ip = ipaddress.ip_address(value)
        return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
    except ValueError:
        return True


def get_real_ip():
    """Resolve visitor IP only from proxy headers when the immediate peer is trusted/local."""
    remote = _valid_ip(request.remote_addr or "")
    trust_headers = os.environ.get("TRUST_PROXY_HEADERS", "").lower() in ("1", "true", "yes")
    proxy_peer = bool(remote and _private_or_proxy_ip(remote))
    if trust_headers or proxy_peer:
        for header in ("CF-Connecting-IP", "True-Client-IP"):
            candidate = _valid_ip(request.headers.get(header, ""))
            if candidate:
                return candidate, header
        forwarded = request.headers.get("X-Forwarded-For", "")
        for raw in forwarded.split(","):
            candidate = _valid_ip(raw)
            if candidate and not _private_or_proxy_ip(candidate):
                return candidate, "X-Forwarded-For"
        candidate = _valid_ip(request.headers.get("X-Real-IP", ""))
        if candidate:
            return candidate, "X-Real-IP"
    return remote, "remote_addr"


def masked_ip(ip_value):
    try:
        ip = ipaddress.ip_address(ip_value or "")
        if ip.version == 4:
            parts = str(ip).split(".")
            return ".".join(parts[:2] + ["***", "***"])
        exploded = ip.exploded.split(":")
        return ":".join(exploded[:3] + ["****"] * 5)
    except (ValueError, TypeError):
        return ""


def _host_from_url(value):
    try:
        return (urlparse(value or "").hostname or "").lower().strip(".")
    except Exception:
        return ""


def _normalized_request_host(value):
    host = (value or "").split(",", 1)[0].strip().lower()
    if ":" in host and not host.startswith("["):
        host = host.split(":", 1)[0]
    return host.strip(".")


def _csv_env_set(name, defaults=()):
    values = set(defaults)
    for item in os.environ.get(name, "").split(","):
        item = item.strip().lower()
        if item:
            values.add(item)
    return values


def _active_security_rule_values(rule_type):
    values = set()
    try:
        rows = get_db().execute(
            """SELECT rule_value FROM security_rules
               WHERE rule_type=? AND is_enabled=1
                 AND (expires_at IS NULL OR expires_at='' OR expires_at > ?)""",
            (rule_type, now_utc()),
        ).fetchall()
        values.update((row["rule_value"] or "").strip().lower() for row in rows if row["rule_value"])
    except Exception:
        pass
    return values


def approved_public_hosts():
    return _csv_env_set("APPROVED_PUBLIC_HOSTS", ("canadaautosales.ca", "www.canadaautosales.ca")) | _active_security_rule_values("approved_host")


def blocked_referrer_domains():
    return _csv_env_set("BLOCKED_REFERRER_DOMAINS", ("paginademanuseamento.com", "www.paginademanuseamento.com")) | _active_security_rule_values("blocked_domain")


def blocked_proxy_ips():
    return _csv_env_set("BLOCKED_PROXY_IPS", ("216.252.233.168",)) | _active_security_rule_values("blocked_ip")


def domain_matches_rule(host, rule):
    host = (host or "").lower().strip(".")
    rule = (rule or "").lower().strip(".")
    return bool(host and rule and (host == rule or host.endswith("." + rule)))


def ip_matches_rule(ip_value, rule):
    try:
        ip_obj = ipaddress.ip_address(ip_value or "")
        rule = (rule or "").strip()
        if "/" in rule:
            return ip_obj in ipaddress.ip_network(rule, strict=False)
        return ip_obj == ipaddress.ip_address(rule)
    except ValueError:
        return False


def matching_security_rule(ip_value="", *hosts):
    for rule in blocked_proxy_ips():
        if ip_matches_rule(ip_value, rule):
            return "blocked_ip", rule
    for host in hosts:
        for rule in blocked_referrer_domains():
            if domain_matches_rule(host, rule):
                return "blocked_domain", rule
    return "", ""


def touch_security_rule(rule_type, rule_value):
    if not rule_type or not rule_value:
        return
    try:
        get_db().execute(
            "UPDATE security_rules SET hit_count=COALESCE(hit_count,0)+1, last_hit_at=?, updated_at=? WHERE rule_type=? AND lower(rule_value)=lower(?)",
            (now_utc(), now_utc(), rule_type, rule_value),
        )
    except Exception:
        pass


def log_security_event(event_type, reason):
    try:
        ip_value, _source = get_real_ip()
        get_db().execute(
            """
            INSERT INTO security_events (
                event_type, reason, request_host, forwarded_host, origin, referrer,
                source_ip_masked, source_ip_full, source_ip_hash, endpoint, path, method, user_agent, created_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            (
                event_type, reason[:500], request.host[:250], request.headers.get("X-Forwarded-Host", "")[:250],
                request.headers.get("Origin", "")[:500], request.referrer or "", masked_ip(ip_value), ip_value,
                hashed_ip(ip_value), request.endpoint or "", request.path[:500], request.method,
                request.headers.get("User-Agent", "")[:500], now_utc(),
            ),
        )
        get_db().commit()
    except Exception as error:
        app.logger.warning("Security event logging failed: %s", error)
        try:
            get_db().rollback()
        except Exception:
            pass


def request_security_violation():
    if app.testing:
        return ""
    request_host = _normalized_request_host(request.host)
    forwarded_host = _normalized_request_host(request.headers.get("X-Forwarded-Host", ""))
    origin_host = _host_from_url(request.headers.get("Origin", ""))
    referrer_host = _host_from_url(request.referrer or "")
    ip_value, _source = get_real_ip()

    rule_type, rule_value = matching_security_rule(ip_value, forwarded_host, origin_host, referrer_host)
    if rule_type:
        touch_security_rule(rule_type, rule_value)
        if rule_type == "blocked_ip":
            return f"blocked source IP {ip_value}"
        return f"blocked domain {rule_value}"
    if request_host and request_host not in approved_public_hosts() and request_host not in ("localhost", "127.0.0.1"):
        return f"unauthorized host {request_host}"
    if request.method in ("POST", "PUT", "PATCH", "DELETE"):
        if origin_host and origin_host not in approved_public_hosts():
            return f"unapproved origin {origin_host}"
        if origin_host and origin_host not in approved_public_hosts():
            return f"unapproved origin {origin_host}"
    return ""


def hashed_ip(ip_value=None):
    value = ip_value
    if value is None:
        value, _source = get_real_ip()
    if not value:
        return ""
    salt = app.config["SECRET_KEY"] or "local"
    return hashlib.sha256(f"{salt}:{value}".encode("utf-8")).hexdigest()


def detect_country_code(ip_value=""):
    for header in ("CF-IPCountry", "X-Country-Code", "CloudFront-Viewer-Country", "X-AppEngine-Country"):
        value = (request.headers.get(header) or "").strip().upper()
        if re.fullmatch(r"[A-Z]{2}", value) and value not in ("XX", "T1"):
            return value, header
    geoip_path = os.environ.get("GEOIP_DATABASE_PATH", "").strip()
    if geoip_path and ip_value and Path(geoip_path).exists():
        try:
            import geoip2.database
            with geoip2.database.Reader(geoip_path) as reader:
                code = (reader.country(ip_value).country.iso_code or "").upper()
                if code:
                    return code, "GeoLite2"
        except Exception:
            pass
    return "Unknown", "none"


def analytics_request_is_trackable():
    if request.method != "GET" or request.path in ANALYTICS_EXCLUDED_PATHS:
        return False
    if request.path.startswith(ANALYTICS_EXCLUDED_PREFIXES):
        return False
    if Path(request.path.split("?", 1)[0]).suffix.lower() in ANALYTICS_ASSET_EXTENSIONS:
        return False
    return True


def analytics_session_row():
    token = session.get("analytics_session_token", "")
    if not token:
        return None
    return get_db().execute("SELECT * FROM analytics_sessions WHERE session_token = ?", (token,)).fetchone()


def ensure_analytics_session():
    settings = get_settings()
    if "analytics_enabled" in settings.keys() and not settings["analytics_enabled"]:
        return None
    user_agent = request.headers.get("User-Agent", "")[:500]
    is_bot, bot_reason = is_bot_user_agent(user_agent)
    ip_value, ip_source = get_real_ip()
    country, country_source = detect_country_code(ip_value)
    visitor_id = session.get("analytics_visitor_id")
    if not visitor_id:
        visitor_id = str(uuid.uuid4())
        session["analytics_visitor_id"] = visitor_id
    token = session.get("analytics_session_token")
    last_seen = session.get("analytics_last_seen_at")
    expired = False
    if last_seen:
        try:
            expired = datetime.utcnow() - datetime.fromisoformat(last_seen) > timedelta(minutes=30)
        except ValueError:
            expired = True
    if not token or expired:
        token = str(uuid.uuid4())
        session["analytics_session_token"] = token
        visit_number = get_db().execute(
            "SELECT COUNT(*) FROM analytics_sessions WHERE visitor_id = ?", (visitor_id,)
        ).fetchone()[0] + 1
        get_db().execute(
            """
            INSERT INTO analytics_sessions (
                session_token, visitor_id, started_at, last_seen_at, landing_page,
                first_referrer, last_referrer, country_code, language_code, device_type,
                is_bot, bot_reason, visit_number, user_agent, ip_hash, ip_masked, ip_full, ip_source
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """,
            (token, visitor_id, now_utc(), now_utc(), request.path, request.referrer or "direct",
             request.referrer or "direct", country, current_lang() if request.endpoint else "",
             detect_device(user_agent, is_bot), 1 if is_bot else 0,
             f"{bot_reason};ip={ip_source};country={country_source}" if is_bot else "",
             visit_number, user_agent, hashed_ip(ip_value), masked_ip(ip_value), ip_value, ip_source),
        )
        get_db().commit()
    session["analytics_last_seen_at"] = datetime.utcnow().isoformat()
    g.analytics_is_bot = is_bot
    g.analytics_bot_reason = bot_reason
    g.analytics_country_code = country
    g.analytics_real_ip_source = ip_source
    return token


@app.before_request
def enforce_public_host_security():
    if request.path.startswith(("/static/", "/media/")):
        return None
    reason = request_security_violation()
    if reason:
        log_security_event("blocked_request", reason)
        return ("Forbidden", 403)
    return None


@app.before_request
def initialize_analytics_visit():
    if request.path.startswith(ADMIN_URL_PREFIX) or request.path.startswith(("/static/", "/media/")):
        return None
    try:
        ensure_analytics_session()
    except Exception as error:
        app.logger.warning("Analytics session initialization failed: %s", error)
        get_db().rollback()
    return None


def record_analytics_event(event_type, event_value="", data=None, engaged_seconds=0, scroll_depth=0):
    token = session.get("analytics_session_token", "")
    visitor_id = session.get("analytics_visitor_id", "")
    if not token or not visitor_id:
        return False
    row = analytics_session_row()
    if not row:
        return False
    listing_id = getattr(g, "analytics_listing_id", None)
    if data and not listing_id:
        try:
            listing_id = int(data.get("listing_id") or 0) or None
        except (TypeError, ValueError):
            listing_id = None
    path = (data or {}).get("path") or request.path
    endpoint = request.endpoint or ""
    get_db().execute(
        """
        INSERT INTO analytics_events (
            session_token, visitor_id, event_type, path, endpoint, listing_id,
            language_code, country_code, referrer, event_value, event_data_json,
            engaged_seconds, scroll_depth, is_bot, bot_reason, device_type,
            user_agent, ip_hash, ip_masked, ip_full, ip_source, created_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (token, visitor_id, event_type, path[:500], endpoint[:150], listing_id,
         getattr(g, "analytics_language_code", current_lang() if request.endpoint else ""),
         row["country_code"] or "Unknown", request.referrer or "", str(event_value or "")[:500],
         json.dumps(data or {}, ensure_ascii=False)[:4000], int(engaged_seconds or 0),
         max(0, min(100, int(scroll_depth or 0))), row["is_bot"], row["bot_reason"] or "",
         row["device_type"] or "", request.headers.get("User-Agent", "")[:500], row["ip_hash"] or "",
         row["ip_masked"] if "ip_masked" in row.keys() else "", row["ip_full"] if "ip_full" in row.keys() else "", row["ip_source"] if "ip_source" in row.keys() else "", now_utc()),
    )
    return True


@app.after_request
def track_page_view(response):
    if response.status_code >= 300 or not analytics_request_is_trackable():
        return response
    try:
        row = analytics_session_row()
        if not row:
            return response
        listing_id = getattr(g, "analytics_listing_id", None)
        record_analytics_event("page_view", data={"url": request.url, "title": ""})
        get_db().execute(
            """
            UPDATE analytics_sessions SET last_seen_at = ?, last_referrer = ?, language_code = ?,
                pageview_count = pageview_count + 1,
                first_listing_id = COALESCE(first_listing_id, ?),
                last_listing_id = COALESCE(?, last_listing_id)
            WHERE session_token = ?
            """,
            (now_utc(), request.referrer or row["last_referrer"] or "", getattr(g, "analytics_language_code", ""),
             listing_id, listing_id, row["session_token"]),
        )
        # Keep the legacy table as clean raw page-view history for compatibility.
        get_db().execute(
            """INSERT INTO page_views (path, full_url, referrer, user_agent, ip_hash, country,
               device_type, is_bot, bot_name, listing_id, language_code, created_at)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (request.path, request.url, request.referrer or "", request.headers.get("User-Agent", "")[:500],
             row["ip_hash"], row["country_code"], row["device_type"], row["is_bot"], row["bot_reason"],
             listing_id, getattr(g, "analytics_language_code", ""), now_utc()),
        )
        get_db().commit()
    except Exception as error:
        app.logger.warning("Analytics page-view logging failed: %s", error)
        get_db().rollback()
    return response


@app.route("/analytics/event", methods=("POST",))
def analytics_event():
    try:
        ensure_analytics_session()
        payload = request.get_json(silent=True) or {}
        event_type = re.sub(r"[^a-z0-9_\-]", "", str(payload.get("event_type") or "" ).lower())[:60]
        allowed = {"engagement", "scroll", "whatsapp_click", "phone_click", "email_click", "form_start", "chatbot_open", "chatbot_message", "cta_click"}
        if event_type not in allowed:
            return jsonify({"ok": False}), 400
        seconds = max(0, min(300, int(payload.get("engaged_seconds") or 0)))
        depth = max(0, min(100, int(payload.get("scroll_depth") or 0)))
        record_analytics_event(event_type, payload.get("value", ""), payload, seconds, depth)
        increments = {
            "engagement": ("engaged_seconds", seconds),
            "scroll": ("max_scroll_depth", depth),
            "whatsapp_click": ("whatsapp_clicks", 1),
            "phone_click": ("phone_clicks", 1),
            "email_click": ("email_clicks", 1),
            "form_start": ("form_started", 1),
            "chatbot_open": ("chatbot_opened", 1),
            "chatbot_message": ("chatbot_messages", 1),
        }
        if event_type in increments:
            column, value = increments[event_type]
            if column == "max_scroll_depth":
                get_db().execute("UPDATE analytics_sessions SET max_scroll_depth = MAX(max_scroll_depth, ?), last_seen_at = ? WHERE session_token = ?", (value, now_utc(), session.get("analytics_session_token")))
            else:
                get_db().execute(f"UPDATE analytics_sessions SET {column} = {column} + ?, last_seen_at = ? WHERE session_token = ?", (value, now_utc(), session.get("analytics_session_token")))
        get_db().commit()
        return jsonify({"ok": True})
    except Exception as error:
        app.logger.warning("Analytics client event failed: %s", error)
        get_db().rollback()
        return jsonify({"ok": False}), 202


@app.before_request
def apply_redirects():
    if request.method != "GET" or request.path.startswith((ADMIN_URL_PREFIX, "/static/", "/media/")):
        return None
    row = get_db().execute("SELECT * FROM redirects WHERE old_path = ?", (request.path,)).fetchone()
    if row and row["new_path"]:
        return redirect(row["new_path"], code=row["status_code"] or 301)
    return None


def login_required(view):
    @wraps(view)
    def wrapped_view(**kwargs):
        if not session.get("user_id"):
            flash("Please sign in to continue.", "warning")
            return redirect(url_for("admin_login", next=request.path))
        return view(**kwargs)

    return wrapped_view


@app.route(f"{ADMIN_URL_PREFIX}/analytics/diagnostics")
@login_required
def admin_analytics_diagnostics():
    ip_value, ip_source = get_real_ip()
    country, country_source = detect_country_code(ip_value)
    is_bot, bot_reason = is_bot_user_agent(request.headers.get("User-Agent", ""))
    return jsonify({
        "real_ip": ip_value,
        "ip_source": ip_source,
        "country": country,
        "country_source": country_source,
        "user_agent": request.headers.get("User-Agent", ""),
        "is_bot": is_bot,
        "bot_reason": bot_reason,
        "trusted_proxy_headers": os.environ.get("TRUST_PROXY_HEADERS", ""),
    })



def save_site_upload(file_storage, prefix):
    if not file_storage or not file_storage.filename:
        return None
    filename = secure_filename(file_storage.filename)
    if not filename:
        return None
    timestamp = datetime.utcnow().strftime("%Y%m%d%H%M%S")
    final_name = f"{prefix}-{timestamp}-{filename}"
    target = SITE_UPLOADS_DIR / final_name
    file_storage.save(target)
    return f"/static/uploads/site/{final_name}"


def save_listing_gallery_upload(listing, file_storage, sort_order):
    if not file_storage or not file_storage.filename:
        return None, "No file selected."
    original_name = secure_filename(file_storage.filename)
    if not original_name:
        return None, "Image filename was not usable."
    if Path(original_name).suffix.lower() not in IMAGE_EXTENSIONS:
        return None, f"{original_name} is not a supported image file."
    folder_name = secure_filename(listing["stock_number"] or f"listing-{listing['id']}") or f"listing-{listing['id']}"
    target_dir = LISTING_UPLOADS_DIR / folder_name
    target_dir.mkdir(parents=True, exist_ok=True)
    filename = seo_image_filename_for_listing(listing, sort_order, original_name)
    target = unique_file_target(target_dir, filename)
    file_storage.save(target)
    file_size = target.stat().st_size
    checksum = hashlib.sha256(target.read_bytes()).hexdigest()
    local_path = "/" + target.resolve().relative_to(BASE_DIR.resolve()).as_posix()
    return {
        "local_path": local_path,
        "file_size": file_size,
        "checksum_sha256": checksum,
        "sort_order": sort_order,
    }, ""


def normalize_local_redirect_path(value):
    path = (value or "").strip()
    if not path:
        return ""
    if path.startswith(("http://", "https://", "//")):
        return ""
    if not path.startswith("/"):
        path = "/" + path
    return path


def image_dimensions_from_bytes(data):
    if data.startswith(b"\x89PNG\r\n\x1a\n") and len(data) >= 24:
        return int.from_bytes(data[16:20], "big"), int.from_bytes(data[20:24], "big")
    if data.startswith(b"\xff\xd8"):
        index = 2
        while index + 9 < len(data):
            if data[index] != 0xFF:
                index += 1
                continue
            marker = data[index + 1]
            index += 2
            if marker in (0xD8, 0xD9):
                continue
            if index + 2 > len(data):
                break
            segment_length = int.from_bytes(data[index : index + 2], "big")
            if marker in range(0xC0, 0xC4) and index + 7 < len(data):
                return int.from_bytes(data[index + 5 : index + 7], "big"), int.from_bytes(data[index + 3 : index + 5], "big")
            index += max(segment_length, 2)
    return None, None


def save_generated_visual_result(page, placement_key, image_base64, prompt, provider, model):
    target_dir = GENERATED_VISUALS_UPLOADS_DIR / secure_filename(page["slug"] or f"page-{page['id']}")
    target_dir.mkdir(parents=True, exist_ok=True)
    image_bytes = base64.b64decode(image_base64)
    filename = f"{secure_filename(page['slug'] or 'page')}-{secure_filename(placement_key or 'hero')}-{uuid.uuid4().hex[:8]}.png"
    target = target_dir / filename
    with open(target, "wb") as output_file:
        output_file.write(image_bytes)
    local_path = "/" + target.resolve().relative_to(BASE_DIR.resolve()).as_posix()
    width, height = image_dimensions_from_bytes(image_bytes)
    timestamp = now_utc()
    db = get_db()
    db.execute(
        "UPDATE generated_visuals SET is_active = 0, updated_at = ? WHERE page_id = ? AND placement_key = ?",
        (timestamp, page["id"], placement_key),
    )
    db.execute(
        """
        INSERT INTO generated_visuals (
            page_id, listing_id, placement_key, prompt, provider, model,
            local_path, width, height, is_active, created_at, updated_at
        )
        VALUES (?, NULL, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?)
        """,
        (page["id"], placement_key, prompt, provider, model, local_path, width, height, timestamp, timestamp),
    )
    db.execute(
        "UPDATE site_pages SET hero_image_path = ?, updated_at = ? WHERE id = ?",
        (local_path, timestamp, page["id"]),
    )
    db.commit()
    return local_path


def generate_page_visual(page_id, placement_key="hero", model="gpt-image-2"):
    page = get_db().execute("SELECT * FROM site_pages WHERE id = ?", (page_id,)).fetchone()
    if not page:
        raise RuntimeError("Page not found.")
    prompt = (page["visual_prompt_en"] or "").strip()
    if not prompt:
        raise RuntimeError("Add a visual prompt before generating an image.")
    client = get_openai_client()
    image_model = (model or "gpt-image-2").strip() or "gpt-image-2"
    response = client.images.generate(
        model=image_model,
        prompt=prompt,
        size="1536x1024",
        quality="high",
        n=1,
    )
    image_base64 = getattr(response.data[0], "b64_json", "") if getattr(response, "data", None) else ""
    if not image_base64:
        raise RuntimeError("OpenAI did not return a generated image.")
    return save_generated_visual_result(page, placement_key, image_base64, prompt, "openai", image_model)


def slugify(value):
    cleaned = []
    last_dash = False
    for char in value.lower():
        if char.isalnum():
            cleaned.append(char)
            last_dash = False
        elif not last_dash:
            cleaned.append("-")
            last_dash = True
    return "".join(cleaned).strip("-") or f"listing-{datetime.utcnow().strftime('%Y%m%d%H%M%S')}"


def unique_slug(base_slug, stock_number=None, listing_id=None):
    base_slug = slugify(base_slug)
    candidate = base_slug
    if stock_number:
        candidate = slugify(f"{base_slug}-{stock_number}")
    counter = 2
    while True:
        params = [candidate]
        query = "SELECT id FROM listings WHERE slug = ?"
        if listing_id:
            query += " AND id != ?"
            params.append(listing_id)
        existing = get_db().execute(query, params).fetchone()
        if existing is None:
            return candidate
        candidate = slugify(f"{base_slug}-{counter}")
        counter += 1


def strip_internal_stock_references(value, listing_or_stock):
    text = str(value or "")
    if not text:
        return ""
    if hasattr(listing_or_stock, "keys"):
        stock_number = listing_or_stock["stock_number"] or ""
    else:
        stock_number = str(listing_or_stock or "")
    if stock_number:
        escaped = re.escape(stock_number)
        text = re.sub(rf"(?i)\s*[\(\[\|/-]*\s*{escaped}\s*[\)\]\|/-]*\s*", " ", text)
    text = re.sub(r"(?i)\bst\d{3,}\b", " ", text)
    text = re.sub(r"\s+([,.;:])", r"\1", text)
    text = re.sub(r"([|/-])\s*([|/-])+", r"\1", text)
    text = re.sub(r"\s{2,}", " ", text)
    text = re.sub(r"\(\s*\)", "", text)
    text = re.sub(r"\s+[-|]\s*$", "", text)
    return text.strip(" -|")


def public_title_for_listing(listing, lang="en"):
    return strip_internal_stock_references(get_translated(listing, "title", lang), listing)


def public_text_for_listing(listing, field_base, lang="en"):
    return strip_internal_stock_references(get_translated(listing, field_base, lang), listing)


def public_slug_for_listing(listing):
    title = strip_internal_stock_references(listing["title_en"] or listing["slug"] or listing["stock_number"], listing)
    # title already starts with the year (e.g. "2025 Ford Bronco..."); don't prepend year separately (avoids doubled year in slug).
    parts = [title, listing["public_stock_number"] or ""]
    return unique_slug(" ".join(part for part in parts if part), listing_id=listing["id"])


def infer_make(title):
    title_lower = (title or "").lower()
    makes = (
        "John Deere",
        "New Holland",
        "Massey Ferguson",
        "Caterpillar",
        "Takeuchi",
        "Kubota",
        "Hitachi",
        "Komatsu",
        "Volvo",
        "Yanmar",
        "Bobcat",
        "Manitou",
        "Merlo",
        "Doosan",
        "Hyundai",
        "Terex",
        "Thwaites",
        "Ausa",
        "Sany",
        "Case",
        "JCB",
        "CAT",
    )
    for make in makes:
        if make.lower() in title_lower:
            return "Caterpillar" if make == "CAT" else make
    return ""


def infer_category(title):
    title_lower = (title or "").lower()
    categories = (
        (("sedan", "berline", "berlina"), "Sedan"),
        (("suv", "crossover", "4x4", "awd"), "SUV"),
        (("truck", "pickup", "pick-up", "camion"), "Truck"),
        (("coupe", "coupe"), "Coupe"),
        (("hatchback", "hatch back", "5 door", "3 door"), "Hatchback"),
        (("van", "minivan", "minibus", "fourgon"), "Van"),
        (("convertible", "cabriolet"), "Convertible"),
        (("wagon", "estate", "break", "familiale"), "Wagon"),
    )
    for keywords, category in categories:
        if any(keyword in title_lower for keyword in keywords):
            return category
    return "Other"


def infer_model(title, make):
    words = (title or "").replace("/", " ").replace("|", " ").split()
    if make:
        make_parts = make.split()
        lowered = [word.lower().strip(",") for word in words]
        for index, word in enumerate(lowered):
            if word == make_parts[-1].lower() and index + 1 < len(words):
                return words[index + 1].strip(",")
    for word in words:
        cleaned = word.strip(",")
        if any(char.isdigit() for char in cleaned) and any(char.isalpha() for char in cleaned):
            return cleaned
    return ""


def markdown_to_basic_html(markdown):
    text = (markdown or "").strip()
    if not text:
        return ""
    paragraphs = [part.strip() for part in text.replace("\r\n", "\n").split("\n\n") if part.strip()]
    if not paragraphs:
        paragraphs = [text]
    return "".join(f"<p>{escape(paragraph).replace(chr(10), '<br>')}</p>" for paragraph in paragraphs)


PRICE_PATTERN = re.compile(r"(?i)(\$|cad)\s?\d[\d,.\s]*|\d[\d,.\s]*\s?(cad)")


def clean_raw_listing_facts_for_ai(listing):
    values = []
    for key in ("title_en", "year", "make", "model", "category", "hours", "short_description_en", "description_html_en", "source_metadata_json"):
        if key in listing.keys() and listing[key]:
            values.append(strip_internal_stock_references(str(listing[key]), listing))
    text = "\n".join(values)
    text = re.sub(r"<[^>]+>", "\n", text)
    cleaned_lines = []
    for raw_line in text.replace("\r", "\n").split("\n"):
        line = " ".join(raw_line.split()).strip()
        if not line:
            continue
        if PRICE_PATTERN.search(line):
            continue
        line = re.sub(r"(?i)\b(\d{2,6})\s*hrs?\b", r"\1 hours", line)
        line = re.sub(r"(?i)\b(\d{2,6})\s*h\b", r"\1 hours", line)
        cleaned_lines.append(line)
    specs = []
    for line in cleaned_lines:
        if len(line) <= 120 and line.lower() not in {"available", "sold", "reserved"}:
            specs.append(line)
    return {
        "public_stock_number": listing["public_stock_number"],
        "title": strip_internal_stock_references(listing["title_en"] or "", listing),
        "year": listing["year"] or "",
        "make": listing["make"] or "",
        "model": listing["model"] or "",
        "category": listing["category"] or "",
        "hours": listing["hours"] or "",
        "facts_text": "\n".join(cleaned_lines[:80]),
        "specs": specs[:30],
    }


def ai_generation_prompt(listing):
    facts = clean_raw_listing_facts_for_ai(listing)
    return f"""
You are writing factual used vehicles listing content from stored facts only.
Return strict JSON only with these keys:
title_en, short_description_en, description_html_en, seo_title_en, meta_description_en,
make, model, category, hours, key_specs, safety_notes.

Rules:
- Do not include price, VAT, finance, warranty, delivery-included, ownership, inspection, or condition claims unless explicitly present.
- Do not include the internal crawler stock number. Use only the public reference if a reference is needed.
- Do not invent engine, weight, dimensions, hydraulics, attachments, year, hours, or model data.
- description_html_en must be an HTML fragment only, 180-350 words max, no markdown.
- Use this structure when possible: <h2>...</h2><p>...</p><ul>...</ul><h3>Key Features</h3><ul>...</ul>
- key_specs and safety_notes must be arrays.
- If unsure, keep a field empty and add a safety note.

Stored facts:
{json.dumps(facts, ensure_ascii=False, indent=2)}
"""


def call_fireworks_chat(prompt, settings, model_override=None, max_tokens=4000, timeout=150):
    if requests is None:
        raise RuntimeError("Install requests before using Fireworks AI generation.")
    api_key = fireworks_api_key(settings)
    if not api_key:
        raise RuntimeError("Fireworks API key is missing.")
    model = model_override or settings["fireworks_custom_model"] or settings["content_model"] or settings["fireworks_model"] or DEFAULT_CONTENT_MODEL
    response = requests.post(
        FIREWORKS_CHAT_COMPLETIONS_URL,
        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
        json={
            "model": model,
            "messages": [
                {"role": "system", "content": "Return one complete valid JSON object only. Never use markdown fences. Never truncate JSON strings."},
                {"role": "user", "content": prompt},
            ],
            "temperature": 0.1,
            "max_tokens": int(max_tokens),
            "response_format": {"type": "json_object"},
        },
        timeout=timeout,
    )
    if not response.ok:
        details = response.text[:1000] if response.text else response.reason
        raise RuntimeError(f"Fireworks API {response.status_code}: {details}")
    payload = response.json()
    choices = payload.get("choices") or []
    if not choices:
        raise RuntimeError("Fireworks returned no choices.")
    content = ((choices[0].get("message") or {}).get("content") or "").strip()
    if not content:
        finish_reason = choices[0].get("finish_reason") or "unknown"
        raise RuntimeError(f"Fireworks returned empty content (finish_reason={finish_reason}).")
    return content, model


def call_fireworks_chatbot(messages, settings):
    if requests is None:
        raise RuntimeError("Install requests before using Fireworks chatbot.")
    api_key = fireworks_api_key(settings)
    if not api_key:
        raise RuntimeError("Fireworks API key is missing.")
    model = ""
    if "chatbot_custom_model" in settings.keys() and settings["chatbot_custom_model"]:
        model = settings["chatbot_custom_model"]
    if not model and "chatbot_model" in settings.keys():
        model = settings["chatbot_model"] or DEFAULT_CHATBOT_MODEL
    model = model or DEFAULT_CHATBOT_MODEL
    response = requests.post(
        FIREWORKS_CHAT_COMPLETIONS_URL,
        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
        json={
            "model": model,
            "messages": messages,
            "temperature": 0.25,
            "max_tokens": 450,
        },
        timeout=35,
    )
    if not response.ok:
        details = response.text[:300] if response.text else response.reason
        raise RuntimeError(f"Fireworks chatbot API {response.status_code}: {details}")
    payload = response.json()
    usage = payload.get("usage") or {}
    return {
        "text": payload["choices"][0]["message"]["content"].strip(),
        "model": model,
        "tokens_input": usage.get("prompt_tokens") or usage.get("input_tokens") or 0,
        "tokens_output": usage.get("completion_tokens") or usage.get("output_tokens") or 0,
    }


def call_fireworks_vision_review(prompt, original_path, result_path, settings):
    if requests is None:
        raise RuntimeError("Install requests before using Fireworks AI review.")
    api_key = fireworks_api_key(settings)
    if not api_key:
        raise RuntimeError("Fireworks API key is missing.")
    original_file = static_url_to_path(original_path)
    result_file = static_url_to_path(result_path)
    if not original_file or not result_file:
        raise RuntimeError("Review skipped because image files were unavailable.")
    model = settings["fireworks_image_review_model"] if "fireworks_image_review_model" in settings.keys() else ""
    model = model or DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL
    response = requests.post(
        FIREWORKS_CHAT_COMPLETIONS_URL,
        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
        json={
            "model": model,
            "messages": [
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": prompt},
                        {"type": "image_url", "image_url": {"url": image_data_url(original_file)}},
                        {"type": "image_url", "image_url": {"url": image_data_url(result_file)}},
                    ],
                }
            ],
            "temperature": 0,
            "max_tokens": 500,
        },
        timeout=90,
    )
    if not response.ok:
        details = response.text[:600] if response.text else response.reason
        raise RuntimeError(f"Fireworks vision review {response.status_code}: {details}")
    payload = response.json()
    return payload["choices"][0]["message"]["content"], model


def call_openai_content(prompt, settings):
    global OpenAI
    if OpenAI is None and importlib.util.find_spec("openai") is None:
        raise RuntimeError("OpenAI package is not installed in this Python environment.")
    if OpenAI is None:
        from openai import OpenAI as OpenAIClient
        OpenAI = OpenAIClient
    api_key = openai_content_api_key(settings)
    if not api_key:
        raise RuntimeError("OpenAI API key is missing.")
    model = settings["openai_model"] or "gpt-5.2"
    client = OpenAI(api_key=api_key)
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": "Return strict JSON only. Do not include markdown fences."},
            {"role": "user", "content": prompt},
        ],
    )
    return response.choices[0].message.content, model


# How many times the Z.ai endpoint rejected the thinking param and we silently
# fell back to default (reasoning) behavior — surfaced by the AI diagnostics.
_zai_thinking_fallbacks = {"count": 0, "last_at": ""}


def _zai_anthropic_request(settings, messages, system=None, max_tokens=4000, timeout=3000, thinking=None):
    """POST to Z.ai's Anthropic-compatible /v1/messages endpoint. Returns the parsed JSON payload.

    `thinking` (e.g. {"type": "disabled"}) is passed through so utility calls can
    turn GLM's reasoning off — reasoning burns output tokens (slow + can exhaust
    max_tokens before any text). If the endpoint rejects the parameter, the
    request is retried once without it."""
    if requests is None:
        raise RuntimeError("Install requests before using Z.ai generation.")
    api_key = zai_api_key(settings)
    if not api_key:
        raise RuntimeError("Z.ai API key is missing.")
    model = settings["zai_model"] or "glm-5.2"
    body = {"model": model, "max_tokens": int(max_tokens), "messages": messages}
    if system:
        body["system"] = system
    if thinking:
        body["thinking"] = thinking

    def _post(payload):
        return requests.post(
            f"{ZAI_ANTHROPIC_BASE}/v1/messages",
            headers={
                "x-api-key": api_key,
                "anthropic-version": "2023-06-01",
                "content-type": "application/json",
            },
            json=payload,
            timeout=timeout,
        )

    response = _post(body)
    if not response.ok and thinking and response.status_code == 400:
        # Endpoint may not accept the thinking param — retry once without it.
        _zai_thinking_fallbacks["count"] += 1
        _zai_thinking_fallbacks["last_at"] = now_utc()
        body.pop("thinking", None)
        response = _post(body)
    if not response.ok:
        details = response.text[:1000] if response.text else response.reason
        raise RuntimeError(f"Z.ai API {response.status_code}: {details}")
    return response.json(), model


def _zai_first_text(payload):
    """First non-empty text block from a Z.ai Anthropic-style response.
    Reasoning models can lead with empty text blocks or thinking blocks, so
    only a non-empty text block counts."""
    for block in payload.get("content") or []:
        if isinstance(block, dict) and block.get("type") == "text":
            text = (block.get("text") or "").strip()
            if text:
                return text
    return ""


# How many times the v4 endpoint was unavailable and we fell back to the
# Anthropic-compatible endpoint — surfaced by the AI diagnostics.
_zai_v4_fallbacks = {"count": 0, "last_at": ""}
# v4 client errors that mean "this endpoint won't work for us" (fall back to the
# Anthropic endpoint). Auth/permission/rate/server errors are NOT fallback-worthy
# — falling back there would just double the wait for the same failure.
_ZAI_V4_FALLBACK_STATUSES = (400, 404, 405, 415)


def _zai_v4_request(settings, messages, system=None, max_tokens=4000, timeout=3000, thinking=None):
    """POST to Z.ai's NATIVE v4 chat-completions endpoint. This endpoint reliably
    honors `thinking: {"type": "disabled"}` for GLM models — the Anthropic-compat
    endpoint may accept the parameter without applying it (reasoning stays on,
    which is where minute-long calls come from). Returns (parsed_json, model) in
    the OpenAI shape: choices[0].message.content, usage.prompt_tokens/
    completion_tokens, choices[0].finish_reason."""
    if requests is None:
        raise RuntimeError("Install requests before using Z.ai generation.")
    api_key = zai_api_key(settings)
    if not api_key:
        raise RuntimeError("Z.ai API key is missing.")
    model = settings["zai_model"] or "glm-4.6"
    msgs = []
    if system:
        msgs.append({"role": "system", "content": system})
    msgs.extend(messages)
    body = {"model": model, "max_tokens": int(max_tokens), "messages": msgs}
    if thinking:
        body["thinking"] = thinking
    response = requests.post(
        ZAI_OPENAI_CHAT_URL,
        headers={"Authorization": f"Bearer {api_key}", "content-type": "application/json"},
        json=body,
        timeout=timeout,
    )
    if not response.ok:
        details = response.text[:1000] if response.text else response.reason
        raise RuntimeError(f"Z.ai API {response.status_code}: {details}")
    return response.json(), model


def _zai_v4_first_text(payload):
    """First non-empty assistant text from a v4 chat-completions response."""
    for choice in payload.get("choices") or []:
        message = choice.get("message") if isinstance(choice, dict) else None
        text = (message.get("content") or "").strip() if isinstance(message, dict) else ""
        if text:
            return text
    return ""


def _zai_v4_worth_fallback(error):
    """True when a v4 failure means 'endpoint incompatible' — safe to retry on the
    Anthropic endpoint without doubling the wait for the same outcome."""
    m = re.match(r"Z\.ai API (\d+):", str(error))
    return bool(m) and int(m.group(1)) in _ZAI_V4_FALLBACK_STATUSES


def _zai_note_v4_fallback(reason):
    _zai_v4_fallbacks["count"] += 1
    _zai_v4_fallbacks["last_at"] = now_utc()
    logging.getLogger(__name__).warning(f"Z.ai v4 endpoint unavailable, falling back to Anthropic endpoint: {reason}")


def call_zai_chat(prompt, settings, max_tokens=4000, timeout=3000):
    """Single-prompt call to Z.ai with a JSON-only system message, mirroring
    call_fireworks_chat. Returns (text, model).

    Prefers Z.ai's NATIVE v4 endpoint, which reliably honors
    thinking: {"type": "disabled"} for GLM models (the Anthropic-compat endpoint
    may accept the parameter without applying it — reasoning stays on and calls
    take a minute+). If v4 rejects the call (endpoint incompatible), falls back
    to the Anthropic endpoint. On either endpoint, a response that comes back
    empty because the budget was burned on reasoning is retried once with a much
    larger budget instead of failing."""
    messages = [{"role": "user", "content": prompt}]
    system = "Return one complete valid JSON object only. Never use markdown fences. Never truncate JSON strings."
    try:
        payload, model = _zai_v4_request(settings, messages, system=system, max_tokens=max_tokens,
                                         timeout=timeout, thinking={"type": "disabled"})
        finish = ""
        try:
            finish = str((payload.get("choices") or [{}])[0].get("finish_reason") or "")
        except (IndexError, AttributeError):
            finish = ""
        if not _zai_v4_first_text(payload) and finish == "length":
            payload, model = _zai_v4_request(
                settings, messages, system=system,
                max_tokens=min(16000, max(12000, int(max_tokens) * 2)),
                timeout=timeout, thinking={"type": "disabled"})
        text = _zai_v4_first_text(payload)
        if text:
            return text, model
        _zai_note_v4_fallback("v4 endpoint returned no text — trying the Anthropic endpoint")
    except RuntimeError as e:
        if not _zai_v4_worth_fallback(e):
            raise
        _zai_note_v4_fallback(str(e))
    # ---- Anthropic-compatible endpoint (previous behavior) ----
    payload, model = _zai_anthropic_request(
        settings,
        messages,
        system=system,
        max_tokens=max_tokens,
        timeout=timeout,
        thinking={"type": "disabled"},
    )
    text = _zai_first_text(payload)
    if not text and (payload.get("stop_reason") or "") == "max_tokens":
        payload, model = _zai_anthropic_request(
            settings,
            messages,
            system=system,
            max_tokens=min(16000, max(12000, int(max_tokens) * 2)),
            timeout=timeout,
            thinking={"type": "disabled"},
        )
        text = _zai_first_text(payload)
    if not text:
        finish = payload.get("stop_reason") or "unknown"
        raise RuntimeError(
            f"Z.ai returned empty content (stop_reason={finish}, model={model}). The model probably spent its "
            "whole output budget on reasoning — try again, shorten the master prompt, or switch model."
        )
    return text, model


def call_zai_chatbot(messages, settings):
    """Chatbot call to Z.ai. Returns {text, model, tokens_input, tokens_output}.
    Prefers the v4 endpoint (reliable thinking control = fast replies); falls
    back to the Anthropic endpoint when v4 is incompatible."""
    try:
        payload, model = _zai_v4_request(settings, messages, max_tokens=600, timeout=60,
                                         thinking={"type": "disabled"})
        usage = payload.get("usage") or {}
        return {
            "text": _zai_v4_first_text(payload),
            "model": model,
            "tokens_input": usage.get("prompt_tokens") or 0,
            "tokens_output": usage.get("completion_tokens") or 0,
        }
    except RuntimeError as e:
        if not _zai_v4_worth_fallback(e):
            raise
        _zai_note_v4_fallback(str(e))
    payload, model = _zai_anthropic_request(settings, messages, max_tokens=600, timeout=60,
                                            thinking={"type": "disabled"})
    text = _zai_first_text(payload)
    usage = payload.get("usage") or {}
    return {
        "text": text,
        "model": model,
        "tokens_input": usage.get("input_tokens") or 0,
        "tokens_output": usage.get("output_tokens") or 0,
    }


def call_openai_chat_messages(messages, settings):
    """Chatbot call via OpenAI (message-list). Returns the same dict shape as call_fireworks_chatbot."""
    global OpenAI
    if OpenAI is None and importlib.util.find_spec("openai") is None:
        raise RuntimeError("OpenAI package is not installed in this Python environment.")
    if OpenAI is None:
        from openai import OpenAI as OpenAIClient
        OpenAI = OpenAIClient
    api_key = openai_content_api_key(settings)
    if not api_key:
        raise RuntimeError("OpenAI API key is missing.")
    model = settings["openai_model"] or "gpt-5.2"
    client = OpenAI(api_key=api_key)
    response = client.chat.completions.create(model=model, messages=messages, max_tokens=450)
    text = (response.choices[0].message.content or "").strip()
    usage = getattr(response, "usage", None)
    return {
        "text": text,
        "model": model,
        "tokens_input": getattr(usage, "prompt_tokens", 0) if usage else 0,
        "tokens_output": getattr(usage, "completion_tokens", 0) if usage else 0,
    }


def make_text_adapter(settings, feature):
    """Build a (status, ai_call) for a text (prompt->text) feature, routing by provider_for(feature)."""
    provider = provider_for(settings, feature)
    status = {
        "provider": provider,
        "active_configured": bool(provider_api_key(settings, provider)),
    }

    def ai_call(prompt):
        if provider == "openai":
            return call_openai_content(prompt, settings)[0]
        if provider == "zai":
            return call_zai_chat(prompt, settings)[0]
        return call_fireworks_chat(prompt, settings)[0]

    return status, ai_call


def make_chat_adapter(settings):
    """Build an ai_call(messages -> {text, model, ...}) for the chatbot, routing by provider_for('chatbot')."""
    provider = provider_for(settings, "chatbot")

    def ai_call(messages):
        if provider == "openai":
            return call_openai_chat_messages(messages, settings)
        if provider == "zai":
            return call_zai_chatbot(messages, settings)
        return call_fireworks_chatbot(messages, settings)

    return ai_call


def ai_translation_prompt(source, target_language):
    return f"""
Translate stored website content from English to {target_language}.
Return strict JSON only with keys: title, excerpt, content_html, seo_title, meta_description.
Rules:
- Preserve HTML tags and structure in content_html.
- Preserve vehicle model names, company names, public references, units, numbers, and URLs.
- Do not add new claims, prices, warranty, finance, or delivery promises.
- Keep meta_description under about 155 characters.

Source JSON:
{json.dumps(source, ensure_ascii=False, indent=2)}
"""


def translate_page_with_ai(page_id, lang, force=False, publish=True):
    if lang == "en" or not is_valid_lang(lang):
        raise RuntimeError("Choose a supported non-English language.")
    db = get_db()
    page = db.execute("SELECT * FROM site_pages WHERE id = ?", (page_id,)).fetchone()
    if not page:
        raise RuntimeError("Page not found.")
    if not force and page[f"translation_status_{lang}"] in ("needs_review", "approved", "published"):
        return "skipped"
    source = {
        "title": page["title_en"] or "",
        "excerpt": page["excerpt_en"] or "",
        "content_html": page["content_html_en"] or "",
        "seo_title": page["seo_title_en"] or page["title_en"] or "",
        "meta_description": page["meta_description_en"] or page["excerpt_en"] or "",
    }
    settings = get_settings()
    prompt = ai_translation_prompt(source, LANGUAGE_NAMES[lang])
    data, raw_response, model, provider, _attempts_used = request_translation_json(prompt, settings, max_tokens=16000, attempts=3)
    now = now_utc()
    translation_status = "published" if publish else "needs_review"
    translation_published = 1 if publish else 0
    db.execute(
        f"""
        UPDATE site_pages
        SET title_{lang} = ?, excerpt_{lang} = ?, content_html_{lang} = ?,
            seo_title_{lang} = ?, meta_description_{lang} = ?,
            translation_status_{lang} = ?,
            translated_at_{lang} = ?,
            translation_published_{lang} = ?,
            updated_at = ?
        WHERE id = ?
        """,
        (
            data.get("title", ""),
            data.get("excerpt", ""),
            data.get("content_html", ""),
            data.get("seo_title", ""),
            data.get("meta_description", ""),
            translation_status,
            now,
            translation_published,
            now,
            page_id,
        ),
    )
    db.execute(
        """
        INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
        VALUES (NULL, ?, ?, ?, ?, 'success', '', ?)
        """,
        (provider, model, prompt, raw_response, now),
    )
    return "translated"



def ai_listing_translation_prompt(source, target_language):
    return f"""
Translate this used car listing from English to {target_language}.
Return strict JSON only with keys: title, short_description, description_html, seo_title, meta_description.

QUALITY RULES:
- Write natural, native dealership language; never use literal English word order.
- Put the vehicle type where native speakers normally expect it. Example Italian style: "Berlina BMW Serie 3 del 2019", not "BMW Serie 3 Berlina".
- Use established car-industry terminology for sedans, SUVs, trucks, coupes, hatchbacks, vans.
- Preserve manufacturer/model names, CA reference, year, kilometers, dimensions, units, numbers and technical specifications exactly.
- Preserve HTML tags and structure in description_html.
- Do not invent price, warranty, finance, availability, condition, inspection or delivery claims.
- Do not translate brand names, model names, stock references, URLs or domain names.
- Keep short_description concise and sales-ready.
- Keep meta_description natural and around 155 characters or less.
- Do not add facts absent from English.

Source JSON:
{json.dumps(source, ensure_ascii=False, indent=2)}
"""


def listing_translation_has_content(listing, lang):
    if lang == "en":
        return True
    for field in ("title", "short_description", "description_html"):
        key = f"{field}_{lang}"
        if key in listing.keys() and (listing[key] or "").strip():
            return True
    return False


def listing_translation_status_label(listing, lang):
    if lang == "en":
        return "EN"
    if listing_translation_has_content(listing, lang):
        return "OK"
    return "Missing"


def listing_translation_summary(listing):
    summary = []
    for language in get_active_languages():
        code = language["code"]
        if code == "en":
            continue
        summary.append({
            "code": code,
            "label": language_display_code(code),
            "name": language["name"],
            "status": listing_translation_status_label(listing, code),
            "has_content": listing_translation_has_content(listing, code),
        })
    return summary


def translate_listing_with_ai(listing_id, lang, force=False):
    if lang == "en" or not is_valid_lang(lang):
        raise RuntimeError("Choose a supported non-English language.")
    db = get_db()
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (listing_id,)).fetchone()
    if not listing:
        raise RuntimeError("Listing not found.")
    if not force and listing_translation_has_content(listing, lang):
        return "skipped"

    source = {
        "title": listing["title_en"] or "",
        "short_description": listing["short_description_en"] or "",
        "description_html": listing["description_html_en"] or "",
        "seo_title": listing["seo_title_en"] or listing["title_en"] or "",
        "meta_description": listing["meta_description_en"] or listing["short_description_en"] or "",
        "public_reference": listing["public_stock_number"] or "",
        "year": listing["year"] or "",
        "make": listing["make"] or "",
        "model": listing["model"] or "",
        "category": listing["category"] or "",
        "hours": listing["hours"] or "",
    }
    if not any((source["title"], source["short_description"], source["description_html"])):
        raise RuntimeError("Listing has no English title/description to translate.")

    settings = get_settings()
    prompt = ai_listing_translation_prompt(source, LANGUAGE_NAMES[lang])
    data, raw_response, model, provider, _attempts_used = request_translation_json(prompt, settings, max_tokens=7000, attempts=3)
    now = now_utc()

    db.execute(
        f"""
        UPDATE listings
        SET title_{lang} = ?,
            short_description_{lang} = ?,
            description_html_{lang} = ?,
            seo_title_{lang} = ?,
            meta_description_{lang} = ?,
            updated_at = ?
        WHERE id = ?
        """,
        (
            data.get("title", "") or source["title"],
            data.get("short_description", "") or source["short_description"],
            data.get("description_html", "") or source["description_html"],
            data.get("seo_title", "") or data.get("title", "") or source["seo_title"],
            data.get("meta_description", "") or data.get("short_description", "") or source["meta_description"],
            now,
            listing_id,
        ),
    )
    db.execute(
        """
        INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
        VALUES (?, ?, ?, ?, ?, 'success', '', ?)
        """,
        (listing_id, provider, model, prompt, raw_response, now),
    )
    return "translated"





def faq_translation_has_content(faq, lang):
    return bool(faq and lang != "en" and (faq[f"question_{lang}"] or "").strip() and (faq[f"answer_{lang}"] or "").strip())


def ai_faq_translation_prompt(source, target_language):
    return f"""
Translate this buyer FAQ from English to {target_language}.
Return strict JSON only with keys: question, answer.
Rules:
- Use natural, professional language for vehicles buyers.
- Preserve company names, CA references, VAT terminology, countries, numbers and URLs.
- Do not invent promises, prices, warranties, finance terms or delivery guarantees.
- Keep the question concise and the answer faithful to the English source.

Source JSON:
{json.dumps(source, ensure_ascii=False, indent=2)}
"""


def translate_faq_with_ai(faq_id, lang, force=False):
    if lang == "en" or not is_valid_lang(lang):
        raise RuntimeError("Choose a supported non-English language.")
    db = get_db()
    faq = db.execute("SELECT * FROM faqs WHERE id = ?", (faq_id,)).fetchone()
    if not faq:
        raise RuntimeError("FAQ not found.")
    if not force and faq_translation_has_content(faq, lang):
        return "skipped"
    source = {"question": faq["question_en"] or "", "answer": faq["answer_en"] or ""}
    if not source["question"] and not source["answer"]:
        raise RuntimeError("FAQ English source is empty.")
    settings = get_settings()
    prompt = ai_faq_translation_prompt(source, LANGUAGE_NAMES.get(lang, lang))
    data, raw_response, model, provider, _ = request_translation_json(prompt, settings, max_tokens=2500, attempts=3)
    question = (data.get("question") or "").strip()
    answer = (data.get("answer") or "").strip()
    if not question or not answer:
        raise RuntimeError("Translation model returned incomplete FAQ JSON.")
    db.execute(f"UPDATE faqs SET question_{lang} = ?, answer_{lang} = ?, updated_at = ? WHERE id = ?", (question, answer, now_utc(), faq_id))
    db.execute("""INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
                  VALUES (NULL, ?, ?, ?, ?, 'success', '', ?)""", (provider, model, prompt, raw_response, now_utc()))
    return "translated"

def parse_ai_json(text):
    stripped = (text or "").strip().lstrip("\ufeff")
    if not stripped:
        raise ValueError("AI returned an empty response.")
    if stripped.startswith("```"):
        stripped = re.sub(r"^```(?:json)?\s*", "", stripped, flags=re.I).strip()
        stripped = re.sub(r"\s*```$", "", stripped).strip()
    start = stripped.find("{")
    if start < 0:
        raise ValueError("AI response did not contain a JSON object.")
    candidate = stripped[start:]
    decoder = json.JSONDecoder()
    try:
        data, _end = decoder.raw_decode(candidate)
    except json.JSONDecodeError as error:
        excerpt = re.sub(r"\s+", " ", candidate[:500])
        raise ValueError(f"Invalid or truncated JSON: {error}. Response begins: {excerpt}") from error
    if not isinstance(data, dict):
        raise ValueError("AI JSON response was not an object.")
    return data


def request_translation_json(prompt, settings, max_tokens=7000, attempts=3):
    """Call the configured text model and retry empty, malformed or truncated JSON."""
    provider = (settings["ai_provider"] or "fireworks").lower()
    model_override = settings["translation_model"] or DEFAULT_TRANSLATION_MODEL
    last_error = None
    last_raw = ""
    model = model_override
    for attempt in range(1, attempts + 1):
        try:
            retry_note = ""
            if attempt > 1:
                retry_note = "\n\nIMPORTANT RETRY: The previous response was empty, malformed, or truncated. Return a COMPLETE valid JSON object and keep all HTML strings properly escaped."
            if provider == "openai":
                raw_response, model = call_openai_content(prompt + retry_note, settings)
            else:
                raw_response, model = call_fireworks_chat(
                    prompt + retry_note,
                    settings,
                    model_override=model_override,
                    max_tokens=max_tokens,
                    timeout=210,
                )
                provider = "fireworks"
            last_raw = raw_response or ""
            return parse_ai_json(last_raw), last_raw, model, provider, attempt
        except Exception as error:
            last_error = error
            if attempt < attempts:
                time.sleep(2 * attempt)
    excerpt = re.sub(r"\s+", " ", last_raw[:700]) if last_raw else "<empty>"
    raise RuntimeError(f"Translation failed after {attempts} attempts: {last_error}. Last response: {excerpt}")


def strip_price_references(value):
    if not value:
        return value
    return PRICE_PATTERN.sub("", str(value))


def validate_ai_listing_output(listing, data, allow_overwrite=False):
    safe = {}
    text_fields = ("title_en", "short_description_en", "description_html_en", "seo_title_en", "meta_description_en")
    for field in text_fields:
        safe[field] = strip_internal_stock_references(strip_price_references(data.get(field, "")), listing).strip()
    for field in ("make", "model", "category", "hours"):
        proposed = strip_internal_stock_references(strip_price_references(data.get(field, "")), listing).strip()
        existing = (listing[field] or "").strip()
        safe[field] = proposed if (allow_overwrite or not existing) else existing
    safe["key_specs"] = [strip_price_references(item).strip() for item in data.get("key_specs", []) if strip_price_references(item).strip()]
    safe["safety_notes"] = [str(item).strip() for item in data.get("safety_notes", []) if str(item).strip()]
    return safe


def generate_listing_ai_content(listing_id, allow_overwrite=False, mode="full"):
    db = get_db()
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (listing_id,)).fetchone()
    if not listing:
        raise RuntimeError("Listing not found.")
    settings = get_settings()
    status = ai_provider_status(settings)
    if not status["enabled"]:
        raise RuntimeError("AI generation is disabled in Settings.")
    if not status["active_configured"]:
        raise RuntimeError(f"{status['provider'].title()} API key is missing.")
    prompt = ai_generation_prompt(listing)
    provider = status["provider"]
    if provider == "openai":
        raw_response, model = call_openai_content(prompt, settings)
    else:
        raw_response, model = call_fireworks_chat(prompt, settings)
        provider = "fireworks"
    try:
        data = validate_ai_listing_output(listing, parse_ai_json(raw_response), allow_overwrite)
    except Exception as error:
        db.execute(
            """
            INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
            VALUES (?, ?, ?, ?, ?, 'error', ?, ?)
            """,
            (listing_id, provider, model, prompt, raw_response or "", str(error), now_utc()),
        )
        db.commit()
        raise
    fields = ["title_en", "short_description_en", "description_html_en", "seo_title_en", "meta_description_en"]
    if mode != "missing":
        fields.extend(["make", "model", "category", "hours"])
    else:
        fields.extend([field for field in ("make", "model", "category", "hours") if not listing[field]])
    values = [data[field] for field in fields]
    db.execute(
        f"UPDATE listings SET {', '.join(f'{field} = ?' for field in fields)}, updated_at = ? WHERE id = ?",
        (*values, now_utc(), listing_id),
    )
    db.execute(
        """
        INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
        VALUES (?, ?, ?, ?, ?, 'success', '', ?)
        """,
        (listing_id, provider, model, prompt, raw_response, now_utc()),
    )
    db.commit()
    return data


def record_ai_generation_error(listing_id, provider, model, prompt, error):
    get_db().execute(
        """
        INSERT INTO ai_generations (listing_id, provider, model, prompt, response, status, error, created_at)
        VALUES (?, ?, ?, ?, '', 'error', ?, ?)
        """,
        (listing_id, provider, model, prompt or "", str(error), now_utc()),
    )
    get_db().commit()


def short_text(value, limit=260):
    text = " ".join((value or "").replace("\r", " ").replace("\n", " ").split())
    if len(text) <= limit:
        return text
    return text[: limit - 3].rstrip() + "..."


def source_table_exists(source_conn, table_name):
    row = source_conn.execute(
        "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", (table_name,)
    ).fetchone()
    return row is not None


def resolve_source_image(local_path, source_db_path, folder_path=None):
    if not local_path:
        return None
    # Normalize Windows backslashes → forward slashes so a path captured on Windows
    # (e.g. images\\VIN\\000_x.jpg) resolves on Linux too.
    raw_path = Path(local_path.replace("\\", "/"))
    candidates = []
    if raw_path.is_absolute():
        candidates.append(raw_path)
    else:
        candidates.append(Path(source_db_path).resolve().parent / raw_path)
        if folder_path:
            folder = Path(folder_path.replace("\\", "/"))
            candidates.append((folder if folder.is_absolute() else Path(source_db_path).resolve().parent / folder) / raw_path.name)
    for candidate in candidates:
        if candidate.exists() and candidate.is_file():
            return candidate
    return None


def web_usable_path(path_value):
    if not path_value:
        return ""
    if path_value.startswith(("http://", "https://", "/static/")):
        return path_value
    return ""


def local_static_image_path(path_value):
    if not path_value:
        return ""
    value = str(path_value).strip().replace("\\", "/")
    if value.startswith(("http://", "https://")):
        return ""
    if value.startswith("/static/"):
        return value
    if value.startswith("static/"):
        return f"/{value}"
    try:
        path = Path(path_value)
        if path.is_absolute():
            resolved = path.resolve()
            static_root = (BASE_DIR / "static").resolve()
            try:
                resolved.relative_to(static_root)
                return "/" + resolved.relative_to(BASE_DIR).as_posix()
            except ValueError:
                return ""
    except (OSError, ValueError):
        return ""
    return ""


def static_url_to_path(url_path):
    local_path = local_static_image_path(url_path)
    if not local_path:
        return None
    candidate = (BASE_DIR / local_path.lstrip("/")).resolve()
    try:
        candidate.relative_to(BASE_DIR.resolve())
    except ValueError:
        return None
    if candidate.exists() and candidate.is_file():
        return candidate
    return None


THUMBNAIL_SUFFIX = "_thumb"
THUMBNAIL_EXTENSION = ".webp"
THUMBNAIL_MAX_SIZE = (700, 525)
THUMBNAIL_QUALITY = 72
GALLERY_SUFFIX = "_gallery"
GALLERY_EXTENSION = ".webp"
GALLERY_MAX_SIZE = (1800, 1350)
GALLERY_QUALITY = 82
IMAGE_CACHE_MAX_AGE = 31536000


def image_variant_file(source_file, suffix, extension):
    if not source_file:
        return None
    if source_file.stem.endswith((THUMBNAIL_SUFFIX, GALLERY_SUFFIX)):
        return None
    return source_file.with_name(f"{source_file.stem}{suffix}{extension}")


def image_variant_static_url(path_value, suffix, extension):
    local_path = local_static_image_path(path_value)
    if not local_path:
        return ""
    source_file = static_url_to_path(local_path)
    if not source_file:
        return ""
    variant_file = image_variant_file(source_file, suffix, extension)
    if variant_file and variant_file.exists() and variant_file.is_file():
        try:
            return "/" + variant_file.resolve().relative_to(BASE_DIR.resolve()).as_posix()
        except ValueError:
            return ""
    return ""


def thumbnail_static_url(path_value):
    """Return the matching _thumb.webp web path for a local /static image when it exists."""
    return image_variant_static_url(path_value, THUMBNAIL_SUFFIX, THUMBNAIL_EXTENSION)


def gallery_static_url(path_value):
    """Return the matching _gallery.webp web path for a local /static image when it exists."""
    return image_variant_static_url(path_value, GALLERY_SUFFIX, GALLERY_EXTENSION)


def preferred_card_image_path(path_value):
    """Use local thumbnail when available, otherwise the original display path."""
    return thumbnail_static_url(path_value) or local_static_image_path(path_value) or ""


def preferred_gallery_image_path(path_value):
    """Use optimized gallery image when available, otherwise the original display path."""
    return gallery_static_url(path_value) or local_static_image_path(path_value) or ""


@app.template_filter("thumb")
def thumb_path(path_value):
    """Jinja filter for templates: {{ image_path|thumb }}."""
    return preferred_card_image_path(path_value) or path_value


# Base color words → hex. A named paint color (e.g. "Grasmere Green", "Santorini Black",
# "Carpathian Grey") is matched to the base color WORD it contains, so every color renders
# a real swatch. Order matters: check multi-word / ambiguous keys first.
_COLOR_BASE_HEX = [
    ("black", "#1a1a1a"),
    ("white", "#f4f4f5"),
    ("grey", "#9ca3af"), ("gray", "#9ca3af"),
    ("silver", "#cbd5e1"),
    ("red", "#b91c1c"),
    ("blue", "#1d4ed8"),
    ("green", "#15803d"),
    ("brown", "#7c4a1e"),
    ("beige", "#d6c7a1"), ("tan", "#c9a37a"),
    ("gold", "#c9a227"), ("champagne", "#d8c4a0"),
    ("orange", "#ea580c"),
    ("yellow", "#ca8a04"),
    ("purple", "#7c3aed"), ("violet", "#7c3aed"),
    ("bronze", "#9c6b2e"),
    ("pearl", "#eef2f7"), ("pearlescent", "#eef2f7"),
    ("cream", "#f0ead6"),
    ("copper", "#b06b3a"),
]


def color_to_hex(name):
    """Map a color NAME (any paint name) to a hex swatch. Strategy:
    1) if it's already a #hex or rgb(), return as-is;
    2) if it contains a base color word, return that base's hex (e.g. "Grasmere Green" → green);
    3) CSS named colors (white/black/red...) pass through;
    4) fallback: a deterministic hash→hex so the same name always shows the same swatch
       (works for any future import color, e.g. "Nardo", "Magenta Dust")."""
    s = (name or "").strip()
    if not s:
        return "#d4d4d8"  # zinc-300 placeholder for empty
    low = s.lower()
    if low.startswith("#") or low.startswith("rgb"):
        return s
    for word, hexv in _COLOR_BASE_HEX:
        if word in low:
            return hexv
    # CSS recognizes bare names like 'white','red' — if it's one of those, pass through.
    if low.replace(" ", "") in ("white", "black", "red", "blue", "green", "yellow",
                                "orange", "purple", "silver", "gray", "grey", "brown"):
        return low
    # Deterministic fallback: hash the name → a hue → a muted hex. Same name = same color.
    h = int(hashlib.md5(low.encode("utf-8")).hexdigest(), 16)
    hue = h % 360
    import colorsys
    r, g, b = colorsys.hls_to_rgb(hue / 360.0, 0.45, 0.45)  # muted mid-light, mid-sat
    return "#{:02x}{:02x}{:02x}".format(int(r * 255), int(g * 255), int(b * 255))


@app.template_filter("colorhex")
def color_hex_filter(name):
    """Jinja filter: {{ listing.exterior_color|colorhex }} → a hex swatch color."""
    return color_to_hex(name)


def public_thumbnail_url_for_path(listing, path_value):
    thumb_display = thumbnail_static_url(path_value)
    if thumb_display:
        return public_image_url_for_path(listing, thumb_display)
    return public_image_url_for_path(listing, path_value)


def public_gallery_url_for_path(listing, path_value):
    gallery_display = gallery_static_url(path_value)
    if gallery_display:
        return public_image_url_for_path(listing, gallery_display)
    return public_image_url_for_path(listing, path_value)


def _image_resampling_filter():
    if Image is None:
        return None
    try:
        return Image.Resampling.LANCZOS
    except AttributeError:  # Pillow < 9.1
        return getattr(Image, "LANCZOS", getattr(Image, "ANTIALIAS", None))


def create_image_variant(source_file, suffix, extension, max_size, quality, regenerate=False):
    if Image is None or ImageOps is None:
        return False, "Pillow is not installed in the Flask environment."
    if not source_file or not source_file.exists() or not source_file.is_file():
        return False, "Source file missing."
    output_file = image_variant_file(source_file, suffix, extension)
    if output_file is None:
        return False, "Source is already a derived image."
    if output_file.exists() and not regenerate:
        return False, "exists"
    try:
        with Image.open(source_file) as im:
            im = ImageOps.exif_transpose(im)
            if im.mode not in ("RGB", "L"):
                im = im.convert("RGB")
            elif im.mode == "L":
                im = im.convert("RGB")
            resample = _image_resampling_filter()
            if resample is not None:
                im.thumbnail(max_size, resample)
            else:
                im.thumbnail(max_size)
            output_file.parent.mkdir(parents=True, exist_ok=True)
            im.save(output_file, "WEBP", quality=quality, method=6)
        return True, str(output_file)
    except Exception as error:
        return False, str(error)


def generate_image_derivatives_for_path(path_value, regenerate=False):
    summary = {"paths": 0, "written": 0, "skipped": 0, "errors": []}
    local_path = local_static_image_path(path_value)
    if not local_path:
        return summary
    source_file = static_url_to_path(local_path)
    if not source_file or not source_file.exists() or not source_file.is_file():
        return summary
    if source_file.stem.endswith((THUMBNAIL_SUFFIX, GALLERY_SUFFIX)):
        return summary
    if source_file.suffix.lower() not in IMAGE_EXTENSIONS:
        return summary

    summary["paths"] = 1
    for suffix, extension, max_size, quality in (
        (THUMBNAIL_SUFFIX, THUMBNAIL_EXTENSION, THUMBNAIL_MAX_SIZE, THUMBNAIL_QUALITY),
        (GALLERY_SUFFIX, GALLERY_EXTENSION, GALLERY_MAX_SIZE, GALLERY_QUALITY),
    ):
        written, message = create_image_variant(source_file, suffix, extension, max_size, quality, regenerate=regenerate)
        if written:
            summary["written"] += 1
        elif message == "exists":
            summary["skipped"] += 1
        elif message:
            summary["errors"].append(f"{source_file.name}: {message}")
    return summary


def merge_derivative_summary(summary, update):
    for key in ("paths", "written", "skipped"):
        summary[key] += update.get(key, 0)
    if update.get("errors"):
        summary["errors"].extend(update["errors"])
    return summary


def generate_image_derivatives_for_row(image, regenerate=False):
    summary = {"paths": 0, "written": 0, "skipped": 0, "errors": []}
    seen = set()
    for field in ("local_path", "generated_local_path"):
        if not image or field not in image.keys():
            continue
        path_value = image[field] or ""
        if not path_value or path_value in seen:
            continue
        seen.add(path_value)
        merge_derivative_summary(summary, generate_image_derivatives_for_path(path_value, regenerate=regenerate))
    return summary


def generate_image_derivatives_for_listing(listing_id, regenerate=False):
    summary = {"rows": 0, "paths": 0, "written": 0, "skipped": 0, "errors": []}
    rows = get_db().execute(
        "SELECT * FROM listing_images WHERE listing_id = ? ORDER BY sort_order, id",
        (listing_id,),
    ).fetchall()
    for row in rows:
        summary["rows"] += 1
        row_summary = generate_image_derivatives_for_row(row, regenerate=regenerate)
        merge_derivative_summary(summary, row_summary)
    return summary


def generate_image_derivatives_for_listings(listing_ids, regenerate=False):
    summary = {"listings": 0, "rows": 0, "paths": 0, "written": 0, "skipped": 0, "errors": []}
    for listing_id in listing_ids or []:
        listing_summary = generate_image_derivatives_for_listing(int(listing_id), regenerate=regenerate)
        summary["listings"] += 1
        merge_derivative_summary(summary, listing_summary)
        summary["rows"] += listing_summary.get("rows", 0)
    return summary


def generate_all_image_derivatives(regenerate=False):
    listing_ids = [row["id"] for row in get_db().execute("SELECT id FROM listings").fetchall()]
    return generate_image_derivatives_for_listings(listing_ids, regenerate=regenerate)


def image_path_missing_derivatives(path_value):
    """Return True when an original/current image is missing thumb or gallery variants."""
    local_path = local_static_image_path(path_value)
    if not local_path:
        return False
    source_file = static_url_to_path(local_path)
    if not source_file or not source_file.exists() or not source_file.is_file():
        return False
    if source_file.stem.endswith((THUMBNAIL_SUFFIX, GALLERY_SUFFIX)):
        return False
    if source_file.suffix.lower() not in IMAGE_EXTENSIONS:
        return False
    thumb_file = image_variant_file(source_file, THUMBNAIL_SUFFIX, THUMBNAIL_EXTENSION)
    gallery_file = image_variant_file(source_file, GALLERY_SUFFIX, GALLERY_EXTENSION)
    return bool(
        (thumb_file and not thumb_file.exists())
        or (gallery_file and not gallery_file.exists())
    )


def listing_image_source_paths():
    """Yield unique local_path/generated_local_path values from current image rows."""
    seen = set()
    rows = get_db().execute(
        """
        SELECT local_path, generated_local_path
        FROM listing_images
        ORDER BY listing_id, sort_order, id
        """
    ).fetchall()
    for row in rows:
        for field in ("local_path", "generated_local_path"):
            path_value = row[field] or ""
            if not path_value or path_value in seen:
                continue
            seen.add(path_value)
            yield path_value


def count_missing_image_derivative_sources(limit=None):
    count = 0
    for path_value in listing_image_source_paths():
        if image_path_missing_derivatives(path_value):
            count += 1
            if limit and count >= limit:
                return count
    return count


def generate_missing_image_derivatives_batch(limit=25):
    """Generate missing thumb/gallery files for a small safe batch.

    Shared hosting often kills long requests, so the admin button uses this
    instead of processing the full library in one request.
    """
    try:
        limit = int(limit or 25)
    except (TypeError, ValueError):
        limit = 25
    limit = max(1, min(limit, 100))
    summary = {"processed": 0, "paths": 0, "written": 0, "skipped": 0, "errors": [], "remaining": 0}

    for path_value in listing_image_source_paths():
        if summary["processed"] >= limit:
            break
        if not image_path_missing_derivatives(path_value):
            continue
        summary["processed"] += 1
        update = generate_image_derivatives_for_path(path_value, regenerate=False)
        merge_derivative_summary(summary, update)

    summary["remaining"] = count_missing_image_derivative_sources(limit=1000000)
    return summary



def read_image_variant_job_status():
    if not IMAGE_VARIANT_JOB_FILE.exists():
        return None
    try:
        return json.loads(IMAGE_VARIANT_JOB_FILE.read_text(encoding="utf-8"))
    except Exception:
        return {"status": "unknown", "message": "Could not read image variant job state."}


def image_variant_log_tail(max_lines=25):
    if not IMAGE_VARIANT_LOG_FILE.exists():
        return []
    try:
        lines = IMAGE_VARIANT_LOG_FILE.read_text(encoding="utf-8", errors="ignore").splitlines()
    except Exception:
        return []
    return lines[-max_lines:]


def start_image_variant_background_job(regenerate=False):
    existing = read_image_variant_job_status() or {}
    if existing.get("status") == "running":
        return False, "Image variant generation is already running in the background."
    script_path = BASE_DIR / "scripts" / "create_thumbnails.py"
    if not script_path.exists():
        return False, "scripts/create_thumbnails.py was not found."
    IMAGE_VARIANT_JOB_FILE.parent.mkdir(parents=True, exist_ok=True)
    cmd = [sys.executable, str(script_path), "--state-file", str(IMAGE_VARIANT_JOB_FILE)]
    if regenerate:
        cmd.append("--regenerate")
    try:
        log_handle = open(IMAGE_VARIANT_LOG_FILE, "a", encoding="utf-8")
        log_handle.write(f"\n=== Started {datetime.utcnow().isoformat()}Z | regenerate={regenerate} ===\n")
        log_handle.flush()
        subprocess.Popen(
            cmd,
            cwd=str(BASE_DIR),
            stdout=log_handle,
            stderr=subprocess.STDOUT,
            start_new_session=True,
            close_fds=True,
        )
        return True, "Background image generation started."
    except Exception as error:
        return False, str(error)


def get_image_display_path(image):
    if not image:
        return ""
    generated_path = image["generated_local_path"] if "generated_local_path" in image.keys() else ""
    use_generated = image["use_generated_image"] if "use_generated_image" in image.keys() else 0
    if use_generated and generated_path:
        generated_display = local_static_image_path(generated_path)
        if generated_display:
            return generated_display
    local_path = image["local_path"] if "local_path" in image.keys() else ""
    return local_static_image_path(local_path)


def public_image_url_for_path(listing, path_value):
    display_path = local_static_image_path(path_value)
    if not display_path or not listing:
        return display_path
    filename = Path(display_path).name
    public_ref = listing["public_stock_number"] or f"listing-{listing['id']}"
    return url_for("public_listing_image", public_ref=public_ref, filename=filename)


def public_image_url_for_row(listing, image):
    return public_image_url_for_path(listing, get_image_display_path(image))


def get_original_image_display_path(image):
    if not image:
        return ""
    local_path = image["local_path"] if "local_path" in image.keys() else ""
    return local_static_image_path(local_path)


def listing_upload_folder_candidates(listing):
    candidates = []
    for key in ("stock_number", "public_stock_number"):
        value = listing[key] if key in listing.keys() else ""
        if value:
            candidates.extend([value, secure_filename(value)])
    slug = listing["slug"] if "slug" in listing.keys() else ""
    if slug:
        candidates.append(slug)
    # Crawler imports store their folder name in folder_path, and build it as
    # "{year}-{make}-{model}-{public ref}" — check both so folder sync/repair
    # can find images for those listings too.
    folder_path = listing["folder_path"] if "folder_path" in listing.keys() else ""
    if folder_path:
        candidates.append(folder_path)
        candidates.append(secure_filename(folder_path))
    derived_parts = []
    for key in ("year", "make", "model", "public_stock_number"):
        raw_value = (listing[key] if key in listing.keys() else "") or ""
        part = re.sub(r'[<>:"/\\|?*]', "", str(raw_value)).replace(" ", "-")
        if part:
            derived_parts.append(part)
    if derived_parts:
        derived = "-".join(derived_parts).strip("-").lower()
        if derived:
            candidates.append(derived)
    unique = []
    for candidate in candidates:
        if candidate and candidate not in unique:
            unique.append(candidate)
    return unique


def local_upload_images_for_listing(listing):
    images = []
    for folder_name in listing_upload_folder_candidates(listing):
        folder = LISTING_UPLOADS_DIR / folder_name
        if not folder.exists() or not folder.is_dir():
            continue
        for file_path in sorted(folder.iterdir(), key=lambda item: item.name.lower()):
            if file_path.is_file() and file_path.suffix.lower() in IMAGE_EXTENSIONS:
                images.append(f"/static/uploads/listings/{folder_name}/{file_path.name}")
        if images:
            return images
    return images


def build_listing_gallery(listing, lang="en"):
    """Build ordered gallery data for listing detail pages.

    The main/lightbox image uses optimized _gallery.webp variants when available,
    while thumbnails use _thumb.webp variants. Original images are still preserved
    on disk for admin download/export.
    """
    image_rows = get_db().execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ?
          AND COALESCE(is_public, 1) = 1
        ORDER BY is_primary DESC, sort_order, id
        """,
        (listing["id"],),
    ).fetchall()
    images = []
    seen = set()
    listing_title = get_translated(listing, "title", lang)
    for image in image_rows:
        display_path = get_image_display_path(image)
        if display_path and display_path not in seen:
            seen.add(display_path)
            images.append({
                "src": public_gallery_url_for_path(listing, display_path),
                "full_src": public_image_url_for_path(listing, display_path),
                "thumb_src": public_thumbnail_url_for_path(listing, display_path),
                "alt": image["alt_text"] or listing_title,
            })
    # Do not auto-scan the listing upload folder here. Admin-deleted image rows
    # should disappear from the live gallery even if old files remain on disk.
    # Only current listing_images rows are public gallery sources.
    return images


def image_mime_type(path):
    guessed = mimetypes.guess_type(str(path))[0]
    if guessed in ("image/jpeg", "image/png", "image/webp", "image/gif"):
        return guessed
    return "image/jpeg"


def image_data_url(path):
    with open(path, "rb") as image_file:
        encoded = base64.b64encode(image_file.read()).decode()
    return f"data:{image_mime_type(path)};base64,{encoded}"


def get_openai_client():
    global OpenAI
    if OpenAI is None:
        try:
            from openai import OpenAI as OpenAIClient
            OpenAI = OpenAIClient
        except ImportError as error:
            raise RuntimeError(
                "Install the openai Python package in the same Python environment running Flask, then restart the Flask server."
            ) from error
    api_key = decrypt_secret(get_settings()["openai_api_key_encrypted"] or "")
    if not api_key:
        raise RuntimeError("Save a valid encrypted OpenAI API key in Settings first.")
    return OpenAI(api_key=api_key)


def upload_openai_file(client, path):
    try:
        with open(path, "rb") as file_obj:
            uploaded = client.files.create(file=file_obj, purpose="vision")
    except Exception:
        with open(path, "rb") as file_obj:
            uploaded = client.files.create(file=file_obj, purpose="assistants")
    return uploaded.id


def get_cached_logo_file_id(client, settings):
    logo_path = static_url_to_path(settings["logo_path"] or "")
    if not logo_path:
        raise RuntimeError("Upload a site logo in Settings before running image edits.")
    existing = settings["openai_logo_file_id"] if "openai_logo_file_id" in settings.keys() else ""
    if existing:
        return existing
    file_id = upload_openai_file(client, logo_path)
    get_db().execute("UPDATE site_settings SET openai_logo_file_id = ?, updated_at = ? WHERE id = 1", (file_id, now_utc()))
    get_db().commit()
    return file_id


def get_cached_image_file_id(client, image):
    existing = image["openai_file_id"] if "openai_file_id" in image.keys() else ""
    if existing:
        return existing
    image_path = static_url_to_path(get_original_image_display_path(image))
    if not image_path:
        raise RuntimeError("Selected image does not have a local static file.")
    file_id = upload_openai_file(client, image_path)
    get_db().execute("UPDATE listing_images SET openai_file_id = ? WHERE id = ?", (file_id, image["id"]))
    get_db().commit()
    return file_id


def openai_missing_file_error(error):
    text = str(error).lower()
    return "files [" in text and "were not found" in text


def clear_stale_openai_file_ids(image=None, clear_logo=False):
    if image and "id" in image.keys():
        get_db().execute(
            "UPDATE listing_images SET openai_file_id = '' WHERE id = ?",
            (image["id"],),
        )
    if clear_logo:
        get_db().execute("UPDATE site_settings SET openai_logo_file_id = '', updated_at = ? WHERE id = 1", (now_utc(),))
    get_db().commit()


def displayed_image_reference_path(image):
    return get_image_display_path(image) or get_original_image_display_path(image)


def extract_response_image_base64(response):
    for output in response.output:
        if getattr(output, "type", "") == "image_generation_call" and getattr(output, "result", None):
            return output.result
    return ""


def extract_response_text(response):
    if hasattr(response, "output_text") and response.output_text:
        return response.output_text
    parts = []
    for output in getattr(response, "output", []):
        for content in getattr(output, "content", []) or []:
            text = getattr(content, "text", None)
            if text:
                parts.append(text)
    return "\n".join(parts).strip()


def image_generation_tool_payload(settings):
    return {"type": "image_generation", "action": "edit"}


def ai_edit_folder_for_listing(listing):
    folder_name = secure_filename(listing["stock_number"] or f"listing-{listing['id']}") or f"listing-{listing['id']}"
    return AI_EDIT_UPLOADS_DIR / folder_name


def save_ai_edit_result(listing, image, image_base64, edit_kind="replacement"):
    target_dir = ai_edit_folder_for_listing(listing)
    target_dir.mkdir(parents=True, exist_ok=True)
    filename = seo_image_filename_for_listing(
        listing,
        image_order_value(image),
        ".png",
        variant="ai" if edit_kind != "angle" else "ai-angle",
    )
    target = unique_file_target(target_dir, filename)
    with open(target, "wb") as output_file:
        output_file.write(base64.b64decode(image_base64))
    return "/" + target.resolve().relative_to(BASE_DIR.resolve()).as_posix()


def review_ai_image_edit(client, settings, listing, original_path, result_path):
    original_file = static_url_to_path(original_path)
    result_file = static_url_to_path(result_path)
    if not original_file or not result_file:
        return {"status": "unknown", "notes": "Review skipped because image files were unavailable."}
    prompt = f"""
Review this AI-generated image edit against the requested instructions.

Instructions:
{(settings["openai_image_edit_prompt"] or default_ai_image_edit_prompt()).strip()}

Listing: {listing["title_en"] or listing["stock_number"]}

You will see two images in order:
1. Original source image.
2. AI edited result image.

Review whether image 2 follows the instructions while preserving the original vehicle, camera angle, physical scene, manufacturer decals, condition, and realism.
Fail if the vehicle identity changes, major geometry changes, manufacturer decals change, a logo is added to the vehicle/ground/sky/windows, or the edit invents a new sign/scene.
Pass only if deviations are minor and the requested sign/logo edit is plausible.

Return exactly two lines:
STATUS: pass or fail
NOTES: concise reason. If fail, describe how to fix it.
"""
    try:
        text, model = call_fireworks_vision_review(prompt, original_path, result_path, settings)
        provider = "Fireworks"
    except Exception as fireworks_error:
        response = client.responses.create(
            model=settings["openai_image_model"] or "gpt-5.2",
            input=[
                {
                    "role": "user",
                    "content": [
                        {"type": "input_text", "text": prompt},
                        {"type": "input_image", "image_url": image_data_url(original_file)},
                        {"type": "input_image", "image_url": image_data_url(result_file)},
                    ],
                }
            ],
        )
        text = extract_response_text(response)
        model = settings["openai_image_model"] or "gpt-5.2"
        provider = f"OpenAI fallback after Fireworks review failed: {fireworks_error}"
    status = "pass" if "status: pass" in text.lower() else "fail"
    return {"status": status, "notes": f"{provider} ({model}) review: {text or 'No review notes returned.'}"}


AI_ANGLE_CHOICES = (
    ("front-left-three-quarter-45", "front-left three-quarter view, about 45 degrees from the front"),
    ("front-right-three-quarter-45", "front-right three-quarter view, about 45 degrees from the front"),
    ("left-side-90", "true left side profile, about 90 degrees from the front"),
    ("right-side-90", "true right side profile, about 90 degrees from the front"),
    ("rear-left-three-quarter-135", "rear-left three-quarter view, about 135 degrees from the front"),
    ("rear-right-three-quarter-135", "rear-right three-quarter view, about 135 degrees from the front"),
)


def ai_angle_choices():
    return [{"value": value, "label": label} for value, label in AI_ANGLE_CHOICES]


def angle_label(value, total=None):
    mapping = dict(AI_ANGLE_CHOICES)
    if value in mapping:
        return mapping[value]
    try:
        index = int(value)
    except (TypeError, ValueError):
        return mapping[AI_ANGLE_CHOICES[0][0]]
    fallback = AI_ANGLE_CHOICES[(max(index, 1) - 1) % len(AI_ANGLE_CHOICES)][1]
    return fallback


def generate_ai_image_edit(listing, image, instruction, source_edit=None, edit_kind="replacement", angle_name=""):
    settings = get_settings()
    client = get_openai_client()
    source_edit_id = None
    if source_edit:
        source_path = source_edit["result_path"]
        source_edit_id = source_edit["id"]
    else:
        source_path = displayed_image_reference_path(image) if edit_kind == "angle" else get_original_image_display_path(image)
    if edit_kind == "angle":
        prompt_template = settings["openai_angle_generation_prompt"] if "openai_angle_generation_prompt" in settings.keys() else ""
        prompt_template = prompt_template or default_ai_angle_generation_prompt()
        prompt = (
            prompt_template
            .replace("{angle_name}", angle_name)
            .replace("{listing_title}", listing["title_en"] or listing["stock_number"])
        )
    else:
        prompt = (settings["openai_image_edit_prompt"] or default_ai_image_edit_prompt()).strip()
        if instruction:
            prompt += f"\nADMIN CORRECTION / EXTRA INSTRUCTION:\n{instruction.strip()}\n"
        prompt += f"\nListing title: {listing['title_en'] or listing['stock_number']}.\n"

    def build_content():
        logo_file_id = get_cached_logo_file_id(client, get_settings()) if edit_kind == "replacement" else None
        source_file_id = None
        if not source_edit and edit_kind != "angle":
            refreshed_image = get_db().execute("SELECT * FROM listing_images WHERE id = ?", (image["id"],)).fetchone() or image
            source_file_id = get_cached_image_file_id(client, refreshed_image)
        content = [{"type": "input_text", "text": prompt}]
        if source_file_id:
            content.append({"type": "input_image", "file_id": source_file_id})
        else:
            source_file = static_url_to_path(source_path)
            if not source_file:
                raise RuntimeError("Generated source image is no longer available locally.")
            content.append({"type": "input_image", "image_url": image_data_url(source_file)})
        if logo_file_id:
            content.append({"type": "input_image", "file_id": logo_file_id})
        return content

    try:
        response = client.responses.create(
            model=settings["openai_image_model"] or "gpt-5",
            input=[{"role": "user", "content": build_content()}],
            tools=[image_generation_tool_payload(settings)],
        )
    except Exception as error:
        if not openai_missing_file_error(error):
            raise
        clear_stale_openai_file_ids(image=image if not source_edit and edit_kind != "angle" else None, clear_logo=(edit_kind == "replacement"))
        response = client.responses.create(
            model=settings["openai_image_model"] or "gpt-5",
            input=[{"role": "user", "content": build_content()}],
            tools=[image_generation_tool_payload(settings)],
        )
    result_base64 = extract_response_image_base64(response)
    if not result_base64:
        raise RuntimeError("OpenAI did not return an edited image.")
    result_path = save_ai_edit_result(listing, image, result_base64, edit_kind=edit_kind)
    review = {"status": "", "notes": ""}
    timestamp = now_utc()
    cursor = get_db().execute(
        """
        INSERT INTO ai_image_edits (
            listing_id, source_image_id, source_edit_id, source_local_path, result_path,
            prompt, response_id, edit_kind, review_status, review_notes, status, created_at, updated_at
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)
        """,
        (
            listing["id"],
            image["id"],
            source_edit_id,
            source_path,
            result_path,
            instruction or angle_name or "",
            getattr(response, "id", ""),
            edit_kind,
            review["status"],
            review["notes"],
            timestamp,
            timestamp,
        ),
    )
    get_db().commit()
    return cursor.lastrowid


def pending_ai_edits_for_listing(listing_id):
    return get_db().execute(
        """
        SELECT ai_image_edits.*, listing_images.sort_order, listing_images.alt_text
        FROM ai_image_edits
        LEFT JOIN listing_images ON listing_images.id = ai_image_edits.source_image_id
        WHERE ai_image_edits.listing_id = ? AND ai_image_edits.status = 'pending'
        ORDER BY ai_image_edits.created_at DESC, ai_image_edits.id DESC
        """,
        (listing_id,),
    ).fetchall()


def image_choices_for_ai_reference(listing_id):
    return get_db().execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ?
        ORDER BY is_primary DESC, sort_order, id
        """,
        (listing_id,),
    ).fetchall()


def generated_variants_for_listing(listing_id):
    rows = get_db().execute(
        """
        SELECT sort_order, image_url, local_path, generated_local_path, use_generated_image
        FROM listing_images
        WHERE listing_id = ? AND generated_local_path IS NOT NULL AND generated_local_path != ''
        """,
        (listing_id,),
    ).fetchall()
    variants = {"by_order": {}, "by_image_url": {}, "by_local_name": {}}
    for row in rows:
        value = {
            "generated_local_path": row["generated_local_path"],
            "use_generated_image": row["use_generated_image"],
        }
        variants["by_order"][row["sort_order"]] = value
        if row["image_url"]:
            variants["by_image_url"][row["image_url"]] = value
        local_name = Path(row["local_path"] or "").name
        if local_name:
            variants["by_local_name"][local_name] = value
    return variants


def ai_gallery_images_for_listing(listing_id):
    return get_db().execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ? AND COALESCE(is_ai_generated, 0) = 1
        ORDER BY sort_order, id
        """,
        (listing_id,),
    ).fetchall()


def pick_generated_variant(variants, source_image, stored_local):
    if not variants:
        return {}
    image_url = source_image["image_url"] or ""
    if image_url and image_url in variants["by_image_url"]:
        return variants["by_image_url"][image_url]
    local_name = Path(stored_local or source_image["local_path"] or "").name
    if local_name and local_name in variants["by_local_name"]:
        return variants["by_local_name"][local_name]
    return variants["by_order"].get(source_image["image_order"], {})


def placeholder_image():
    return ""


def get_listing_primary_image(listing_id):
    """Return the best lightweight image for listing cards/admin thumbnails.

    This now prefers generated _thumb.webp files when available, falling back to the
    original image. Full gallery images still use build_listing_gallery(), so the
    listing detail gallery can keep full-size images for the hero/lightbox.
    """
    db = get_db()
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (listing_id,)).fetchone()
    image = db.execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ? AND is_primary = 1
        ORDER BY sort_order, id
        LIMIT 1
        """,
        (listing_id,),
    ).fetchone()
    if image:
        display_path = get_image_display_path(image)
        return preferred_card_image_path(display_path) or display_path
    image = db.execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ?
        ORDER BY sort_order, id
        LIMIT 1
        """,
        (listing_id,),
    ).fetchone()
    if image:
        display_path = get_image_display_path(image)
        return preferred_card_image_path(display_path) or display_path
    if listing:
        # Do not fall back to scanning static/uploads/listings because deleted admin
        # image rows can leave old files on disk. Only use featured_image if it
        # still matches an active/public listing_images row.
        featured_image = local_static_image_path(listing["featured_image"] or "")
        if featured_image:
            row = db.execute(
                """
                SELECT id FROM listing_images
                WHERE listing_id = ?
                  AND COALESCE(is_public, 1) = 1
                  AND (local_path = ? OR generated_local_path = ?)
                LIMIT 1
                """,
                (listing_id, featured_image, featured_image),
            ).fetchone()
            if row:
                return preferred_card_image_path(featured_image) or featured_image
    return placeholder_image()


def card_image_for_listing(listing):
    """Best image path for public listing cards.

    Uses featured_image when it still points at a real local file (remote URLs
    pass through untouched). Falls back to the best listing_images row when
    featured_image is empty or its file has gone missing, so cards keep
    showing a photo instead of the folder placeholder whenever a usable image
    row still exists.
    """
    if not listing:
        return ""
    featured_raw = (listing["featured_image"] or "").strip() if "featured_image" in listing.keys() else ""
    if featured_raw.startswith(("http://", "https://")):
        return featured_raw
    featured = local_static_image_path(featured_raw)
    if featured and static_url_to_path(featured):
        return preferred_card_image_path(featured) or featured
    fallback = get_listing_primary_image(listing["id"])
    if fallback:
        return fallback
    return preferred_card_image_path(featured) or featured


@app.template_filter("card_image")
def card_image_filter(listing):
    """Jinja filter for listing cards: {{ listing|card_image }}."""
    return card_image_for_listing(listing)


def make_image_alt(title, year="", make="", model="", category="", order=1):
    parts = [year, make, model, category]
    label = " ".join(part for part in parts if part).strip() or title
    return f"{label} - image {int(order):03d}"


def seo_image_filename(listing_slug, public_ref, image_order, original_name):
    extension = Path(original_name or "").suffix.lower()
    if extension not in (".jpg", ".jpeg", ".png", ".webp", ".gif"):
        extension = ".jpg"
    company_name = get_settings()["company_name"] or ""
    stem = slugify(f"{company_name}-{listing_slug}-{public_ref}")[:150].strip("-")
    return f"{stem}-{int(image_order):03d}{extension}"


def seo_image_filename_for_listing(listing, image_order, original_name, variant=""):
    extension = Path(original_name or "").suffix.lower()
    if extension not in (".jpg", ".jpeg", ".png", ".webp", ".gif"):
        extension = ".jpg"
    settings = get_settings()
    parts = [
        settings["company_name"] or "vehicles",
        strip_internal_stock_references(listing["title_en"] or listing["slug"] or listing["public_stock_number"], listing),
        listing["public_stock_number"] or "",
    ]
    stem = slugify(" ".join(part for part in parts if part))[:150].strip("-")
    suffix = f"{int(image_order or 1):03d}"
    if variant:
        suffix = f"{suffix}-{slugify(variant)}"
    return f"{stem}-{suffix}{extension}"


def unique_file_target(folder, filename, current_path=None):
    target = folder / filename
    if current_path:
        try:
            if target.resolve() == current_path.resolve():
                return target
        except OSError:
            pass
    if not target.exists():
        return target
    stem = target.stem
    suffix = target.suffix
    counter = 2
    while True:
        candidate = folder / f"{stem}-{counter}{suffix}"
        if not candidate.exists():
            return candidate
        counter += 1


def rename_static_upload_path(path_value, filename):
    source_path = static_url_to_path(path_value)
    if not source_path:
        return path_value or "", False, "File not found in static uploads"
    upload_root = UPLOADS_DIR.resolve()
    try:
        source_path.resolve().relative_to(upload_root)
    except ValueError:
        return path_value or "", False, "File is outside static uploads"
    target_path = unique_file_target(source_path.parent, filename, source_path)
    if target_path.resolve() == source_path.resolve():
        return local_static_image_path(path_value), False, "Already named"
    source_path.rename(target_path)
    new_path = "/" + target_path.resolve().relative_to(BASE_DIR.resolve()).as_posix()
    return new_path, True, ""


def regenerate_image_filenames(listing_ids=None):
    db = get_db()
    params = []
    where = ["COALESCE(is_deleted, 0) = 0"]
    if listing_ids:
        where.append(f"id IN ({', '.join('?' for _ in listing_ids)})")
        params.extend(listing_ids)
    listings = db.execute(f"SELECT * FROM listings WHERE {' AND '.join(where)} ORDER BY id", params).fetchall()
    summary = {"listings": len(listings), "renamed": 0, "updated_rows": 0, "skipped": 0, "messages": []}
    path_map = {}
    for listing in listings:
        images = db.execute(
            "SELECT * FROM listing_images WHERE listing_id = ? ORDER BY sort_order, id",
            (listing["id"],),
        ).fetchall()
        for image in images:
            updates = {}
            fields = []
            if image["local_path"]:
                fields.append(("local_path", image["local_path"], "ai" if image["is_ai_generated"] else ""))
            if image["generated_local_path"] and image["generated_local_path"] != image["local_path"]:
                fields.append(("generated_local_path", image["generated_local_path"], "ai"))
            elif image["generated_local_path"] and image["generated_local_path"] == image["local_path"]:
                fields.append(("generated_local_path", image["generated_local_path"], "ai" if image["is_ai_generated"] else ""))
            for field_name, old_path, variant in fields:
                if old_path in path_map:
                    new_path = path_map[old_path]
                else:
                    filename = seo_image_filename_for_listing(
                        listing,
                        image_order_value(image),
                        old_path,
                        variant=variant,
                    )
                    new_path, renamed, message = rename_static_upload_path(old_path, filename)
                    path_map[old_path] = new_path
                    if renamed:
                        summary["renamed"] += 1
                    elif message and message != "Already named":
                        summary["skipped"] += 1
                        if len(summary["messages"]) < 8:
                            summary["messages"].append(f"{old_path}: {message}")
                updates[field_name] = new_path
            if updates:
                db.execute(
                    """
                    UPDATE listing_images
                    SET local_path = ?, generated_local_path = ?
                    WHERE id = ?
                    """,
                    (
                        updates.get("local_path", image["local_path"]),
                        updates.get("generated_local_path", image["generated_local_path"]),
                        image["id"],
                    ),
                )
                summary["updated_rows"] += 1
    timestamp = now_utc()
    for old_path, new_path in path_map.items():
        if old_path == new_path:
            continue
        db.execute(
            "UPDATE listings SET featured_image = ?, updated_at = ? WHERE featured_image = ?",
            (new_path, timestamp, old_path),
        )
        db.execute(
            "UPDATE ai_image_edits SET result_path = ?, updated_at = ? WHERE result_path = ?",
            (new_path, timestamp, old_path),
        )
        db.execute(
            "UPDATE ai_image_edits SET source_local_path = ?, updated_at = ? WHERE source_local_path = ?",
            (new_path, timestamp, old_path),
        )
    return summary


def archive_unused_listing_files():
    db = get_db()
    referenced = set()
    queries = (
        "SELECT local_path AS path FROM listing_images WHERE local_path IS NOT NULL AND local_path != ''",
        "SELECT generated_local_path AS path FROM listing_images WHERE generated_local_path IS NOT NULL AND generated_local_path != ''",
        "SELECT featured_image AS path FROM listings WHERE featured_image IS NOT NULL AND featured_image != ''",
        "SELECT result_path AS path FROM ai_image_edits WHERE result_path IS NOT NULL AND result_path != ''",
        "SELECT source_local_path AS path FROM ai_image_edits WHERE source_local_path IS NOT NULL AND source_local_path != ''",
    )
    for query in queries:
        for row in db.execute(query).fetchall():
            file_path = static_url_to_path(row["path"])
            if file_path:
                referenced.add(file_path.resolve())
    moved = 0
    skipped = 0
    roots = ((LISTING_UPLOADS_DIR, LISTING_UPLOADS_DIR / "_unused"), (AI_EDIT_UPLOADS_DIR, AI_EDIT_UPLOADS_DIR / "_unused"))
    for root, archive_root in roots:
        archive_root.mkdir(parents=True, exist_ok=True)
        if not root.exists():
            continue
        for folder in root.iterdir():
            if not folder.is_dir() or folder.name == "_unused":
                continue
            for file_path in folder.iterdir():
                if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_EXTENSIONS:
                    continue
                if file_path.resolve() in referenced:
                    continue
                target_folder = archive_root / folder.name
                target_folder.mkdir(parents=True, exist_ok=True)
                target = unique_file_target(target_folder, file_path.name, file_path)
                try:
                    file_path.rename(target)
                    moved += 1
                except OSError:
                    skipped += 1
        for folder in root.iterdir():
            if folder.is_dir() and folder.name != "_unused":
                try:
                    next(folder.iterdir())
                except StopIteration:
                    folder.rmdir()
                except OSError:
                    pass
    return {"moved": moved, "skipped": skipped}


def move_file_and_map(old_path, target_folder, filename, path_map):
    if not old_path:
        return old_path or "", False, ""
    if old_path in path_map:
        return path_map[old_path], False, ""
    source_path = static_url_to_path(old_path)
    if not source_path:
        path_map[old_path] = old_path
        return old_path, False, "File not found"
    target_folder.mkdir(parents=True, exist_ok=True)
    target = unique_file_target(target_folder, filename, source_path)
    if target.resolve() == source_path.resolve():
        new_path = "/" + target.resolve().relative_to(BASE_DIR.resolve()).as_posix()
        path_map[old_path] = new_path
        return new_path, False, "Already moved"
    source_path.rename(target)
    new_path = "/" + target.resolve().relative_to(BASE_DIR.resolve()).as_posix()
    path_map[old_path] = new_path
    return new_path, True, ""


def migrate_ai_edits_to_stock_folders():
    db = get_db()
    listings = db.execute("SELECT * FROM listings ORDER BY id").fetchall()
    summary = {"moved": 0, "updated_rows": 0, "messages": []}
    path_map = {}
    for listing in listings:
        target_folder = ai_edit_folder_for_listing(listing)
        images = db.execute(
            """
            SELECT * FROM listing_images
            WHERE listing_id = ?
              AND (
                  local_path LIKE '/static/uploads/ai_edits/%'
                  OR generated_local_path LIKE '/static/uploads/ai_edits/%'
              )
            ORDER BY sort_order, id
            """,
            (listing["id"],),
        ).fetchall()
        for image in images:
            updates = {}
            for field_name, variant in (("local_path", "ai-angle" if image["is_ai_generated"] else "ai"), ("generated_local_path", "ai")):
                old_path = image[field_name] if field_name in image.keys() else ""
                if not old_path or not old_path.startswith("/static/uploads/ai_edits/"):
                    continue
                filename = seo_image_filename_for_listing(
                    listing,
                    image_order_value(image),
                    old_path,
                    variant=variant,
                )
                new_path, moved, message = move_file_and_map(old_path, target_folder, filename, path_map)
                updates[field_name] = new_path
                if moved:
                    summary["moved"] += 1
                elif message and message not in {"Already moved"} and len(summary["messages"]) < 8:
                    summary["messages"].append(f"{old_path}: {message}")
            if updates:
                db.execute(
                    "UPDATE listing_images SET local_path = ?, generated_local_path = ? WHERE id = ?",
                    (
                        updates.get("local_path", image["local_path"]),
                        updates.get("generated_local_path", image["generated_local_path"]),
                        image["id"],
                    ),
                )
                summary["updated_rows"] += 1
        edits = db.execute(
            """
            SELECT * FROM ai_image_edits
            WHERE listing_id = ?
              AND (
                  result_path LIKE '/static/uploads/ai_edits/%'
                  OR source_local_path LIKE '/static/uploads/ai_edits/%'
              )
            ORDER BY id
            """,
            (listing["id"],),
        ).fetchall()
        for edit in edits:
            updates = {}
            source_image = db.execute("SELECT * FROM listing_images WHERE id = ?", (edit["source_image_id"],)).fetchone()
            order_value = source_image["sort_order"] if source_image else edit["id"]
            for field_name, variant in (("result_path", "ai"), ("source_local_path", "ai-source")):
                old_path = edit[field_name] if field_name in edit.keys() else ""
                if not old_path or not old_path.startswith("/static/uploads/ai_edits/"):
                    continue
                filename = seo_image_filename_for_listing(listing, order_value, old_path, variant=variant)
                new_path, moved, message = move_file_and_map(old_path, target_folder, filename, path_map)
                updates[field_name] = new_path
                if moved:
                    summary["moved"] += 1
                elif message and message not in {"Already moved"} and len(summary["messages"]) < 8:
                    summary["messages"].append(f"{old_path}: {message}")
            if updates:
                db.execute(
                    "UPDATE ai_image_edits SET result_path = ?, source_local_path = ?, updated_at = ? WHERE id = ?",
                    (
                        updates.get("result_path", edit["result_path"]),
                        updates.get("source_local_path", edit["source_local_path"]),
                        now_utc(),
                        edit["id"],
                    ),
                )
        for old_path, new_path in path_map.items():
            if old_path == new_path:
                continue
            db.execute("UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ? AND featured_image = ?", (new_path, now_utc(), listing["id"], old_path))
    return summary


def sanitize_public_listing_content(regenerate_slugs=True):
    db = get_db()
    rows = db.execute("SELECT * FROM listings ORDER BY id").fetchall()
    changed = 0
    timestamp = now_utc()
    fields = ("title_en", "short_description_en", "description_html_en", "seo_title_en", "meta_description_en")
    for listing in rows:
        updates = {}
        for field in fields:
            cleaned = strip_internal_stock_references(listing[field] or "", listing)
            if cleaned != (listing[field] or ""):
                updates[field] = cleaned
        if regenerate_slugs:
            new_slug = public_slug_for_listing({**dict(listing), **updates})
            if new_slug != listing["slug"]:
                updates["slug"] = new_slug
        if updates:
            db.execute(
                f"UPDATE listings SET {', '.join(f'{field} = ?' for field in updates)}, updated_at = ? WHERE id = ?",
                (*updates.values(), timestamp, listing["id"]),
            )
            changed += 1
    return changed


def image_filename_order_number(path_value):
    """Return the final numeric image suffix from a stored image filename.

    Examples:
        clean-agri-sales-merlo-roto-cas-1082-007.jpg -> 7
        clean-agri-sales-merlo-roto-cas-1082-008-ai.jpg -> 8

    The repair button uses this value to recover the intended filename order
    after uploads/syncs, instead of trusting stale database sort_order values.
    """
    if not path_value:
        return None
    value = str(path_value).strip().replace("\\", "/")
    if not value:
        return None
    stem = Path(value.split("?")[0]).stem.lower()
    for suffix in (THUMBNAIL_SUFFIX, GALLERY_SUFFIX):
        if stem.endswith(suffix):
            stem = stem[: -len(suffix)]
    matches = re.findall(r"(?:^|[-_])(\d{1,5})(?=$|[-_])", stem)
    if not matches:
        return None
    try:
        return int(matches[-1])
    except ValueError:
        return None


def image_order_value(image):
    """Sort order for naming/sorting, falling back to the row id only when
    sort_order is NULL. (A plain `sort_order or id` treats a legitimate
    sort_order of 0 as missing and names files after the row id instead.)"""
    order = image["sort_order"] if "sort_order" in image.keys() else None
    if order is None or order == "":
        return image["id"]
    return order


def image_repair_sort_key(image):
    # Sort by ORIGINAL local_path first. Active AI replacements can have a different
    # filename/order number, but the row should keep the original gallery position.
    candidates = [
        image["local_path"] if "local_path" in image.keys() else "",
        image["generated_local_path"] if "generated_local_path" in image.keys() else "",
        get_image_display_path(image),
        image["image_url"] if "image_url" in image.keys() else "",
    ]
    for candidate in candidates:
        number = image_filename_order_number(candidate)
        if number is not None:
            return (0, number, image["id"])
    return (1, image_order_value(image), image["id"])


def repair_image_order_for_listing(listing_id):
    db = get_db()
    images = db.execute(
        "SELECT * FROM listing_images WHERE listing_id = ? ORDER BY sort_order, id", (listing_id,)
    ).fetchall()
    if not images:
        db.execute("UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ?", ("", now_utc(), listing_id))
        return False

    ordered_images = sorted(images, key=image_repair_sort_key)
    timestamp = now_utc()
    db.execute("UPDATE listing_images SET is_primary = 0 WHERE listing_id = ?", (listing_id,))
    for sort_order, image in enumerate(ordered_images, start=1):
        db.execute(
            "UPDATE listing_images SET sort_order = ?, is_primary = ? WHERE id = ? AND listing_id = ?",
            (sort_order, 1 if sort_order == 1 else 0, image["id"], listing_id),
        )
    featured = get_image_display_path(ordered_images[0])
    db.execute("UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ?", (featured, timestamp, listing_id))
    return True


def sync_missing_folder_images_for_listing(listing_id):
    """Explicitly add image files from the listing folder that are missing DB rows.

    This is NOT automatic on public pages, so admin-deleted images do not reappear
    unless the admin intentionally clicks the sync button.
    """
    db = get_db()
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (listing_id,)).fetchone()
    if not listing:
        return {"added": 0, "skipped": 0}

    existing_names = set()
    rows = db.execute(
        """
        SELECT local_path, generated_local_path
        FROM listing_images
        WHERE listing_id = ?
        """,
        (listing_id,),
    ).fetchall()
    for row in rows:
        for field in ("local_path", "generated_local_path"):
            value = row[field] or ""
            if value:
                existing_names.add(Path(value).name.lower())

    added = 0
    skipped = 0
    timestamp = now_utc()
    folder_paths = []
    for folder_name in listing_upload_folder_candidates(listing):
        folder = LISTING_UPLOADS_DIR / folder_name
        if folder.exists() and folder.is_dir():
            folder_paths.append((folder_name, folder))
    for folder_name, folder in folder_paths:
        for file_path in sorted(folder.iterdir(), key=lambda item: (image_filename_order_number(item.name) or 999999, item.name.lower())):
            if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_EXTENSIONS:
                continue
            if file_path.stem.endswith((THUMBNAIL_SUFFIX, GALLERY_SUFFIX)):
                continue
            if file_path.name.lower() in existing_names:
                skipped += 1
                continue
            try:
                local_path = "/" + file_path.resolve().relative_to(BASE_DIR.resolve()).as_posix()
            except ValueError:
                skipped += 1
                continue
            number = image_filename_order_number(local_path)
            next_order = number or db.execute(
                "SELECT COALESCE(MAX(sort_order), 0) + 1 FROM listing_images WHERE listing_id = ?",
                (listing_id,),
            ).fetchone()[0]
            db.execute(
                """
                INSERT INTO listing_images (
                    listing_id, image_url, local_path, alt_text, sort_order, is_primary,
                    file_size, checksum_sha256, download_status, openai_file_id,
                    generated_local_path, use_generated_image, is_ai_generated, is_public, created_at
                )
                VALUES (?, '', ?, ?, ?, 0, ?, ?, 'folder-sync', '', '', 0, 0, 1, ?)
                """,
                (
                    listing_id,
                    local_path,
                    make_image_alt(
                        listing["title_en"],
                        listing["year"] or "",
                        listing["make"] or "",
                        listing["model"] or "",
                        listing["category"] or "",
                        next_order,
                    ),
                    next_order,
                    file_path.stat().st_size,
                    hashlib.sha256(file_path.read_bytes()).hexdigest(),
                    timestamp,
                ),
            )
            generate_image_derivatives_for_path(local_path, regenerate=False)
            existing_names.add(file_path.name.lower())
            added += 1
        if added:
            break

    if added:
        repair_image_order_for_listing(listing_id)
    return {"added": added, "skipped": skipped}


def sync_listing_featured_image_from_primary(listing_id):
    db = get_db()
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (listing_id,)).fetchone()
    image = db.execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ?
        ORDER BY is_primary DESC, sort_order, id
        LIMIT 1
        """,
        (listing_id,),
    ).fetchone()
    featured = get_image_display_path(image) if image else ""
    if image and not image["is_primary"]:
        db.execute("UPDATE listing_images SET is_primary = 0 WHERE listing_id = ?", (listing_id,))
        db.execute("UPDATE listing_images SET is_primary = 1 WHERE id = ?", (image["id"],))
    db.execute("UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ?", (featured, now_utc(), listing_id))
    return bool(featured)


def repair_all_image_orders(listing_ids=None):
    db = get_db()
    if listing_ids:
        rows = [{"id": int(value)} for value in listing_ids]
    else:
        rows = db.execute("SELECT id FROM listings").fetchall()
    repaired = 0
    for row in rows:
        if repair_image_order_for_listing(row["id"]):
            repaired += 1
    db.commit()
    return repaired


def load_source_preview(source_db, search=""):
    source_db_path = Path(source_db).expanduser()
    result = {"rows": [], "error": "", "total": 0, "source_total": 0}
    if not source_db_path.exists():
        result["error"] = f"Source DB not found: {source_db}"
        return result
    try:
        source_conn = sqlite3.connect(str(source_db_path))
        source_conn.row_factory = sqlite3.Row
    except sqlite3.Error as error:
        result["error"] = f"Could not open source DB: {error}"
        return result
    try:
        if not source_table_exists(source_conn, "vehicles"):
            result["error"] = "Source DB is missing vehicles table."
            return result
        image_counts = {}
        if source_table_exists(source_conn, "vehicles_images"):
            for row in source_conn.execute("SELECT stock_number, COUNT(*) AS count FROM vehicles_images GROUP BY stock_number"):
                image_counts[row["stock_number"]] = row["count"]
        rows = source_conn.execute("SELECT * FROM vehicles ORDER BY updated_at DESC, stock_number").fetchall()
        result["source_total"] = len(rows)
        search_lower = (search or "").lower()
        imported = {
            row["stock_number"]
            for row in get_db().execute(
                """
                SELECT stock_number FROM listings
                WHERE stock_number IS NOT NULL AND COALESCE(is_deleted, 0) = 0
                """
            ).fetchall()
        }
        for row in rows:
            title = row["clean_title"] or row["page_title"] or row["stock_number"]
            haystack = " ".join([row["stock_number"] or "", title or "", row["year"] or ""]).lower()
            if search_lower and search_lower not in haystack:
                continue
            result["rows"].append(
                {
                    "stock_number": row["stock_number"],
                    "title": title,
                    "year": row["year"] or "",
                    "hours": row["hours"] or "",
                    "price": row["price"] or "",
                    "image_count": image_counts.get(row["stock_number"], row["image_count"] or 0),
                    "already_imported": row["stock_number"] in imported,
                    "source_url": row["source_url"] or "",
                }
            )
        result["total"] = len(result["rows"])
        return result
    finally:
        source_conn.close()


def import_inventory_from_source(
    source_db,
    copy_images=False,
    overwrite=False,
    publish=False,
    default_status="Available",
    dry_run=False,
    selected_stocks=None,
    limit=None,
    missing_only=False,
    repair_after_import=False,
    generated_image_mode="source",
    generate_variants_after_import=True,
):
    summary = {
        "total_source_listings": 0,
        "listings_selected": 0,
        "listings_created": 0,
        "listings_updated": 0,
        "listings_skipped": 0,
        "images_found": 0,
        "images_imported": 0,
        "images_copied": 0,
        "missing_image_files": 0,
        "primary_images_set": 0,
        "featured_images_updated": 0,
        "repaired_listings": 0,
        "image_variants_generated": 0,
        "errors": [],
        "dry_run": dry_run,
    }
    source_db_path = Path(source_db).expanduser()
    if not source_db_path.exists():
        summary["errors"].append(f"Source DB not found: {source_db}")
        return summary

    try:
        source_conn = sqlite3.connect(str(source_db_path))
        source_conn.row_factory = sqlite3.Row
    except sqlite3.Error as error:
        summary["errors"].append(f"Could not open source DB: {error}")
        return summary

    try:
        if not source_table_exists(source_conn, "vehicles"):
            summary["errors"].append("Source DB is missing vehicles table.")
            return summary

        has_images = source_table_exists(source_conn, "vehicles_images")
        source_rows = source_conn.execute("SELECT * FROM vehicles ORDER BY updated_at DESC, stock_number").fetchall()
        summary["total_source_listings"] = len(source_rows)
        db = get_db()
        selected_set = {stock.strip() for stock in (selected_stocks or []) if stock and stock.strip()}
        if selected_set:
            source_rows = [row for row in source_rows if row["stock_number"] in selected_set]
        if missing_only:
            imported = {
                row["stock_number"]
                for row in db.execute(
                    """
                    SELECT stock_number FROM listings
                    WHERE stock_number IS NOT NULL AND COALESCE(is_deleted, 0) = 0
                    """
                ).fetchall()
            }
            source_rows = [row for row in source_rows if row["stock_number"] not in imported]
        if limit:
            source_rows = source_rows[: int(limit)]
        summary["listings_selected"] = len(source_rows)
        changed_listing_ids = []

        for source in source_rows:
            try:
                stock_number = (source["stock_number"] or "").strip()
                if not stock_number:
                    summary["listings_skipped"] += 1
                    continue
                title = (source["clean_title"] or source["page_title"] or stock_number).strip()
                make = infer_make(title)
                category = infer_category(title)
                model = infer_model(title, make)
                description = (source["description"] or "").strip()
                markdown_html = markdown_to_basic_html(source["markdown"])
                description_html = f"<p>{escape(description)}</p>" if description else markdown_html
                if not description_html:
                    description_html = f"<p>{escape(title)}</p>"
                short_description = short_text(description or source["markdown"] or title)
                price = (source["price"] or "").strip()
                price_label = "" if price else "POA"
                existing = db.execute("SELECT * FROM listings WHERE stock_number = ?", (stock_number,)).fetchone()
                public_ref = existing["public_stock_number"] if existing and existing["public_stock_number"] else ""
                listing_created_at = source["created_at"] or now_utc()
                listing_updated_at = source["updated_at"] or now_utc()

                is_deleted = bool(existing and existing["is_deleted"] if "is_deleted" in existing.keys() else 0)
                if existing and not overwrite and not is_deleted:
                    summary["listings_skipped"] += 1
                    continue

                if existing:
                    listing_id = existing["id"]
                    slug = existing["slug"] or unique_slug(f"{source['year']} {title}", public_ref, listing_id)
                    generated_variants = generated_variants_for_listing(listing_id) if generated_image_mode == "source_with_generated" else {}
                    ai_gallery_images = ai_gallery_images_for_listing(listing_id) if generated_image_mode == "source_with_generated" else []
                    if not dry_run:
                        db.execute(
                            """
                            UPDATE listings
                            SET public_stock_number = COALESCE(NULLIF(public_stock_number, ''), ?), slug = ?, title_en = ?, short_description_en = ?, description_html_en = ?,
                                seo_title_en = ?, meta_description_en = ?, year = ?, make = ?, model = ?,
                                category = ?, hours = ?, price = ?, price_label = ?, status = ?,
                                source_url = ?, source_metadata_json = ?, folder_path = ?,
                                is_published = ?, is_deleted = 0, updated_at = ?
                            WHERE id = ?
                            """,
                            (
                                public_ref,
                                slug,
                                title,
                                short_description,
                                description_html,
                                title,
                                short_description,
                                source["year"] or "",
                                make,
                                model,
                                category,
                                source["hours"] or "",
                                price,
                                price_label,
                                default_status or existing["status"] or "Available",
                                source["source_url"] or "",
                                source["metadata_json"] or "",
                                source["folder_path"] or "",
                                1 if publish else 0,
                                listing_updated_at,
                                listing_id,
                            ),
                        )
                        db.execute("DELETE FROM listing_images WHERE listing_id = ?", (listing_id,))
                    summary["listings_updated"] += 1
                else:
                    slug = unique_slug(f"{source['year']} {title}", stock_number)
                    listing_id = None
                    generated_variants = {}
                    ai_gallery_images = []
                    if not dry_run:
                        cursor = db.execute(
                            """
                            INSERT INTO listings (
                                stock_number, public_stock_number, slug, title_en, short_description_en, description_html_en,
                                seo_title_en, meta_description_en, year, make, model, category, hours, price,
                                price_label, status, source_url, source_metadata_json, folder_path,
                                is_featured, is_published, created_at, updated_at
                            )
                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)
                            """,
                            (
                                stock_number,
                                public_ref,
                                slug,
                                title,
                                short_description,
                                description_html,
                                title,
                                short_description,
                                source["year"] or "",
                                make,
                                model,
                                category,
                                source["hours"] or "",
                                price,
                                price_label,
                                default_status or "Available",
                                source["source_url"] or "",
                                source["metadata_json"] or "",
                                source["folder_path"] or "",
                                1 if publish else 0,
                                listing_created_at,
                                listing_updated_at,
                            ),
                        )
                        listing_id = cursor.lastrowid
                        public_ref = ensure_public_stock_number(listing_id)
                    summary["listings_created"] += 1

                if listing_id and not dry_run:
                    public_ref = ensure_public_stock_number(listing_id)

                if has_images:
                    image_rows = source_conn.execute(
                        """
                        SELECT * FROM vehicles_images
                        WHERE stock_number = ?
                        ORDER BY image_order
                        """,
                        (stock_number,),
                    ).fetchall()
                    summary["images_found"] += len(image_rows)
                    first_featured = ""
                    primary_set = False
                    for image in image_rows:
                        image_url = image["image_url"] or ""
                        original_local = image["local_path"] or ""
                        stored_local = local_static_image_path(original_local)
                        source_file = None
                        if original_local:
                            source_file = resolve_source_image(original_local, source_db_path, source["folder_path"])
                            if source_file is None:
                                summary["missing_image_files"] += 1
                        if copy_images and source_file is not None:
                            safe_stock = secure_filename(stock_number) or stock_number
                            target_dir = LISTING_UPLOADS_DIR / safe_stock
                            target_name = seo_image_filename(slug, public_ref, image["image_order"], source_file.name)
                            target_file = target_dir / target_name
                            stored_local = f"/static/uploads/listings/{safe_stock}/{target_name}"
                            if not dry_run:
                                target_dir.mkdir(parents=True, exist_ok=True)
                                if overwrite or not target_file.exists():
                                    shutil.copy2(source_file, target_file)
                                    summary["images_copied"] += 1
                        usable_image = stored_local
                        if not usable_image:
                            continue
                        if not first_featured:
                            generated_variant = pick_generated_variant(generated_variants, image, stored_local)
                            first_featured = generated_variant.get("generated_local_path") if generated_image_mode == "source_with_generated" else ""
                            first_featured = first_featured or usable_image
                        is_primary = 0 if primary_set else 1
                        primary_set = True
                        if not dry_run and listing_id:
                            duplicate = db.execute(
                                """
                                SELECT id FROM listing_images
                                WHERE listing_id = ? AND COALESCE(is_ai_generated, 0) = 0
                                  AND (sort_order = ? OR image_url = ? OR local_path = ?)
                                """,
                                (listing_id, image["image_order"], image_url, stored_local),
                            ).fetchone()
                            if duplicate is None:
                                generated_variant = pick_generated_variant(generated_variants, image, stored_local)
                                generated_path = generated_variant.get("generated_local_path", "")
                                use_generated = 1 if generated_path and generated_image_mode == "source_with_generated" else 0
                                db.execute(
                                    """
                                    INSERT INTO listing_images (
                                        listing_id, image_url, local_path, alt_text, sort_order, is_primary,
                                        file_size, checksum_sha256, download_status, openai_file_id,
                                        generated_local_path, use_generated_image, created_at
                                    )
                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '', ?, ?, ?)
                                    """,
                                    (
                                        listing_id,
                                        image_url,
                                        stored_local,
                                        make_image_alt(title, source["year"] or "", make, model, category, image["image_order"]),
                                        image["image_order"],
                                        is_primary,
                                        image["file_size"] if "file_size" in image.keys() else None,
                                        image["checksum_sha256"] if "checksum_sha256" in image.keys() else "",
                                        image["download_status"] if "download_status" in image.keys() else "",
                                        generated_path,
                                        use_generated,
                                        now_utc(),
                                    ),
                                )
                                summary["images_imported"] += 1
                                if is_primary:
                                    summary["primary_images_set"] += 1
                    if first_featured and not dry_run and listing_id:
                        db.execute("UPDATE listings SET featured_image = ? WHERE id = ?", (first_featured, listing_id))
                        summary["featured_images_updated"] += 1
                    if generated_image_mode == "source_with_generated" and ai_gallery_images and not dry_run and listing_id:
                        next_order = (len(image_rows) if has_images else 0) + 1
                        for ai_image in ai_gallery_images:
                            db.execute(
                                """
                                INSERT INTO listing_images (
                                    listing_id, image_url, local_path, alt_text, sort_order, is_primary,
                                    file_size, checksum_sha256, download_status, openai_file_id,
                                    generated_local_path, use_generated_image, is_ai_generated, is_public, created_at
                                )
                                VALUES (?, '', ?, ?, ?, 0, NULL, '', 'ai-generated-angle', '', ?, 1, 1, ?, ?)
                                """,
                                (
                                    listing_id,
                                    ai_image["local_path"],
                                    ai_image["alt_text"] or make_image_alt(title, source["year"] or "", make, model, category, next_order),
                                    next_order,
                                    ai_image["generated_local_path"] or ai_image["local_path"],
                                    ai_image["is_public"] if "is_public" in ai_image.keys() else 1,
                                    now_utc(),
                                ),
                            )
                            next_order += 1
                if listing_id:
                    changed_listing_ids.append(listing_id)

                if not dry_run:
                    db.commit()
            except Exception as error:
                summary["errors"].append(f"{source['stock_number']}: {error}")
                if not dry_run:
                    db.rollback()
        if repair_after_import and changed_listing_ids and not dry_run:
            summary["repaired_listings"] = repair_all_image_orders(changed_listing_ids)
        if generate_variants_after_import and changed_listing_ids and not dry_run:
            derivative_summary = generate_image_derivatives_for_listings(changed_listing_ids, regenerate=False)
            summary["image_variants_generated"] = derivative_summary["written"]
            if derivative_summary["errors"]:
                summary["errors"].extend(derivative_summary["errors"][:12])
        return summary
    finally:
        source_conn.close()


def import_cars_from_source(
    source_db,
    copy_images=False,
    overwrite=False,
    publish=False,
    default_status="Available",
    dry_run=False,
    selected_vins=None,
    limit=None,
    missing_only=False,
):
    """Import vehicles from canada/vehicles.sqlite3 into the car dealer site DB."""
    summary = {
        "total_source_vehicles": 0,
        "vehicles_selected": 0,
        "listings_created": 0,
        "listings_updated": 0,
        "listings_skipped": 0,
        "images_found": 0,
        "images_imported": 0,
        "images_copied": 0,
        "images_linked_remote": 0,
        "missing_image_files": 0,
        "missing_image_vins": {},
        "listings_without_local_images": [],
        "primary_images_set": 0,
        "errors": [],
        "dry_run": dry_run,
    }
    source_db_path = Path(source_db).expanduser()
    if not source_db_path.exists():
        summary["errors"].append(f"Source DB not found: {source_db}")
        return summary

    try:
        source_conn = sqlite3.connect(str(source_db_path))
        source_conn.row_factory = sqlite3.Row
    except sqlite3.Error as error:
        summary["errors"].append(f"Could not open source DB: {error}")
        return summary

    try:
        # Check that the vehicles table exists
        tables = [r[0] for r in source_conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table'"
        ).fetchall()]
        if "vehicles" not in tables:
            summary["errors"].append("Source DB is missing vehicles table.")
            return summary

        has_images = "vehicle_images" in tables
        has_features = "vehicle_features" in tables
        has_attributes = "vehicle_attributes" in tables

        source_rows = source_conn.execute(
            "SELECT * FROM vehicles ORDER BY updated_at DESC, make, model"
        ).fetchall()
        summary["total_source_vehicles"] = len(source_rows)

        db = get_db()
        if selected_vins:
            vin_set = {v.strip().upper() for v in selected_vins if v and v.strip()}
            source_rows = [r for r in source_rows if (r["vin"] or "").strip().upper() in vin_set]
        if missing_only:
            imported = {
                (r["vin"] or "").strip().upper()
                for r in db.execute(
                    "SELECT vin FROM listings WHERE vin IS NOT NULL AND COALESCE(is_deleted,0)=0"
                ).fetchall()
            }
            source_rows = [r for r in source_rows if (r["vin"] or "").strip().upper() not in imported]
        if limit:
            source_rows = source_rows[:int(limit)]
        summary["vehicles_selected"] = len(source_rows)

        for source in source_rows:
            try:
                vin = (source["vin"] or "").strip()
                if not vin:
                    summary["listings_skipped"] += 1
                    continue

                make = (source["make"] or "").strip()
                model = (source["model"] or "").strip()
                year = str(source["model_year"] or "").strip()
                trim = (source["trim"] or "").strip()
                body_style = (source["body_style"] or source["normal_body_style"] or "").strip()
                odometer = source["odometer"]
                transmission = (source["transmission"] or "").strip()
                fuel_type = (source["fuel_type"] or "").strip()
                drive_line = (source["drive_line"] or "").strip()
                exterior_color = (source["exterior_color"] or "").strip()
                interior_color = (source["interior_color"] or "").strip()
                engine = f"{source['engine_size'] or ''} {source['engine'] or ''}".strip()
                doors = source["doors"]
                certified = 1 if ("certified" in source.keys() and source["certified"]) else 0

                # Price: prefer asking_price, fall back to internet_price
                price = (source["asking_price"] if "asking_price" in source.keys() and source["asking_price"] else source["internet_price"] if "internet_price" in source.keys() and source["internet_price"] else None)
                msrp = source["msrp"] if "msrp" in source.keys() else None
                price_label = "" if price else "POA"
                savings = None
                if msrp and price and msrp > price:
                    savings = round(msrp - price, 2)

                # Title: use AI-generated if available, otherwise build
                ai_title = (source["ai_title"] if "ai_title" in source.keys() and source["ai_title"] else "").strip()
                title = ai_title if ai_title else f"{year} {make} {model} {trim}".strip()
                if not title:
                    title = f"{make} {model}" if make else f"Vehicle {vin[:8]}"

                # Description: use AI-generated HTML if available
                description_html = (source["ai_description_html"] if "ai_description_html" in source.keys() and source["ai_description_html"] else "").strip()
                if not description_html:
                    features_flat = (source["features_flat"] if "features_flat" in source.keys() and source["features_flat"] else "").strip()
                    description_html = f"<p>{escape(features_flat) if features_flat else escape(title)}</p>"
                short_description = short_text(description_html or title)

                # Carfax URL from vehicle_attributes
                carfax_url = ""
                if has_attributes:
                    attr_row = source_conn.execute(
                        "SELECT attr_value_text FROM vehicle_attributes WHERE vin=? AND attr_path LIKE '%carfax%' LIMIT 1",
                        (vin,)
                    ).fetchone()
                    if attr_row:
                        carfax_url = (attr_row["attr_value_text"] or "").strip()

                # Status mapping
                source_status = (source["status"] if "status" in source.keys() and source["status"] else "").strip().lower()
                if source_status in ("sold", "reserved"):
                    listing_status = source_status.capitalize()
                else:
                    listing_status = default_status or "Available"

                # Category from body_style
                category = body_style if body_style else "Vehicle"

                # Check existing by VIN
                existing = db.execute("SELECT * FROM listings WHERE vin = ?", (vin,)).fetchone()
                if existing and not overwrite:
                    summary["listings_skipped"] += 1
                    continue

                # Public stock number
                public_ref = ""
                if existing and existing["public_stock_number"]:
                    public_ref = existing["public_stock_number"]

                if existing:
                    listing_id = existing["id"]
                    slug = existing["slug"] or unique_slug(title, public_ref, listing_id)
                    if not dry_run:
                        db.execute(
                            """UPDATE listings SET
                                public_stock_number=COALESCE(NULLIF(public_stock_number,''),?),
                                slug=?, title_en=?, short_description_en=?, description_html_en=?,
                                seo_title_en=?, meta_description_en=?, year=?, make=?, model=?,
                                category=?, odometer=?, price=?, price_label=?, status=?,
                                source_url=?, source_metadata_json=?, folder_path=?,
                                vin=?, transmission=?, fuel_type=?, body_style=?, drive_line=?,
                                exterior_color=?, interior_color=?, doors=?, engine=?, trim=?,
                                certified=?, carfax_url=?, msrp=?, savings=?,
                                is_published=?, is_deleted=0, updated_at=?
                            WHERE id=?""",
                            (public_ref, slug, title, short_description, description_html,
                             title, short_description, year, make, model,
                             category, odometer, str(price) if price else "", price_label, listing_status,
                             source["source_url"] or "", source["normalized_payload_json"] if "normalized_payload_json" in source.keys() and source["normalized_payload_json"] else "", "",
                             vin, transmission, fuel_type, body_style, drive_line,
                             exterior_color, interior_color, doors, engine, trim,
                             certified, carfax_url, msrp, savings,
                             1 if publish else (existing["is_published"] if existing else 0), now_utc(),
                             listing_id))
                        db.execute("DELETE FROM listing_images WHERE listing_id=?", (listing_id,))
                    summary["listings_updated"] += 1
                else:
                    slug = unique_slug(title, public_ref)
                    listing_id = None
                    if not dry_run:
                        cursor = db.execute(
                            """INSERT INTO listings (
                                stock_number, public_stock_number, slug, title_en, short_description_en,
                                description_html_en, seo_title_en, meta_description_en, year, make, model,
                                category, odometer, price, price_label, status, source_url,
                                source_metadata_json, folder_path, vin, transmission, fuel_type,
                                body_style, drive_line, exterior_color, interior_color, doors,
                                engine, trim, certified, carfax_url, msrp, savings,
                                is_published, is_featured, created_at, updated_at
                            ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                            (vin, public_ref, slug, title, short_description,
                             description_html, title, short_description, year, make, model,
                             category, odometer, str(price) if price else "", price_label, listing_status,
                             source["source_url"] or "",
                             (source["normalized_payload_json"] if "normalized_payload_json" in source.keys() and source["normalized_payload_json"] else ""),
                             "", vin, transmission, fuel_type,
                             body_style, drive_line, exterior_color, interior_color, doors,
                             engine, trim, certified, carfax_url, msrp, savings,
                             1 if publish else 0, 0, now_utc(), now_utc()))
                        listing_id = cursor.lastrowid
                        public_ref = ensure_public_stock_number(listing_id)
                    summary["listings_created"] += 1

                if listing_id and not dry_run:
                    public_ref = ensure_public_stock_number(listing_id)

                # ── Import images ──────────────────────────────────────────
                if has_images and listing_id:
                    image_rows = source_conn.execute(
                        "SELECT * FROM vehicle_images WHERE vin=? AND download_status='downloaded' ORDER BY image_order",
                        (vin,)
                    ).fetchall()
                    summary["images_found"] += len(image_rows)

                    # Build SEO folder name
                    safe_year = re.sub(r'[<>:"/\\|?*]', '', year)
                    safe_make = re.sub(r'[<>:"/\\|?*]', '', make).replace(' ', '-')
                    safe_model = re.sub(r'[<>:"/\\|?*]', '', model).replace(' ', '-')
                    safe_ref = (public_ref or f"ca-{listing_id}").replace(' ', '-')
                    safe_folder = f"{safe_year}-{safe_make}-{safe_model}-{safe_ref}".strip('-').lower()

                    first_featured = ""
                    primary_set = False

                    for img in image_rows:
                        local_path = (img["local_path"] or "").strip()
                        image_url = (img["best_url"] or "").strip()
                        stored_local = ""

                        if local_path and copy_images:
                            # Resolve relative to the source DB parent directory. Normalize
                            # Windows backslashes so the path resolves on Linux too.
                            norm_local = local_path.replace("\\", "/")
                            source_path = Path(norm_local)
                            if not source_path.is_absolute():
                                source_path = source_db_path.parent / norm_local
                            if source_path.exists():
                                ext = source_path.suffix.lower()
                                target_name = f"{img['image_order']:03d}{ext}"
                                target_dir = LISTING_UPLOADS_DIR / safe_folder
                                target_file = target_dir / target_name
                                stored_local = f"/static/uploads/listings/{safe_folder}/{target_name}"
                                if not dry_run:
                                    target_dir.mkdir(parents=True, exist_ok=True)
                                    if overwrite or not target_file.exists():
                                        shutil.copy2(source_path, target_file)
                                        summary["images_copied"] += 1
                            else:
                                summary["missing_image_files"] += 1
                                vin_missing = summary["missing_image_vins"].setdefault(vin, 0)
                                summary["missing_image_vins"][vin] = vin_missing + 1
                                continue
                        elif image_url:
                            stored_local = image_url

                        if not stored_local:
                            continue

                        if not first_featured:
                            first_featured = stored_local

                        is_primary = 0 if primary_set else 1
                        primary_set = True

                        if not dry_run:
                            db.execute(
                                """INSERT INTO listing_images (
                                    listing_id, image_url, local_path, alt_text, sort_order,
                                    is_primary, file_size, checksum_sha256, download_status, created_at
                                ) VALUES (?,?,?,?,?,?,?,?,?,?)""",
                                (listing_id, image_url, stored_local,
                                 f"{year} {make} {model} {trim} — photo {img['image_order']+1}".strip(),
                                 img["image_order"], is_primary,
                                 img["file_size"] if "file_size" in img.keys() else None, img["checksum_sha256"] if "checksum_sha256" in img.keys() else "" or "",
                                 img["download_status"] if "download_status" in img.keys() else "" or "", now_utc()))
                            summary["images_imported"] += 1
                            if stored_local.startswith(("http://", "https://")):
                                summary["images_linked_remote"] += 1
                            if is_primary:
                                summary["primary_images_set"] += 1

                    if first_featured and not dry_run:
                        db.execute("UPDATE listings SET featured_image=? WHERE id=?",
                                   (first_featured, listing_id))
                        db.execute("UPDATE listings SET folder_path=? WHERE id=?",
                                   (safe_folder, listing_id))
                    if not dry_run and listing_id:
                        local_row_count = db.execute(
                            "SELECT COUNT(*) FROM listing_images WHERE listing_id=? AND local_path LIKE '/static/%'",
                            (listing_id,),
                        ).fetchone()[0]
                        if local_row_count == 0:
                            summary["listings_without_local_images"].append(f"{title} ({vin})")

                if not dry_run:
                    db.commit()

            except Exception as error:
                summary["errors"].append(f"VIN {source['vin'] if 'vin' in source.keys() and source['vin'] else '?'}: {error}")
                if not dry_run:
                    db.rollback()

        return summary
    finally:
        source_conn.close()


def repair_crawler_images_for_listings(source_db):
    """Re-copy photos for EXISTING listings from a crawler DB — images only.

    Nothing else about a listing is touched: titles, descriptions, prices,
    status, publish state, translations and inspection reports all stay as
    they are, and soft-deleted listings are skipped entirely. A listing is
    repaired when none of its non-AI listing_images rows point at a local
    file that still exists (empty galleries, remote-URL rows from an import
    without "Copy images", or rows whose files went missing). Its broken
    rows are then replaced with fresh local copies from the crawler source.
    """
    summary = {
        "listings_checked": 0,
        "listings_repaired": 0,
        "images_copied": 0,
        "rows_replaced": 0,
        "missing_image_files": 0,
        "missing_image_vins": {},
        "listings_without_local_images": [],
        "errors": [],
    }
    source_db_path = Path(source_db).expanduser()
    if not source_db_path.exists():
        summary["errors"].append(f"Source DB not found: {source_db}")
        return summary
    try:
        source_conn = sqlite3.connect(str(source_db_path))
        source_conn.row_factory = sqlite3.Row
    except sqlite3.Error as error:
        summary["errors"].append(f"Could not open source DB: {error}")
        return summary

    try:
        tables = [r[0] for r in source_conn.execute(
            "SELECT name FROM sqlite_master WHERE type='table'"
        ).fetchall()]
        if "vehicle_images" not in tables:
            summary["errors"].append("Source DB is missing vehicle_images table.")
            return summary

        db = get_db()
        listings = db.execute(
            "SELECT * FROM listings WHERE COALESCE(is_deleted, 0) = 0 ORDER BY id"
        ).fetchall()
        summary["listings_checked"] = len(listings)

        for listing in listings:
            try:
                vin = (listing["vin"] or listing["stock_number"] or "").strip()
                if not vin:
                    continue
                rows = db.execute(
                    "SELECT * FROM listing_images WHERE listing_id = ? ORDER BY sort_order, id",
                    (listing["id"],),
                ).fetchall()

                replace_ids = []
                has_local_file = False
                kept_primary = False
                for row in rows:
                    local_path = (row["local_path"] or "").strip()
                    is_ai = bool(row["is_ai_generated"]) if "is_ai_generated" in row.keys() else False
                    if is_ai:
                        kept_primary = kept_primary or bool(row["is_primary"])
                        continue
                    if not local_path or local_path.startswith(("http://", "https://")):
                        replace_ids.append(row["id"])
                        continue
                    if static_url_to_path(local_path):
                        has_local_file = True
                        kept_primary = kept_primary or bool(row["is_primary"])
                        continue
                    replace_ids.append(row["id"])

                if has_local_file:
                    continue  # gallery already shows local photos — leave untouched

                image_rows = source_conn.execute(
                    "SELECT * FROM vehicle_images WHERE vin=? AND download_status='downloaded' ORDER BY image_order",
                    (vin,),
                ).fetchall()
                if not image_rows:
                    if rows:
                        still = f"{listing['title_en']} ({vin})"
                        if len(summary["listings_without_local_images"]) < 10:
                            summary["listings_without_local_images"].append(still)
                    continue

                # SEO folder name — same pattern the crawler import uses.
                year = str(listing["year"] or "")
                safe_year = re.sub(r'[<>:"/\\|?*]', '', year)
                safe_make = re.sub(r'[<>:"/\\|?*]', '', listing["make"] or "").replace(' ', '-')
                safe_model = re.sub(r'[<>:"/\\|?*]', '', listing["model"] or "").replace(' ', '-')
                safe_ref = (listing["public_stock_number"] or f"ca-{listing['id']}").replace(' ', '-')
                safe_folder = f"{safe_year}-{safe_make}-{safe_model}-{safe_ref}".strip('-').lower()

                inserted_any = False
                first_featured = ""
                for img in image_rows:
                    local_path = (img["local_path"] or "").strip()
                    if not local_path:
                        continue
                    norm_local = local_path.replace("\\", "/")
                    source_path = Path(norm_local)
                    if not source_path.is_absolute():
                        source_path = source_db_path.parent / norm_local
                    if not source_path.exists():
                        summary["missing_image_files"] += 1
                        vin_missing = summary["missing_image_vins"].setdefault(vin, 0)
                        summary["missing_image_vins"][vin] = vin_missing + 1
                        continue
                    ext = source_path.suffix.lower() or ".jpg"
                    target_name = f"{img['image_order']:03d}{ext}"
                    target_dir = LISTING_UPLOADS_DIR / safe_folder
                    target_file = target_dir / target_name
                    stored_local = f"/static/uploads/listings/{safe_folder}/{target_name}"
                    target_dir.mkdir(parents=True, exist_ok=True)
                    if not target_file.exists():
                        shutil.copy2(source_path, target_file)
                        summary["images_copied"] += 1
                    if not first_featured:
                        first_featured = stored_local
                    is_primary = 0
                    if not inserted_any and not kept_primary:
                        is_primary = 1
                    db.execute(
                        """INSERT INTO listing_images (
                            listing_id, image_url, local_path, alt_text, sort_order,
                            is_primary, file_size, checksum_sha256, download_status, created_at
                        ) VALUES (?,?,?,?,?,?,?,?,?,?)""",
                        (listing["id"], img["best_url"] or "", stored_local,
                         f"{listing['title_en']} — photo {img['image_order'] + 1}".strip(),
                         img["image_order"], is_primary,
                         img["file_size"] if "file_size" in img.keys() else None,
                         img["checksum_sha256"] if "checksum_sha256" in img.keys() else "",
                         "downloaded", now_utc()))
                    inserted_any = True

                if not inserted_any:
                    # Broken listing whose source files were all missing —
                    # surface it so the summary names what stayed broken.
                    still = f"{listing['title_en']} ({vin})"
                    if len(summary["listings_without_local_images"]) < 10:
                        summary["listings_without_local_images"].append(still)
                    continue

                if replace_ids:
                    placeholders = ", ".join("?" for _ in replace_ids)
                    db.execute(
                        f"DELETE FROM listing_images WHERE listing_id = ? AND id IN ({placeholders})",
                        (listing["id"], *replace_ids),
                    )
                    summary["rows_replaced"] += len(replace_ids)
                if first_featured:
                    db.execute(
                        "UPDATE listings SET featured_image = ?, folder_path = ?, updated_at = ? WHERE id = ?",
                        (first_featured, safe_folder, now_utc(), listing["id"]),
                    )
                summary["listings_repaired"] += 1
                db.commit()
            except Exception as error:
                summary["errors"].append(f"Listing {listing['public_stock_number'] or listing['id']}: {error}")
                db.rollback()
        return summary
    finally:
        source_conn.close()


def fetch_listing_by_slug(slug, published_only=True):
    query = "SELECT * FROM listings WHERE slug = ?"
    params = [slug]
    if published_only:
        query += " AND is_published = 1 AND COALESCE(is_deleted, 0) = 0"
    listing = get_db().execute(query, params).fetchone()
    if listing:
        return listing
    fallback_query = """
        SELECT * FROM listings
        WHERE stock_number IS NOT NULL AND stock_number != ''
          AND instr(?, lower(stock_number)) > 0
    """
    fallback_params = [slug.lower()]
    if published_only:
        fallback_query += " AND is_published = 1 AND COALESCE(is_deleted, 0) = 0"
    fallback_query += " ORDER BY id DESC LIMIT 1"
    fallback = get_db().execute(fallback_query, fallback_params).fetchone()
    if fallback:
        return fallback
    # Fallback: old VIN-slug (the VIN was the trailing segment of the slug).
    # Extract the last segment and try to match a listing's VIN.
    last_segment = slug.rsplit("-", 1)[-1] if "-" in slug else slug
    if last_segment and len(last_segment) >= 10:
        vin_query = "SELECT * FROM listings WHERE UPPER(vin) = UPPER(?)"
        vin_params = [last_segment]
        if published_only:
            vin_query += " AND is_published = 1 AND COALESCE(is_deleted, 0) = 0"
        vin_query += " ORDER BY id DESC LIMIT 1"
        vin_listing = get_db().execute(vin_query, vin_params).fetchone()
        if vin_listing:
            return vin_listing
    return None


def get_similar_listings(listing, limit=4):
    db = get_db()
    manual = db.execute(
        """
        SELECT related.*
        FROM related_listings
        JOIN listings AS related ON related.id = related_listings.related_listing_id
        WHERE related_listings.listing_id = ? AND related.is_published = 1
        ORDER BY related_listings.sort_order, related.updated_at DESC
        LIMIT ?
        """,
        (listing["id"], limit),
    ).fetchall()
    selected = list(manual)
    selected_ids = {row["id"] for row in selected}
    selected_ids.add(listing["id"])

    year_value = None
    try:
        year_value = int(listing["year"]) if listing["year"] else None
    except ValueError:
        year_value = None

    candidates = db.execute(
        """
        SELECT * FROM listings
        WHERE is_published = 1 AND id != ?
        ORDER BY updated_at DESC, id DESC
        LIMIT 80
        """,
        (listing["id"],),
    ).fetchall()

    scored = []
    for candidate in candidates:
        if candidate["id"] in selected_ids:
            continue
        score = 0
        if listing["category"] and candidate["category"] == listing["category"]:
            score += 40
        if listing["make"] and candidate["make"] == listing["make"]:
            score += 30
        if candidate["status"] == "Available":
            score += 10
        if year_value:
            try:
                candidate_year = int(candidate["year"]) if candidate["year"] else None
            except ValueError:
                candidate_year = None
            if candidate_year and abs(candidate_year - year_value) <= 3:
                score += 15
        scored.append((score, candidate))

    scored.sort(key=lambda item: item[0], reverse=True)
    for _score, candidate in scored:
        if len(selected) >= limit:
            break
        selected.append(candidate)
    return selected[:limit]


def lead_attribution_snapshot():
    row = analytics_session_row()
    if not row:
        return {
            "visitor_id": session.get("analytics_visitor_id", ""),
            "session_token": session.get("analytics_session_token", ""),
            "landing_page": request.path,
            "first_referrer": request.referrer or "direct",
            "first_visit_at": now_utc(),
            "views_before_lead": 0,
            "engaged_seconds": 0,
            "country": getattr(g, "analytics_country_code", "Unknown"),
            "language": current_lang() if request.endpoint else "",
            "device": detect_device(request.headers.get("User-Agent", "")),
            "user_agent": request.headers.get("User-Agent", "")[:500],
            "ip_hash": hashed_ip(),
        }
    return {
        "visitor_id": row["visitor_id"] or "",
        "session_token": row["session_token"] or "",
        "landing_page": row["landing_page"] or request.path,
        "first_referrer": row["first_referrer"] or "direct",
        "first_visit_at": row["started_at"] or now_utc(),
        "views_before_lead": row["pageview_count"] or 0,
        "engaged_seconds": row["engaged_seconds"] or 0,
        "country": row["country_code"] or "Unknown",
        "language": row["language_code"] or (current_lang() if request.endpoint else ""),
        "device": row["device_type"] or "",
        "user_agent": row["user_agent"] or "",
        "ip_hash": row["ip_hash"] or "",
    }


def save_lead(listing_id=None, lang="en"):
    name = (request.form.get("customer_name") or request.form.get("name") or "").strip()
    email = (request.form.get("customer_email") or request.form.get("email") or "").strip()
    phone = (request.form.get("customer_phone") or request.form.get("phone") or "").strip()
    whatsapp = (request.form.get("customer_whatsapp") or request.form.get("whatsapp") or "").strip()
    message = request.form.get("message", "").strip()
    buyer_type = request.form.get("buyer_type", "").strip()
    delivery_postcode = request.form.get("delivery_postcode", "").strip()
    buyer_country = request.form.get("buyer_country", "").strip()
    preferred_contact_method = request.form.get("preferred_contact_method", "").strip()
    campaign_source = request.form.get("campaign_source", "").strip()
    listing_reference = ""
    if listing_id:
        row = get_db().execute("SELECT public_stock_number, stock_number FROM listings WHERE id = ?", (listing_id,)).fetchone()
        if row:
            listing_reference = row["public_stock_number"] or row["stock_number"] or ""
    if not any((name, email, phone, whatsapp, message)):
        return False
    attribution = lead_attribution_snapshot()
    cursor = get_db().execute(
        """
        INSERT INTO leads (
            listing_id, customer_name, customer_email, customer_phone, customer_whatsapp,
            message, source_page, language_code, created_at, is_read, buyer_type,
            delivery_postcode, buyer_country, preferred_contact_method, listing_reference,
            originating_url, campaign_source, visitor_id, analytics_session_token, landing_page,
            first_referrer, first_visit_at, views_before_lead, engaged_seconds_before_lead,
            attribution_country, attribution_language, attribution_device, analytics_user_agent, analytics_ip_hash
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (listing_id, name, email, phone, whatsapp, message, request.path, lang, now_utc(),
         buyer_type, delivery_postcode, buyer_country, preferred_contact_method, listing_reference,
         request.url, campaign_source, attribution["visitor_id"], attribution["session_token"],
         attribution["landing_page"], attribution["first_referrer"], attribution["first_visit_at"],
         attribution["views_before_lead"], attribution["engaged_seconds"], attribution["country"],
         attribution["language"], attribution["device"], attribution["user_agent"], attribution["ip_hash"]),
    )
    if attribution["session_token"]:
        get_db().execute("UPDATE analytics_sessions SET lead_count = lead_count + 1 WHERE session_token = ?", (attribution["session_token"],))
    record_analytics_event("lead_submit", listing_reference, {"listing_id": listing_id, "source": "form"})
    lead_id = cursor.lastrowid
    get_db().commit()
    try:
        send_lead_notification_email(lead_id)
    except Exception as error:
        app.logger.warning("SMTP lead notification crashed for lead %s: %s", lead_id, error)
    return True


def request_json():
    if request.is_json:
        return request.get_json(silent=True) or {}
    return request.form.to_dict()


def chatbot_is_publicly_enabled(settings=None):
    settings = settings or get_settings()
    if "chatbot_enabled" not in settings.keys() or not settings["chatbot_enabled"]:
        return False
    if not fireworks_api_key(settings):
        return False
    return True


def chatbot_model_choices():
    return FIREWORKS_MODELS


def clean_chat_message(value):
    text = re.sub(r"\s+", " ", (value or "").strip())
    return text[:1200]


def strip_tags(value):
    return re.sub(r"<[^>]+>", " ", value or "").replace("&nbsp;", " ").strip()


def tokenize_query(text):
    stop = {"the", "and", "for", "with", "this", "that", "have", "what", "where", "when", "does", "will", "about", "into", "from", "your", "you", "can"}
    return [token for token in re.findall(r"[a-zA-Z0-9-]{3,}", (text or "").lower()) if token not in stop]


def listing_context_line(listing, lang="en"):
    if not listing:
        return ""
    parts = [
        f"Public reference: {public_reference_for_listing(listing)}",
        f"Title: {public_title_for_listing(listing, lang)}",
        f"Status: {listing['status'] or 'not specified'}",
        f"Year: {listing['year'] or 'not specified'}",
        f"Make: {listing['make'] or 'not specified'}",
        f"Model: {listing['model'] or 'not specified'}",
        f"Category: {listing['category'] or 'not specified'}",
        f"Odometer: {listing['odometer'] or 'not specified'} km",
        f"Body style: {listing['body_style'] or 'not specified'}",
        f"Transmission: {listing['transmission'] or 'not specified'}",
        f"Exterior colour: {listing['exterior_color'] or 'not specified'}",
    ]
    if listing["price"]:
        parts.append(f"Visible price: {format_price_for_language(listing['price'], lang, get_settings())}")
    if listing["price_label"]:
        parts.append(f"Price label: {listing['price_label']}")
    if "carfax_url" in listing.keys() and listing["carfax_url"]:
        parts.append(f"CARFAX link available on listing page: {listing['carfax_url']}")
    else:
        parts.append("CARFAX link: not available on listing page (team can forward if needed)")
    short_description = public_text_for_listing(listing, "short_description", lang)
    if short_description:
        parts.append(f"Description: {strip_tags(short_description)[:500]}")
    return "\n".join(parts)


def find_listing_for_chat(message, listing_id=None):
    db = get_db()
    if listing_id:
        listing = db.execute(
            "SELECT * FROM listings WHERE id = ? AND is_published = 1 AND COALESCE(is_deleted, 0) = 0",
            (listing_id,),
        ).fetchone()
        if listing:
            return listing
    refs = re.findall(r"\b[A-Z]{2,6}-\d{3,6}\b", (message or "").upper())
    for ref in refs:
        listing = db.execute(
            "SELECT * FROM listings WHERE UPPER(public_stock_number) = ? AND is_published = 1 AND COALESCE(is_deleted, 0) = 0",
            (ref,),
        ).fetchone()
        if listing:
            return listing
    return None


def relevant_knowledge_snippets(message, lang="en", limit=6):
    tokens = tokenize_query(message)
    db = get_db()
    candidates = []
    rows = db.execute(
        """
        SELECT title, category, content, language_code, sort_order
        FROM chatbot_knowledge
        WHERE is_active = 1 AND language_code IN (?, 'en')
        ORDER BY language_code = ? DESC, sort_order, id
        LIMIT 80
        """,
        (lang, lang),
    ).fetchall()
    for row in rows:
        haystack = f"{row['title']} {row['category']} {row['content']}".lower()
        score = sum(1 for token in tokens if token in haystack)
        if score or len(candidates) < 3:
            candidates.append((score, f"{row['title']} ({row['category'] or 'Knowledge'}): {strip_tags(row['content'])[:700]}"))
    faq_rows = db.execute("SELECT * FROM faqs WHERE is_active = 1 ORDER BY sort_order, id LIMIT 80").fetchall()
    for faq in faq_rows:
        question = get_translated(faq, "question", lang)
        answer = get_translated(faq, "answer", lang)
        haystack = f"{question} {answer}".lower()
        score = sum(1 for token in tokens if token in haystack)
        if score:
            candidates.append((score, f"FAQ: {question}\nAnswer: {strip_tags(answer)[:650]}"))
    candidates.sort(key=lambda item: item[0], reverse=True)
    return [text for _score, text in candidates[:limit]]


def current_page_context(source_page, lang="en"):
    if not source_page:
        return ""
    path = source_page.split("?", 1)[0]
    parts = [part for part in path.strip("/").split("/") if part]
    if len(parts) >= 2 and parts[0] in LANGUAGES:
        if parts[1] == "inventory":
            return "Current page: inventory listing or inventory browse page."
        slug = parts[1]
        page = get_db().execute(
            "SELECT * FROM site_pages WHERE slug = ? AND is_published = 1 AND status IN ('approved','published')",
            (slug,),
        ).fetchone()
        if page:
            title = get_page_text(page, "title", lang)
            excerpt = get_page_text(page, "excerpt", lang)
            return f"Current page: {title}\nSummary: {strip_tags(excerpt)[:500]}"
    return ""


def chatbot_context(message, lang="en", source_page="", listing_id=None):
    listing = find_listing_for_chat(message, listing_id=listing_id)
    snippets = relevant_knowledge_snippets(message, lang=lang)
    context = [
        f"Company: {get_settings()['company_name']}",
        f"Contact phone: {get_settings()['phone'] or 'not specified'}",
        f"Contact email: {get_settings()['email'] or 'not specified'}",
    ]
    page_context = current_page_context(source_page, lang)
    if page_context:
        context.append(page_context)
    if listing:
        context.append("Current/referenced listing:\n" + listing_context_line(listing, lang))
    if snippets:
        context.append("Relevant knowledge:\n" + "\n\n".join(snippets))
    context.append(
        "HST rates by province: ON=13%, AB=5%, BC=12%, QC=14.975%, MB=12%, SK=11%, NS=15%, NB=15%, NL=15%, PE=15%, YT=5%, NT=5%, NU=5%. Licensing fee estimate: $250 CAD."
    )
    context.append(
        "Safety rules: exact location, delivery price/date, HST and licensing details, finance/payment details, availability before payment, and inspection/warranty details must be confirmed by the team unless explicitly stated in the context. Never fabricate a CARFAX link. Never claim clean title, no accidents, or one-owner unless verified data in context confirms it. Only mention free delivery promo if it is shown as active in the system prompt."
    )
    return "\n\n".join(context), listing


def should_prompt_lead(message):
    lower = (message or "").lower()
    triggers = ("price", "available", "availability", "deliver", "delivery", "hst", "financing", "omvic", "trade-in", "location", "where is", "payment", "pay", "video", "photos", "pictures", "buy", "purchase", "quote")
    return any(trigger in lower for trigger in triggers)


def get_or_create_chat_session(token="", lang="en", source_page="", listing_id=None):
    db = get_db()
    row = None
    if token:
        row = db.execute("SELECT * FROM chat_sessions WHERE session_token = ?", (token,)).fetchone()
    timestamp = now_utc()
    if row:
        db.execute(
            "UPDATE chat_sessions SET language_code = ?, source_page = COALESCE(NULLIF(?, ''), source_page), listing_id = COALESCE(?, listing_id), updated_at = ? WHERE id = ?",
            (lang, source_page or "", listing_id, timestamp, row["id"]),
        )
        db.commit()
        return db.execute("SELECT * FROM chat_sessions WHERE id = ?", (row["id"],)).fetchone()
    token = uuid.uuid4().hex
    db.execute(
        """
        INSERT INTO chat_sessions (
            session_token, language_code, source_page, listing_id, ip_hash, user_agent, created_at, updated_at, lead_created
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
        """,
        (token, lang, source_page or "", listing_id, hashed_ip(), request.headers.get("User-Agent", "")[:500], timestamp, timestamp),
    )
    db.commit()
    return db.execute("SELECT * FROM chat_sessions WHERE session_token = ?", (token,)).fetchone()


def record_chat_message(session_id, role, message, model="", provider="", tokens_input=0, tokens_output=0):
    get_db().execute(
        """
        INSERT INTO chat_messages (session_id, role, message, model, provider, tokens_input, tokens_output, created_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (session_id, role, message, model, provider, tokens_input or 0, tokens_output or 0, now_utc()),
    )


@app.route("/api/chat/start", methods=("POST",))
def api_chat_start():
    settings = get_settings()
    if not chatbot_is_publicly_enabled(settings):
        return jsonify({"ok": False, "disabled": True}), 200
    data = request_json()
    lang = data.get("language_code", current_lang())
    if not is_valid_lang(lang):
        lang = settings["default_language"] or "en"
    listing_id = data.get("listing_id")
    try:
        listing_id = int(listing_id) if listing_id else None
    except (TypeError, ValueError):
        listing_id = None
    chat_session = get_or_create_chat_session(
        token=data.get("session_token", ""),
        lang=lang,
        source_page=data.get("source_page", request.path),
        listing_id=listing_id,
    )
    greeting = settings["chatbot_greeting"] if "chatbot_greeting" in settings.keys() else ""
    try:
        record_analytics_event("chatbot_open", "", {"listing_id": listing_id, "source_page": data.get("source_page", request.path)})
        get_db().execute("UPDATE analytics_sessions SET chatbot_opened = chatbot_opened + 1 WHERE session_token = ?", (session.get("analytics_session_token", ""),))
        get_db().commit()
    except Exception:
        get_db().rollback()
    return jsonify(
        {
            "ok": True,
            "session_token": chat_session["session_token"],
            "greeting": greeting or default_chatbot_greeting(),
        }
    )


@app.route("/api/chat/message", methods=("POST",))
def api_chat_message():
    settings = get_settings()
    fallback = settings["chatbot_fallback_message"] if "chatbot_fallback_message" in settings.keys() else ""
    fallback = fallback or default_chatbot_fallback_message()
    if not chatbot_is_publicly_enabled(settings):
        return jsonify({"ok": False, "reply": fallback, "disabled": True}), 200
    data = request_json()
    message = clean_chat_message(data.get("message", ""))
    if not message:
        return jsonify({"ok": False, "reply": "Please type a message first."}), 400
    lang = data.get("language_code", current_lang())
    if not is_valid_lang(lang):
        lang = settings["default_language"] or "en"
    listing_id = data.get("listing_id")
    try:
        listing_id = int(listing_id) if listing_id else None
    except (TypeError, ValueError):
        listing_id = None
    chat_session = get_or_create_chat_session(
        token=data.get("session_token", ""),
        lang=lang,
        source_page=data.get("source_page", request.path),
        listing_id=listing_id,
    )
    recent_threshold = (datetime.utcnow() - timedelta(minutes=2)).isoformat(timespec="seconds")
    recent_count = get_db().execute(
        "SELECT COUNT(*) FROM chat_messages WHERE session_id = ? AND role = 'user' AND created_at >= ?",
        (chat_session["id"], recent_threshold),
    ).fetchone()[0]
    if recent_count >= 8:
        return jsonify({"ok": False, "reply": "Please wait a moment before sending more messages."}), 429
    context, listing = chatbot_context(message, lang=lang, source_page=data.get("source_page", ""), listing_id=listing_id)
    promo_enabled = False
    try:
        promo_enabled = bool(int(settings_value(settings, "free_delivery_promo_enabled", 0)))
    except (TypeError, ValueError):
        pass
    previous = get_db().execute(
        """
        SELECT role, message
        FROM chat_messages
        WHERE session_id = ?
        ORDER BY id DESC
        LIMIT 8
        """,
        (chat_session["id"],),
    ).fetchall()
    messages = [
        {"role": "system", "content": (settings["chatbot_system_prompt"] or "").strip() or default_chatbot_system_prompt(settings["company_name"], free_delivery_promo_enabled=promo_enabled)},
        {"role": "system", "content": f"Reply language: {LANGUAGE_NAMES.get(lang, lang)}.\nContext:\n{context[:5000]}"},
    ]
    for row in reversed(previous):
        if row["role"] in ("user", "assistant") and row["message"]:
            messages.append({"role": row["role"], "content": row["message"][:1200]})
    messages.append({"role": "user", "content": message})
    try:
        chat_call = make_chat_adapter(settings)
        result = chat_call(messages)
        reply = result["text"] or fallback
        model = result["model"]
        tokens_input = result["tokens_input"]
        tokens_output = result["tokens_output"]
    except Exception:
        get_db().rollback()
        reply = "I'm having trouble replying right now. Please use the enquiry form or our contact details and our team will help."
        model = settings["chatbot_model"] if "chatbot_model" in settings.keys() else DEFAULT_FIREWORKS_MODEL
        tokens_input = 0
        tokens_output = 0
    record_chat_message(chat_session["id"], "user", message)
    record_chat_message(chat_session["id"], "assistant", reply, model=model, provider="fireworks", tokens_input=tokens_input, tokens_output=tokens_output)
    get_db().execute("UPDATE chat_sessions SET updated_at = ?, listing_id = COALESCE(?, listing_id) WHERE id = ?", (now_utc(), listing["id"] if listing else None, chat_session["id"]))
    try:
        record_analytics_event("chatbot_message", "", {"listing_id": listing["id"] if listing else listing_id})
        get_db().execute("UPDATE analytics_sessions SET chatbot_messages = chatbot_messages + 1 WHERE session_token = ?", (session.get("analytics_session_token", ""),))
    except Exception:
        pass
    get_db().commit()
    return jsonify(
        {
            "ok": True,
            "reply": reply,
            "session_token": chat_session["session_token"],
            "lead_capture": bool(settings["chatbot_lead_capture_enabled"] if "chatbot_lead_capture_enabled" in settings.keys() else 1) and should_prompt_lead(message),
        }
    )


@app.route("/api/chat/lead", methods=("POST",))
def api_chat_lead():
    settings = get_settings()
    if not chatbot_is_publicly_enabled(settings):
        return jsonify({"ok": False}), 200
    data = request_json()
    chat_session = get_db().execute("SELECT * FROM chat_sessions WHERE session_token = ?", (data.get("session_token", ""),)).fetchone()
    if not chat_session:
        return jsonify({"ok": False, "error": "Chat session not found."}), 404
    name = (data.get("name") or "").strip()[:120]
    email = (data.get("email") or "").strip()[:160]
    phone = (data.get("phone") or "").strip()[:80]
    whatsapp = (data.get("whatsapp") or "").strip()[:80]
    province = (data.get("province") or "").strip()[:20]
    destination = (data.get("destination") or "").strip()[:180]
    message = (data.get("message") or "").strip()[:1000]
    if not any((name, email, phone, whatsapp, message, destination, province)):
        return jsonify({"ok": False, "error": "Please add contact details first."}), 400
    lead_message = message or "Chatbot lead"
    if province:
        lead_message += f"\nProvince: {province}"
    if destination:
        lead_message += f"\nDestination: {destination}"
    attribution = lead_attribution_snapshot()
    cursor = get_db().execute(
        """
        INSERT INTO leads (
            listing_id, customer_name, customer_email, customer_phone, customer_whatsapp,
            message, source_page, language_code, created_at, is_read, visitor_id,
            analytics_session_token, landing_page, first_referrer, first_visit_at,
            views_before_lead, engaged_seconds_before_lead, attribution_country,
            attribution_language, attribution_device, analytics_user_agent, analytics_ip_hash
        )
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (
            chat_session["listing_id"], name, email, phone, whatsapp, lead_message,
            chat_session["source_page"] or request.path, chat_session["language_code"] or current_lang(),
            now_utc(), attribution["visitor_id"], attribution["session_token"],
            attribution["landing_page"], attribution["first_referrer"], attribution["first_visit_at"],
            attribution["views_before_lead"], attribution["engaged_seconds"], attribution["country"],
            attribution["language"], attribution["device"], attribution["user_agent"], attribution["ip_hash"],
        ),
    )
    if attribution["session_token"]:
        get_db().execute("UPDATE analytics_sessions SET lead_count = lead_count + 1 WHERE session_token = ?", (attribution["session_token"],))
    record_analytics_event("lead_submit", "chatbot", {"listing_id": chat_session["listing_id"], "source": "chatbot"})
    lead_id = cursor.lastrowid
    get_db().execute(
        """
        UPDATE chat_sessions
        SET visitor_name = ?, visitor_email = ?, visitor_phone = ?, visitor_whatsapp = ?,
            lead_created = 1, updated_at = ?
        WHERE id = ?
        """,
        (name, email, phone, whatsapp, now_utc(), chat_session["id"]),
    )
    record_chat_message(chat_session["id"], "user", f"Lead details submitted. Destination: {destination}. Message: {message}")
    get_db().commit()
    try:
        send_lead_notification_email(lead_id)
    except Exception as error:
        app.logger.warning("SMTP chatbot lead notification crashed for lead %s: %s", lead_id, error)
    return jsonify({"ok": True, "message": "Thanks. Your details have been sent to the team."})


@app.route("/")
def index():
    return language_redirect_response("home")


@app.route("/set-language/<lang>")
def set_language(lang):
    if not is_valid_lang(lang) or lang not in active_language_codes():
        lang = "en"
    next_path = (request.args.get("next") or url_for("home", lang=lang)).strip()
    if not next_path.startswith("/") or next_path.startswith("//"):
        next_path = url_for("home", lang=lang)
    response = redirect(next_path, code=302)
    if bool(settings_value(get_settings(), "remember_language_choice", 1)):
        response.set_cookie(
            LANGUAGE_COOKIE_NAME,
            lang,
            max_age=LANGUAGE_COOKIE_MAX_AGE,
            secure=request.is_secure,
            httponly=False,
            samesite="Lax",
        )
    return response


@app.route("/inventory")
def legacy_inventory():
    return language_redirect_response("inventory")


@app.route("/inventory/<slug>")
def legacy_listing_detail(slug):
    return language_redirect_response("listing_detail", slug=slug)


@app.route("/<lang>/")
def home(lang):
    if not is_valid_lang(lang):
        abort(404)
    page = get_db().execute(
        "SELECT * FROM site_pages WHERE slug = 'home' AND is_published = 1"
    ).fetchone()
    settings = get_settings()
    try:
        fcc = int(settings["featured_carousel_count"] or 8) if "featured_carousel_count" in settings.keys() else 8
    except (TypeError, ValueError):
        fcc = 8
    fcc = max(2, min(24, fcc))
    featured_listings = get_db().execute(
        f"""SELECT * FROM listings WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0 AND is_featured = 1 ORDER BY created_at DESC, id DESC LIMIT {fcc}"""
    ).fetchall()
    if not featured_listings:
        featured_listings = get_db().execute(
            f"""SELECT * FROM listings WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0 ORDER BY created_at DESC, id DESC LIMIT {fcc}"""
        ).fetchall()
    recent_listings = get_db().execute(
        """
        SELECT * FROM listings
        WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0
        ORDER BY created_at DESC, id DESC
        LIMIT 8
        """
    ).fetchall()
    faqs = get_db().execute(
        "SELECT * FROM faqs WHERE is_active = 1 ORDER BY sort_order, id LIMIT 4"
    ).fetchall()
    cards = get_db().execute(
        "SELECT * FROM homepage_cards WHERE is_active = 1 ORDER BY card_type, sort_order"
    ).fetchall()
    body_style_cards = [c for c in cards if c["card_type"] == "body_style"]
    make_cards = [c for c in cards if c["card_type"] == "make"]

    # Data-driven popular makes: top N by published inventory count.
    import pathlib
    try:
        pm_count = int(settings["popular_makes_count"] or 8)
    except (TypeError, ValueError):
        pm_count = 8
    pm_count = max(1, min(12, pm_count))
    top_makes = get_db().execute(
        """SELECT make, COUNT(*) AS cnt FROM listings
           WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0
             AND make IS NOT NULL AND make != ''
           GROUP BY make ORDER BY cnt DESC, make ASC LIMIT ?""",
        (pm_count,),
    ).fetchall()
    cards_dir = pathlib.Path(BASE_DIR) / "static" / "uploads" / "cards"
    popular_makes = []
    for row in top_makes:
        mk = row["make"]
        sanitized = mk.replace(" ", "-")
        logo_file = cards_dir / f"make_{sanitized}.png"
        image_path = f"uploads/cards/make_{sanitized}.png" if logo_file.is_file() else None
        popular_makes.append({
            "key": mk,
            "label": mk,
            "count": row["cnt"],
            "image_path": image_path,
        })

    BODY_STYLE_EMOJI = {
        "Sedan": "🚗",
        "SUV": "🚙",
        "Truck": "🛻",
        "Coupe": "🏎️",
        "Hatchback": "🚐",
        "Van": "🚌",
    }
    MAKE_COLORS = [
        ("bg-red-100 text-red-700", "bg-red-400"),
        ("bg-blue-100 text-blue-700", "bg-blue-400"),
        ("bg-indigo-100 text-indigo-700", "bg-indigo-400"),
        ("bg-amber-100 text-amber-700", "bg-amber-400"),
        ("bg-emerald-100 text-emerald-700", "bg-emerald-400"),
        ("bg-rose-100 text-rose-700", "bg-rose-400"),
        ("bg-sky-100 text-sky-700", "bg-sky-400"),
        ("bg-violet-100 text-violet-700", "bg-violet-400"),
    ]
    return render_template(
        "home.html",
        page=page,
        featured_listings=featured_listings,
        recent_listings=recent_listings,
        faqs=faqs,
        lang=lang,
        body_style_cards=body_style_cards,
        make_cards=make_cards,
        popular_makes=popular_makes,
        BODY_STYLE_EMOJI=BODY_STYLE_EMOJI,
        MAKE_COLORS=MAKE_COLORS,
    )


SEO_CATEGORY_SLUGS = {
    "en": {"sedans": "Sedan", "suvs": "SUV", "trucks": "Truck"},
    "fr": {"sedans": "Sedan", "vus": "SUV", "camions": "Truck"},
}


def seo_landing_for_slug(lang, slug):
    category = SEO_CATEGORY_SLUGS.get(lang, {}).get(slug) or SEO_CATEGORY_SLUGS["en"].get(slug)
    if category:
        listings = get_db().execute(
            """
            SELECT * FROM listings
            WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0 AND category = ?
            ORDER BY is_featured DESC, updated_at DESC, id DESC
            LIMIT 24
            """,
            (category,),
        ).fetchall()
        title = f"Used {category}s for Sale"
        return {"kind": "category", "title": title, "heading": title, "description": f"Browse available {category.lower()} vehicles for sale in Canada.", "listings": listings, "category": category}
    if slug.startswith("used-"):
        make = slug.replace("used-", "").replace("-", " ").title()
        listings = get_db().execute(
            """
            SELECT * FROM listings
            WHERE is_published = 1 AND COALESCE(is_deleted, 0) = 0
              AND lower(make) = lower(?)
            ORDER BY is_featured DESC, updated_at DESC, id DESC
            LIMIT 24
            """,
            (make,),
        ).fetchall()
        if listings:
            return {"kind": "make", "title": f"Used {make} Cars for Sale", "heading": f"Used {make} Cars for Sale", "description": f"Browse {make} used cars available in Canada. Enquire for delivery and financing.", "listings": listings, "category": ""}
        return None
    return None


@app.route("/<lang>/inventory")
def inventory(lang):
    if not is_valid_lang(lang):
        abort(404)
    g.analytics_language_code = lang
    query, params, filters = build_listing_query(published_only=True)
    listings = get_db().execute(query, params).fetchall()
    filter_options = listing_filter_options(published_only=True)
    result_count = len(listings)
    return render_template(
        "inventory.html",
        listings=listings,
        filters=filters,
        filter_options=filter_options,
        makes=filter_options["makes"],
        years=filter_options["years"],
        result_count=result_count,
        lang=lang,
    )


@app.route("/<lang>/inventory/filter-data")
def inventory_filter_data(lang):
    if not is_valid_lang(lang):
        abort(404)
    query, params, filters = build_listing_query(published_only=True)
    listings = get_db().execute(query, params).fetchall()
    ids = [row["id"] for row in listings]
    options = {"categories": [], "makes": [], "models": [], "years": [], "statuses": []}
    if ids:
        placeholders = ", ".join("?" for _ in ids)
        for key, field in (("categories", "category"), ("makes", "make"), ("models", "model"), ("years", "year"), ("statuses", "status")):
            extra_where = ""
            if field == "model":
                extra_where = (
                    " AND model NOT GLOB 'ST[0-9][0-9][0-9][0-9][0-9]'"
                    " AND model NOT GLOB '(ST[0-9][0-9][0-9][0-9][0-9])'"
                )
            rows = get_db().execute(
                f"SELECT {field} AS value, COUNT(*) AS count FROM listings WHERE id IN ({placeholders}) AND {field} IS NOT NULL AND {field} != ''{extra_where} GROUP BY {field} ORDER BY {field}",
                ids,
            ).fetchall()
            options[key] = [{"value": row["value"], "count": row["count"]} for row in rows]
    return jsonify({"count": len(listings), "filters": filters, "options": options})


@app.route("/media/listings/<public_ref>/<path:filename>")
def public_listing_image(public_ref, filename):
    safe_filename = secure_filename(filename)
    if not safe_filename or safe_filename != Path(filename).name:
        abort(404)
    listing = get_db().execute(
        """
        SELECT * FROM listings
        WHERE public_stock_number = ?
          AND is_published = 1
          AND COALESCE(is_deleted, 0) = 0
        """,
        (public_ref,),
    ).fetchone()
    if not listing:
        abort(404)
    rows = get_db().execute(
        """
        SELECT local_path, generated_local_path
        FROM listing_images
        WHERE listing_id = ?
          AND COALESCE(is_public, 1) = 1
        """,
        (listing["id"],),
    ).fetchall()
    for row in rows:
        for field in ("local_path", "generated_local_path"):
            path_value = row[field] or ""
            # Serve the original stored image.
            if Path(path_value).name == safe_filename:
                file_path = static_url_to_path(path_value)
                if file_path:
                    return send_file(file_path, conditional=True, max_age=IMAGE_CACHE_MAX_AGE)
            # Serve generated thumbnail/gallery variants for the same stored image.
            file_path = static_url_to_path(path_value)
            if file_path:
                thumb_file = image_variant_file(file_path, THUMBNAIL_SUFFIX, THUMBNAIL_EXTENSION)
                if thumb_file and thumb_file.name == safe_filename and thumb_file.exists() and thumb_file.is_file():
                    return send_file(thumb_file, conditional=True, max_age=IMAGE_CACHE_MAX_AGE)
                gallery_file = image_variant_file(file_path, GALLERY_SUFFIX, GALLERY_EXTENSION)
                if gallery_file and gallery_file.name == safe_filename and gallery_file.exists() and gallery_file.is_file():
                    return send_file(gallery_file, conditional=True, max_age=IMAGE_CACHE_MAX_AGE)
    folder_name = secure_filename(listing["stock_number"] or "")
    if folder_name:
        fallback = (LISTING_UPLOADS_DIR / folder_name / safe_filename).resolve()
        try:
            fallback.relative_to(LISTING_UPLOADS_DIR.resolve())
        except ValueError:
            fallback = None
        if fallback and fallback.exists() and fallback.is_file():
            return send_file(fallback, conditional=True, max_age=IMAGE_CACHE_MAX_AGE)
    abort(404)


@app.route("/robots.txt")
def robots_txt():
    body = f"User-agent: *\nAllow: /\nDisallow: {ADMIN_URL_PREFIX}/\nSitemap: {url_for('sitemap_xml', _external=True)}\n"
    return app.response_class(body, mimetype="text/plain")


@app.route("/favicon.ico")
def favicon_ico():
    settings = get_settings()
    favicon_path = static_url_to_path(settings["favicon_path"] or "")
    if favicon_path:
        return send_file(favicon_path)
    logo_path = static_url_to_path(settings["logo_path"] or "")
    if logo_path:
        return send_file(logo_path)
    return app.response_class("", status=204)


@app.route("/sitemap.xml")
def sitemap_xml():
    urls = []
    for lang in LANGUAGES:
        urls.append(url_for("home", lang=lang, _external=True))
        urls.append(url_for("inventory", lang=lang, _external=True))
    pages = get_db().execute("SELECT * FROM site_pages WHERE is_published = 1 AND status IN ('approved','published') ORDER BY slug").fetchall()
    for page in pages:
        if page["slug"] == "home":
            continue
        for lang in LANGUAGES:
            if lang == "en" or page_translation_is_public(page, lang):
                urls.append(url_for("public_page", lang=lang, page_slug=page["slug"], _external=True))
    listings = get_db().execute("SELECT slug FROM listings WHERE is_published = 1 AND COALESCE(is_deleted,0)=0 ORDER BY updated_at DESC").fetchall()
    for listing in listings:
        for lang in LANGUAGES:
            urls.append(url_for("listing_detail", lang=lang, slug=listing["slug"], _external=True))
    for lang, slugs in SEO_CATEGORY_SLUGS.items():
        for slug in slugs:
            urls.append(url_for("public_page", lang=lang, page_slug=slug, _external=True))
    trust_settings = get_settings()
    if "team_page_enabled" in trust_settings.keys() and trust_settings["team_page_enabled"]:
        for lang in LANGUAGES:
            urls.append(url_for("trust.team_page", lang=lang, _external=True))
    if "recent_deliveries_page_enabled" in trust_settings.keys() and trust_settings["recent_deliveries_page_enabled"]:
        for lang in LANGUAGES:
            urls.append(url_for("trust.recent_deliveries_page", lang=lang, _external=True))
    xml = ['<?xml version="1.0" encoding="UTF-8"?>', '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
    for url in sorted(set(urls)):
        xml.append(f"<url><loc>{escape(url)}</loc></url>")
    xml.append("</urlset>")
    return app.response_class("\n".join(xml), mimetype="application/xml")


@app.route("/image-sitemap.xml")
def image_sitemap_xml():
    listings = get_db().execute("SELECT * FROM listings WHERE is_published = 1 AND COALESCE(is_deleted,0)=0 ORDER BY id").fetchall()
    xml = ['<?xml version="1.0" encoding="UTF-8"?>', '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">']
    for listing in listings:
        images = build_listing_gallery(listing, "en")
        if not images:
            continue
        xml.append(f"<url><loc>{escape(url_for('listing_detail', lang='en', slug=listing['slug'], _external=True))}</loc>")
        for image in images[:10]:
            xml.append(f"<image:image><image:loc>{escape(request.url_root.rstrip('/') + image['src'])}</image:loc></image:image>")
        xml.append("</url>")
    xml.append("</urlset>")
    return app.response_class("\n".join(xml), mimetype="application/xml")


@app.route("/carfax-report/<vin>")
def carfax_report(vin):
    """Serve a local Carfax HTML report by VIN. Fallback page if not found.
    The saved HTML contains JavaScript from carfax.ca that tries to fetch fresh
    data and redirect — we neutralize the redirect so the embedded report data renders."""
    safe_vin = re.sub(r"[^A-Za-z0-9]", "", vin)
    if safe_vin != vin or len(safe_vin) < 10:
        abort(404)
    file_path = os.path.join(CARFAX_REPORTS_DIR, f"{safe_vin}.html")
    if not os.path.isfile(file_path):
        return render_template("carfax_not_found.html", vin=safe_vin, lang=current_lang()), 200
    # Read the HTML and neutralize the carfax.ca JavaScript that tries to fetch
    # fresh data and redirect/replace the page. The saved reports contain embedded
    # data but also carfax.ca's SPA loader JS. We:
    # 1. Make isItOutOfDate() always return true → skips the fetch/redirect logic entirely.
    # 2. Neutralize window.location = ... assignments (redirects).
    # 3. Neutralize ShowErrorPageContent (replaces the page with an error page on fetch failure).
    with open(file_path, "r", encoding="utf-8") as f:
        content = f.read()
    content = content.replace("return out_of_date;", "return true;")
    content = re.sub(r"window\.location\s*=\s*", "// window.location = ", content)
    content = content.replace(
        "document.querySelector('html').innerHTML = errPageResponseText;",
        "// document.querySelector('html').innerHTML = errPageResponseText;",
    )
    return app.response_class(content, mimetype="text/html")


@app.route("/carfax-report/assets/<path:filename>")
def carfax_report_assets(filename):
    """Serve Carfax report assets (CSS/JS/fonts/images) so relative paths resolve."""
    return send_from_directory(os.path.join(CARFAX_REPORTS_DIR, "assets"), filename)


def shop_profile():
    """Return the mechanical shop profile dict from site_settings (for report rendering)."""
    s = get_settings()
    return {
        "name": settings_value(s, "shop_name", "") or "",
        "phone": settings_value(s, "shop_phone", "") or "",
        "email": settings_value(s, "shop_email", "") or "",
        "address": settings_value(s, "shop_address", "") or "",
        "license": settings_value(s, "shop_license", "") or "",
        "logo_path": settings_value(s, "shop_logo_path", "") or "",
    }


@app.route("/inspection-report/<token>")
def inspection_report(token):
    """Public inspection report page (token is opaque — no VIN in the URL).
    Only approved reports are shown; anything else renders the not-available page."""
    report = inspection_reports.get_report_by_token(get_db(), token)
    if report is None:
        abort(404)
    if report["status"] != "approved":
        return render_template("inspection_not_available.html", lang="en"), 200
    try:
        data = json.loads(report["report_json"] or "{}")
    except Exception:
        data = {}
    return render_template(
        "inspection_report.html",
        report=report,
        data=data,
        shop=shop_profile(),
        lang="en",
    )


def _shop_token_valid(shop_token):
    """True if the given token matches the configured shop portal token."""
    if not shop_token:
        return False
    s = get_settings()
    expected = settings_value(s, "shop_portal_token", "") or ""
    return bool(expected) and shop_token == expected


@app.route("/shop-portal/<shop_token>")
def shop_portal(shop_token):
    """Shop review portal: lists all inspection reports for bulk review/approve."""
    if not _shop_token_valid(shop_token):
        abort(404)
    status_filter = (request.args.get("status") or "").strip()
    params = []
    where = ""
    if status_filter in inspection_reports.INSPECTION_STATUSES:
        where = "WHERE ir.status = ?"
        params.append(status_filter)
    rows = get_db().execute(
        f"""SELECT ir.*, l.make AS l_make, l.model AS l_model, l.year AS l_year,
                   l.trim AS l_trim, l.public_stock_number AS l_stock
            FROM inspection_reports ir
            LEFT JOIN listings l ON l.id = ir.listing_id
            {where}
            ORDER BY CASE ir.status
                       WHEN 'pending' THEN 0
                       WHEN 'changes_requested' THEN 1
                       WHEN 'approved' THEN 2
                       ELSE 3 END,
                     ir.generated_at DESC""",
        params,
    ).fetchall()
    return render_template("shop_portal.html", rows=rows, shop_token=shop_token,
                           status_filter=status_filter, shop=shop_profile())


@app.route("/shop-portal/<shop_token>/report/<report_token>", methods=("GET", "POST"))
def shop_portal_report(shop_token, report_token):
    """Per-report shop review page: full report + section editor + approve/request/revoke."""
    if not _shop_token_valid(shop_token):
        abort(404)
    report = inspection_reports.get_report_by_token(get_db(), report_token)
    if report is None:
        abort(404)
    if request.method == "POST":
        action = (request.form.get("action") or "").strip()
        if action == "save_edits":
            try:
                data = json.loads(report["report_json"] or "{}")
            except Exception:
                data = {}
            sections = []
            for section in data.get("sections", []):
                sid = section.get("id")
                if not sid:
                    continue
                rating = (request.form.get(f"section_{sid}_rating") or "pass").strip()
                if rating not in inspection_reports.RATINGS:
                    rating = "pass"
                findings_raw = request.form.get(f"section_{sid}_findings") or ""
                findings = [f.strip() for f in findings_raw.splitlines() if f.strip()]
                notes = (request.form.get(f"section_{sid}_notes") or "").strip()
                sections.append({"id": sid, "rating": rating, "findings": findings, "notes": notes})
            overall_rating = (request.form.get("overall_rating") or "pass").strip()
            if overall_rating not in inspection_reports.RATINGS:
                overall_rating = "pass"
            overall_summary = (request.form.get("overall_summary") or "").strip()
            inspection_reports.update_report_sections(
                get_db(), report["listing_id"], sections,
                overall_verdict={"rating": overall_rating, "summary": overall_summary},
            )
            flash("Report edits saved.", "success")
        elif action == "approve":
            inspection_reports.set_report_status(
                get_db(), report["listing_id"], "approved",
                approved_by=(request.form.get("inspector_name") or "").strip(),
                inspector_name=(request.form.get("inspector_name") or "").strip(),
                inspection_date=(request.form.get("inspection_date") or "").strip(),
            )
            flash("Report approved & stamped.", "success")
        elif action == "request_changes":
            inspection_reports.set_report_status(
                get_db(), report["listing_id"], "changes_requested",
                shop_notes=(request.form.get("shop_notes") or "").strip(),
            )
            flash("Changes requested.", "success")
        elif action == "revoke":
            inspection_reports.set_report_status(get_db(), report["listing_id"], "revoked")
            flash("Report revoked.", "success")
        return redirect(url_for("shop_portal_report", shop_token=shop_token, report_token=report_token))
    # GET
    report = inspection_reports.get_report_by_token(get_db(), report_token)
    try:
        data = json.loads(report["report_json"] or "{}")
    except Exception:
        data = {}
    return render_template("shop_portal_report.html", report=report, data=data,
                           shop=shop_profile(), shop_token=shop_token)


@app.route("/shop-portal/<shop_token>/bulk", methods=("POST",))
def shop_portal_bulk(shop_token):
    """Bulk approve / request changes / revoke selected reports."""
    if not _shop_token_valid(shop_token):
        abort(404)
    action = (request.form.get("bulk_action") or "").strip()
    tokens = request.form.getlist("report_tokens")
    inspector_name = (request.form.get("inspector_name") or "").strip()
    inspection_date = (request.form.get("inspection_date") or "").strip()
    date_start = (request.form.get("inspection_date_start") or "").strip()
    date_end = (request.form.get("inspection_date_end") or "").strip()
    use_range = bool(date_start and date_end)
    count = 0
    for tok in tokens:
        row = inspection_reports.get_report_by_token(get_db(), tok)
        if not row:
            continue
        if action == "approve":
            if use_range:
                d_iso, dt_iso = inspection_reports.random_approval_date(date_start, date_end)
                inspection_reports.set_report_status(
                    get_db(), row["listing_id"], "approved",
                    approved_by=inspector_name, inspector_name=inspector_name,
                    inspection_date=d_iso, approved_at=dt_iso,
                )
            else:
                inspection_reports.set_report_status(
                    get_db(), row["listing_id"], "approved",
                    approved_by=inspector_name, inspector_name=inspector_name,
                    inspection_date=inspection_date,
                )
        elif action == "request_changes":
            inspection_reports.set_report_status(get_db(), row["listing_id"], "changes_requested")
        elif action == "revoke":
            inspection_reports.set_report_status(get_db(), row["listing_id"], "revoked")
        else:
            continue
        count += 1
    flash(f"{count} report(s) updated.", "success")
    return redirect(url_for("shop_portal", shop_token=shop_token))


@app.route("/<lang>/inventory/<slug>", methods=("GET", "POST"))
def listing_detail(lang, slug):
    if not is_valid_lang(lang):
        abort(404)
    listing = fetch_listing_by_slug(slug)
    if listing is None:
        abort(404)
    g.current_listing = listing
    g.listing_translation_missing = lang != "en" and not listing_has_translation(listing, lang)
    if listing["slug"] != slug and request.method == "GET":
        return redirect(url_for("listing_detail", lang=lang, slug=listing["slug"]), code=301)
    g.analytics_listing_id = listing["id"]
    g.analytics_language_code = lang
    if request.method == "POST":
        if save_lead(listing["id"], lang):
            flash("Thank you. Your enquiry has been received.", "success")
        else:
            flash("Please add a little contact information before sending.", "warning")
        return redirect(url_for("listing_detail", lang=lang, slug=slug))

    listing_title = get_translated(listing, "title", lang)
    images = build_listing_gallery(listing, lang)
    if not images and get_listing_primary_image(listing["id"]):
        fallback_src = get_listing_primary_image(listing["id"])
        images.append({"src": fallback_src, "full_src": fallback_src, "thumb_src": fallback_src, "alt": listing_title})
    related = get_similar_listings(listing, limit=3)
    report = inspection_reports.get_report_for_listing(get_db(), listing["id"])
    return render_template(
        "listing_detail.html",
        listing=listing,
        images=images,
        listing_images=images,
        related=related,
        similar_listings=related,
        report=report,
        lang=lang,
    )


@app.route("/<lang>/contact", methods=("GET", "POST"))
def contact(lang):
    if not is_valid_lang(lang):
        abort(404)
    if request.method == "POST":
        if save_lead(None, lang):
            flash("Thank you. Your message has been received.", "success")
        else:
            flash("Please add a little contact information before sending.", "warning")
        return redirect(url_for("contact", lang=lang))
    page = get_db().execute(
        "SELECT * FROM site_pages WHERE slug = 'contact' AND is_published = 1"
    ).fetchone()
    return render_template("contact.html", page=page, lang=lang)


@app.route("/<lang>/<page_slug>")
def public_page(lang, page_slug):
    if not is_valid_lang(lang):
        abort(404)
    if page_slug == "home":
        return redirect(url_for("home", lang=lang), code=301)
    page = get_db().execute(
        "SELECT * FROM site_pages WHERE slug = ? AND is_published = 1 AND status IN ('approved','published')", (page_slug,)
    ).fetchone()
    if page is None:
        OLD_UK_REDIRECTS = {
            "shipping-information": "delivery",
            "tax-information": "financing",
            "european-warehouses": "about-us",
            "export": "about-us",
            "export-delivery": "delivery",
            "warranty-inspection": "warranty",
            "brexit-vat": "financing",
            "vat-information": "financing",
        }
        if page_slug in OLD_UK_REDIRECTS:
            return redirect(url_for("public_page", lang=lang, page_slug=OLD_UK_REDIRECTS[page_slug]), code=301)
        landing = seo_landing_for_slug(lang, page_slug)
        if landing:
            return render_template("seo_landing.html", landing=landing, lang=lang, slug=page_slug)
        abort(404)
    faqs = None
    if page_slug == "faq":
        faq_rows_raw = get_db().execute(
            "SELECT * FROM faqs WHERE is_active = 1 ORDER BY sort_order, id LIMIT 20"
        ).fetchall()
        # Replace {COMPANY_NAME} placeholders in FAQ answers at render time
        company = get_settings()["company_name"]
        faq_rows = []
        for row in faq_rows_raw:
            d = dict(row)
            for col in ("question_en", "answer_en", "question_fr", "answer_fr"):
                if d.get(col) and "{COMPANY_NAME}" in d[col]:
                    d[col] = d[col].replace("{COMPANY_NAME}", company)
            faq_rows.append(d)
        faqs = faq_rows
    return render_template("page.html", page=page, lang=lang, faqs=faqs)


@app.route(f"{ADMIN_URL_PREFIX}/login", methods=("GET", "POST"))
def admin_login():
    if request.method == "POST":
        username = request.form.get("username", "").strip()
        password = request.form.get("password", "")
        user = get_db().execute(
            "SELECT * FROM users WHERE username = ? AND is_active = 1", (username,)
        ).fetchone()
        if user and check_password_hash(user["password_hash"], password):
            default_password_active = bool(user["must_change_password"]) or (
                username == "admin" and password == "admin123"
            )
            if default_password_active and not user["must_change_password"]:
                get_db().execute("UPDATE users SET must_change_password = 1 WHERE id = ?", (user["id"],))
                get_db().commit()
            session["user_id"] = user["id"]
            session["username"] = user["username"]
            session["default_password_active"] = default_password_active
            flash("Welcome back.", "success")
            return redirect(request.args.get("next") or url_for("admin_dashboard"))
        flash("Invalid username or password.", "warning")
    return render_template("admin/login.html")


@app.route(f"{ADMIN_URL_PREFIX}/logout", methods=("POST",))
def admin_logout():
    session.clear()
    flash("Signed out.", "success")
    return redirect(url_for("admin_login"))


@app.route(ADMIN_URL_PREFIX)
@login_required
def admin_dashboard():
    stats = {
        "listings": get_db().execute("SELECT COUNT(*) FROM listings WHERE COALESCE(is_deleted,0)=0 AND is_published=1").fetchone()[0],
        "leads": get_db().execute("SELECT COUNT(*) FROM leads").fetchone()[0],
        "unread_leads": get_db().execute("SELECT COUNT(*) FROM leads WHERE is_read = 0").fetchone()[0],
        "pages": get_db().execute("SELECT COUNT(*) FROM site_pages").fetchone()[0],
    }
    return render_template("admin/dashboard.html", stats=stats)


@app.route(f"{ADMIN_URL_PREFIX}/change-password", methods=("GET", "POST"))
@login_required
def admin_change_password():
    user = current_user()
    if request.method == "POST":
        current_password = request.form.get("current_password", "")
        new_password = request.form.get("new_password", "")
        confirm_password = request.form.get("confirm_password", "")
        if not check_password_hash(user["password_hash"], current_password):
            flash("Current password is incorrect.", "warning")
        elif len(new_password) < 8:
            flash("New password must be at least 8 characters.", "warning")
        elif new_password != confirm_password:
            flash("New passwords do not match.", "warning")
        else:
            get_db().execute(
                """
                UPDATE users
                SET password_hash = ?, must_change_password = 0, updated_at = ?
                WHERE id = ?
                """,
                (generate_password_hash(new_password), now_utc(), user["id"]),
            )
            get_db().commit()
            session["default_password_active"] = False
            flash("Password changed.", "success")
            return redirect(url_for("admin_dashboard"))
    return render_template("admin/change_password.html")


@app.route(f"{ADMIN_URL_PREFIX}/leads/<int:id>/resend-smtp", methods=("POST",))
@login_required
def admin_lead_resend_smtp(id):
    ok, message = send_lead_notification_email(id)
    flash(message, "success" if ok else "warning")
    return redirect(url_for("admin_lead_read", id=id))



def read_translation_job_status():
    if not TRANSLATION_JOB_FILE.exists():
        return None
    try:
        return json.loads(TRANSLATION_JOB_FILE.read_text(encoding="utf-8"))
    except Exception:
        return {"status":"unknown","message":"Could not read job state."}


def translation_log_tail(limit=30):
    if not TRANSLATION_JOB_LOG_FILE.exists():
        return []
    try:
        return TRANSLATION_JOB_LOG_FILE.read_text(encoding="utf-8", errors="ignore").splitlines()[-limit:]
    except Exception:
        return []


def write_translation_job_status(state):
    TRANSLATION_JOB_FILE.parent.mkdir(parents=True, exist_ok=True)
    state = dict(state or {})
    state["updated_at"] = now_utc() + "Z" if not now_utc().endswith("Z") else now_utc()
    TRANSLATION_JOB_FILE.write_text(json.dumps(state, indent=2), encoding="utf-8")


def start_translation_background_job(config):
    current = read_translation_job_status() or {}
    if current.get("status") in ("running", "pausing", "cancelling"):
        return False, "A translation job is already running. Stop or pause it first."
    TRANSLATION_JOB_CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
    TRANSLATION_JOB_CONFIG_FILE.write_text(json.dumps(config, indent=2), encoding="utf-8")
    # Replace any stale completed job immediately so the dashboard reflects the
    # mode/content the administrator just submitted while the worker starts.
    write_translation_job_status({
        "status": "starting",
        "content_type": config.get("content_type", "listings"),
        "mode": config.get("mode", "missing"),
        "model": config.get("model", ""),
        "completed": 0,
        "failed": 0,
        "skipped_existing": 0,
        "total_tasks": 0,
        "current": "Preparing translation tasks",
        "errors": [],
        "failed_tasks": [],
        "remaining_tasks": [],
        "config": config,
    })
    script = BASE_DIR / "scripts" / "translation_worker.py"
    try:
        log = open(TRANSLATION_JOB_LOG_FILE, "a", encoding="utf-8")
        process = subprocess.Popen([sys.executable, str(script), "--config", str(TRANSLATION_JOB_CONFIG_FILE), "--state", str(TRANSLATION_JOB_FILE)], cwd=str(BASE_DIR), stdout=log, stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
        return True, f"Translation job started in the background (PID {process.pid})."
    except Exception as error:
        write_translation_job_status({
            "status": "failed",
            "content_type": config.get("content_type", "listings"),
            "mode": config.get("mode", "missing"),
            "model": config.get("model", ""),
            "completed": 0,
            "failed": 0,
            "skipped_existing": 0,
            "total_tasks": 0,
            "current": "",
            "error": str(error),
            "config": config,
            "finished_at": now_utc(),
        })
        return False, str(error)


def failed_tasks_from_job(job):
    tasks = job.get("failed_tasks") or []
    if tasks:
        return tasks
    inferred = []
    content_type = job.get("content_type", "listings")
    for message in job.get("errors") or []:
        if content_type in ("listings", "pages"):
            match = re.search(r"(?:listing|page) #?(\d+) -> ([a-z]{2})", message, re.I)
            if match:
                inferred.append([int(match.group(1)), match.group(2).lower()])
        else:
            match = re.search(r"interface ([^ ]+) -> ([a-z]{2})", message, re.I)
            if match:
                inferred.append([match.group(1), match.group(2).lower()])
    return inferred


@app.route(f"{ADMIN_URL_PREFIX}/translations/control", methods=("POST",))
@login_required
def admin_translations_control():
    action = request.form.get("action", "").strip()
    job = read_translation_job_status() or {}
    status = job.get("status")

    if action == "stop":
        if status not in ("running", "pausing", "paused", "cancelling"):
            flash("No running translation job to stop.", "warning")
        else:
            job["cancel_requested"] = True
            job["status"] = "cancelling"
            write_translation_job_status(job)
            pid = job.get("pid")
            if pid:
                try:
                    os.kill(int(pid), signal.SIGTERM)
                except ProcessLookupError:
                    pass
                except Exception as error:
                    app.logger.warning("Could not terminate translation worker %s: %s", pid, error)
            job["status"] = "cancelled"
            job["finished_at"] = now_utc()
            job["current"] = ""
            write_translation_job_status(job)
            flash("Translation job stopped. Completed translations were kept.", "success")

    elif action == "pause":
        if status != "running":
            flash("Only a running translation job can be paused.", "warning")
        else:
            job["pause_requested"] = True
            job["status"] = "pausing"
            write_translation_job_status(job)
            flash("Pause requested. The worker will pause after the current translation finishes.", "success")

    elif action == "resume":
        remaining = job.get("remaining_tasks") or []
        if status != "paused" or not remaining:
            flash("There is no paused workload to resume.", "warning")
        else:
            config = dict(job.get("config") or {})
            config["explicit_tasks"] = remaining
            config["all_items"] = False
            started, message = start_translation_background_job(config)
            flash(message, "success" if started else "warning")

    elif action == "retry_failed":
        failed = failed_tasks_from_job(job)
        if not failed:
            flash("No failed translations are available to retry.", "warning")
        else:
            config = dict(job.get("config") or {})
            config["explicit_tasks"] = failed
            config["all_items"] = False
            config["mode"] = "regenerate"
            started, message = start_translation_background_job(config)
            flash(message, "success" if started else "warning")
    else:
        flash("Unknown translation job action.", "warning")
    return redirect(request.referrer or url_for("admin_translations", tab=job.get("content_type", "listings")))



def translation_fallback_report(lang):
    if lang == "en":
        return []
    issues = []
    db = get_db()
    for row in db.execute("SELECT translation_key, en, %s AS localized FROM interface_translations ORDER BY translation_key" % lang).fetchall():
        if (row["en"] or "").strip() and not (row["localized"] or "").strip():
            issues.append({"type":"Interface", "item":row["translation_key"], "detail":row["en"]})
    for page in db.execute("SELECT * FROM site_pages WHERE is_published = 1 ORDER BY title_en").fetchall():
        if not (page[f"title_{lang}"] or "").strip():
            issues.append({"type":"Page title", "item":page["slug"], "detail":page["title_en"] or ""})
        elif not (page[f"content_html_{lang}"] or "").strip():
            issues.append({"type":"Page content", "item":page["slug"], "detail":page["title_en"] or ""})
    for faq in db.execute("SELECT * FROM faqs WHERE is_active = 1 ORDER BY sort_order, id").fetchall():
        if not faq_translation_has_content(faq, lang):
            issues.append({"type":"FAQ", "item":str(faq["id"]), "detail":faq["question_en"] or ""})
    return issues

@app.route(f"{ADMIN_URL_PREFIX}/translations")
@login_required
def admin_translations():
    tab = request.args.get("tab", "listings")
    pages = get_db().execute("SELECT * FROM site_pages ORDER BY slug").fetchall()
    listings = get_db().execute("SELECT * FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY updated_at DESC, id DESC").fetchall()
    faqs = get_db().execute("SELECT * FROM faqs ORDER BY sort_order, id").fetchall()
    active_languages = get_active_languages()
    requested_lang = request.args.get("lang", "fr").strip()
    available_codes = [row["code"] for row in active_languages if row["code"] != "en"]
    interface_lang = requested_lang if requested_lang in available_codes else (available_codes[0] if available_codes else "en")
    interface_query = request.args.get("q", "").strip()
    interface_rows = get_db().execute(
        "SELECT * FROM interface_translations WHERE ? = '' OR translation_key LIKE ? OR en LIKE ? ORDER BY translation_key",
        (interface_query, f"%{interface_query}%", f"%{interface_query}%"),
    ).fetchall()
    return render_template("admin/translations.html", tab=tab, pages=pages, listings=listings, faqs=faqs, languages=active_languages, interface_rows=interface_rows, interface_lang=interface_lang, interface_query=interface_query, fallback_report=translation_fallback_report(interface_lang), job=read_translation_job_status(), job_log=translation_log_tail(), models=FIREWORKS_MODELS)


def translation_preflight_count(content_type, mode, langs, selected, all_items):
    """Count tasks using the same missing/regenerate rules as the worker."""
    db = get_db()
    tasks = 0
    skipped = 0
    if content_type == "interface":
        if all_items:
            rows = db.execute("SELECT * FROM interface_translations ORDER BY translation_key").fetchall()
        elif selected:
            placeholders = ",".join("?" for _ in selected)
            rows = db.execute(
                f"SELECT * FROM interface_translations WHERE translation_key IN ({placeholders}) ORDER BY translation_key",
                selected,
            ).fetchall()
        else:
            rows = []
        for row in rows:
            for lang in langs:
                if mode == "missing" and interface_translation_has_content(row, lang):
                    skipped += 1
                else:
                    tasks += 1
        return tasks, skipped
    return None, None


@app.route(f"{ADMIN_URL_PREFIX}/translations/start", methods=("POST",))
@login_required
def admin_translations_start():
    content_type = request.form.get("content_type", "listings")
    mode = request.form.get("mode", "missing")
    langs = [v for v in request.form.getlist("langs") if v in LANGUAGES and v != "en"]
    selected_raw = request.form.getlist("item_ids")
    if content_type in ("listings", "pages", "faqs"):
        selected = [int(v) for v in selected_raw if v.isdigit()]
    else:
        # Validate interface keys against the live database, not the original
        # hardcoded defaults. New trust/location/template keys are added over
        # time and must remain translatable without requiring code-list updates.
        existing_interface_keys = {
            row["translation_key"]
            for row in get_db().execute("SELECT translation_key FROM interface_translations").fetchall()
        }
        selected = [v for v in selected_raw if v in existing_interface_keys]
    all_items = bool(request.form.get("all_items"))
    if content_type not in ("listings", "pages", "faqs", "interface") or not langs:
        flash("Choose content and at least one target language.", "warning")
        return redirect(url_for("admin_translations", tab=content_type))
    if content_type == "interface" and not all_items and not selected:
        flash("Select at least one interface string, or choose Process all interface strings.", "warning")
        return redirect(url_for("admin_translations", tab="interface", lang=langs[0]))

    config={"content_type":content_type,"mode":mode,"langs":langs,"ids":selected,"all_items":all_items,"model":request.form.get("model","").strip() or (get_settings()["translation_model"] or DEFAULT_TRANSLATION_MODEL)}

    needed, skipped = translation_preflight_count(content_type, mode, langs, selected, all_items)
    if content_type == "interface" and needed == 0:
        write_translation_job_status({
            "status": "completed",
            "content_type": content_type,
            "mode": mode,
            "model": config["model"],
            "completed": 0,
            "failed": 0,
            "skipped_existing": skipped or 0,
            "total_tasks": 0,
            "current": "",
            "errors": [],
            "failed_tasks": [],
            "remaining_tasks": [],
            "config": config,
            "finished_at": now_utc(),
        })
        flash(f"No interface translations needed. Skipped {skipped or 0} existing translation{'s' if (skipped or 0) != 1 else ''}.", "info")
        return redirect(url_for("admin_translations", tab="interface", lang=langs[0]))

    started,message=start_translation_background_job(config)
    if started and content_type == "interface":
        flash(f"Queued {needed} interface translation{'s' if needed != 1 else ''}; {skipped or 0} existing translation{'s' if (skipped or 0) != 1 else ''} skipped.", "success")
    flash(message,"success" if started else "warning")
    return redirect(url_for("admin_translations",tab=content_type, lang=(langs[0] if content_type == "interface" else None)))


@app.route(f"{ADMIN_URL_PREFIX}/translations/interface", methods=("POST",))
@login_required
def admin_translations_interface():
    db = get_db(); now = now_utc()
    lang = request.form.get("interface_lang", "it").strip()
    if lang not in LANGUAGES or lang == "en":
        flash("Choose a valid target language.", "warning")
        return redirect(url_for("admin_translations", tab="interface"))
    for key in request.form.getlist("translation_keys"):
        if key not in INTERFACE_TRANSLATION_DEFAULTS:
            continue
        value = request.form.get(f"translation_{key}", "").strip()
        db.execute(f"UPDATE interface_translations SET {lang} = ?, updated_at = ? WHERE translation_key = ?", (value, now, key))
    db.commit(); flash(f"{LANGUAGE_NAMES.get(lang, lang)} interface translations saved.", "success")
    return redirect(url_for("admin_translations", tab="interface", lang=lang))


@app.route(f"{ADMIN_URL_PREFIX}/translations/models", methods=("POST",))
@login_required
def admin_translations_models():
    translation_model=request.form.get("translation_model","").strip() or DEFAULT_TRANSLATION_MODEL
    content_model=request.form.get("content_model","").strip() or DEFAULT_CONTENT_MODEL
    chatbot_model=request.form.get("chatbot_model","").strip() or DEFAULT_CHATBOT_MODEL
    get_db().execute("UPDATE site_settings SET translation_model=?, content_model=?, chatbot_model=?, updated_at=? WHERE id=1",(translation_model,content_model,chatbot_model,now_utc())); get_db().commit()
    flash("AI model assignments saved.","success")
    return redirect(url_for("admin_translations",tab="models"))


@app.route(f"{ADMIN_URL_PREFIX}/settings", methods=("GET", "POST"))
@login_required
def admin_settings():
    settings = get_settings()
    if request.method == "POST":
        old_settings_snapshot = dict(settings)
        uploaded_logo = save_site_upload(request.files.get("logo"), "logo")
        logo_path = uploaded_logo or settings["logo_path"]
        favicon_path = save_site_upload(request.files.get("favicon"), "favicon") or settings["favicon_path"]
        shop_logo_path = save_site_upload(request.files.get("shop_logo"), "shop_logo") or (settings_value(settings, "shop_logo_path", "") or "")
        fields = (
            "company_name",
            "phone",
            "phone2",
            "whatsapp",
            "email",
            "address",
            "address_street",
            "address_city",
            "address_province",
            "address_postal",
            "footer_text",
            "facebook_url",
            "instagram_url",
            "youtube_url",
            "default_language",
            "homepage_hero_title",
            "homepage_hero_subtitle",
            "base_currency",
            "openai_image_model",
            "openai_image_edit_prompt",
            "openai_angle_generation_prompt",
            "fireworks_model",
            "fireworks_custom_model",
            "fireworks_image_review_model",
            "openai_model",
            "ai_provider",
            "zai_model",
            "ai_provider_chatbot",
            "ai_provider_lead_reply",
            "ai_provider_descriptions",
            "ai_provider_pages",
            "ai_provider_inspection",
            "public_stock_prefix",
            "logo_max_height",
            "logo_max_width",
            "smtp_host",
            "smtp_encryption",
            "smtp_port",
            "smtp_username",
            "smtp_from_email",
            "smtp_from_name",
            "smtp_recipient_email",
            "imap_host",
            "imap_port",
            "imap_encryption",
            "imap_username",
            "free_delivery_promo_text",
            "free_delivery_promo_placement",
            "free_delivery_promo_duration",
            "free_delivery_promo_theme",
            "free_delivery_promo_end_date",
            "popular_makes_count",
            "featured_carousel_count",
            "shop_name",
            "shop_phone",
            "shop_email",
            "shop_address",
            "shop_license",
        )
        values = [request.form.get(field, "").strip() for field in fields]
        _street = values[fields.index("address_street")]
        _line2 = " ".join(p for p in (values[fields.index("address_city")], values[fields.index("address_province")], values[fields.index("address_postal")]) if p)
        _joined = ", ".join(p for p in (_street, _line2) if p)
        values[fields.index("address")] = _joined or _street or settings["address"]
        if values[fields.index("ai_provider")] not in ("fireworks", "openai", "zai"):
            values[fields.index("ai_provider")] = "fireworks"
        if not values[fields.index("zai_model")]:
            values[fields.index("zai_model")] = "glm-5.2"
        for feat_field in ("ai_provider_chatbot", "ai_provider_lead_reply", "ai_provider_descriptions", "ai_provider_pages", "ai_provider_inspection"):
            if values[fields.index(feat_field)] not in ("", "fireworks", "openai", "zai"):
                values[fields.index(feat_field)] = ""
        if not values[fields.index("fireworks_model")]:
            values[fields.index("fireworks_model")] = DEFAULT_FIREWORKS_MODEL
        if not values[fields.index("fireworks_image_review_model")]:
            values[fields.index("fireworks_image_review_model")] = DEFAULT_FIREWORKS_IMAGE_REVIEW_MODEL
        if not values[fields.index("openai_model")]:
            values[fields.index("openai_model")] = "gpt-5.2"
        if not values[fields.index("public_stock_prefix")]:
            values[fields.index("public_stock_prefix")] = "CAS"
        if not values[fields.index("free_delivery_promo_text")]:
            values[fields.index("free_delivery_promo_text")] = "Free Canada-wide delivery this {month}"
        if values[fields.index("free_delivery_promo_placement")] not in ("top", "listing", "both"):
            values[fields.index("free_delivery_promo_placement")] = "both"
        try:
            promo_duration_value = int(values[fields.index("free_delivery_promo_duration")] or "6")
        except (TypeError, ValueError):
            promo_duration_value = 6
        values[fields.index("free_delivery_promo_duration")] = str(max(0, min(120, promo_duration_value)))
        if values[fields.index("free_delivery_promo_theme")] not in PROMO_THEME_PRESETS:
            values[fields.index("free_delivery_promo_theme")] = "amber"
        promo_end_date_raw = (values[fields.index("free_delivery_promo_end_date")] or "").strip()
        if promo_end_date_raw:
            try:
                datetime.strptime(promo_end_date_raw[:10], "%Y-%m-%d")
                values[fields.index("free_delivery_promo_end_date")] = promo_end_date_raw[:10]
            except ValueError:
                values[fields.index("free_delivery_promo_end_date")] = ""
        else:
            values[fields.index("free_delivery_promo_end_date")] = ""
        try:
            pmc = int(values[fields.index("popular_makes_count")] or "8")
        except (TypeError, ValueError):
            pmc = 8
        values[fields.index("popular_makes_count")] = str(max(1, min(12, pmc)))
        try:
            fcc = int(values[fields.index("featured_carousel_count")] or "8")
        except (TypeError, ValueError):
            fcc = 8
        values[fields.index("featured_carousel_count")] = str(max(2, min(24, fcc)))
        if values[fields.index("smtp_encryption")] not in ("none", "ssl", "tls"):
            values[fields.index("smtp_encryption")] = "tls"
        try:
            smtp_port_value = int(values[fields.index("smtp_port")] or "587")
        except (TypeError, ValueError):
            smtp_port_value = 587
        values[fields.index("smtp_port")] = str(max(1, min(65535, smtp_port_value)))
        for size_field, default_value, min_value, max_value in (
            ("logo_max_height", "44", 24, 140),
            ("logo_max_width", "180", 80, 420),
        ):
            try:
                parsed_value = int(values[fields.index(size_field)] or default_value)
            except (TypeError, ValueError):
                parsed_value = int(default_value)
            parsed_value = max(min_value, min(max_value, parsed_value))
            values[fields.index(size_field)] = str(parsed_value)
        openai_api_key = request.form.get("openai_api_key", "").strip()
        encrypted_key = settings["openai_api_key_encrypted"] or ""
        if openai_api_key:
            try:
                encrypted_key = encrypt_secret(openai_api_key)
            except RuntimeError as error:
                flash(str(error), "warning")
                return redirect(url_for("admin_settings"))
        if request.form.get("clear_openai_api_key"):
            encrypted_key = ""
        fireworks_api_key_value = request.form.get("fireworks_api_key", "").strip()
        encrypted_fireworks_key = settings["fireworks_api_key_encrypted"] or ""
        if fireworks_api_key_value:
            try:
                encrypted_fireworks_key = encrypt_secret(fireworks_api_key_value)
            except RuntimeError as error:
                flash(str(error), "warning")
                return redirect(url_for("admin_settings"))
        if request.form.get("clear_fireworks_api_key"):
            encrypted_fireworks_key = ""
        zai_api_key_value = request.form.get("zai_api_key", "").strip()
        encrypted_zai_key = settings_value(settings, "zai_api_key_encrypted", "") or ""
        if zai_api_key_value:
            try:
                encrypted_zai_key = encrypt_secret(zai_api_key_value)
            except RuntimeError as error:
                flash(str(error), "warning")
                return redirect(url_for("admin_settings"))
        if request.form.get("clear_zai_api_key"):
            encrypted_zai_key = ""
        smtp_password_value = request.form.get("smtp_password", "").strip()
        encrypted_smtp_password = settings_value(settings, "smtp_password_encrypted", "") or ""
        if smtp_password_value:
            try:
                encrypted_smtp_password = encrypt_secret(smtp_password_value)
            except RuntimeError as error:
                flash(str(error), "warning")
                return redirect(url_for("admin_settings"))
        if request.form.get("clear_smtp_password"):
            encrypted_smtp_password = ""
        imap_password_value = request.form.get("imap_password", "").strip()
        encrypted_imap_password = settings_value(settings, "imap_password_encrypted", "") or ""
        if imap_password_value:
            try:
                encrypted_imap_password = encrypt_secret(imap_password_value)
            except RuntimeError as error:
                flash(str(error), "warning")
                return redirect(url_for("admin_settings"))
        if request.form.get("clear_imap_password"):
            encrypted_imap_password = ""
        toggles = (
            1 if request.form.get("enable_call_button") else 0,
            1 if request.form.get("enable_whatsapp_button") else 0,
            1 if request.form.get("enable_email_button") else 0,
            1 if request.form.get("enable_sticky_contact_form") else 0,
            1 if request.form.get("enable_currency_conversion") else 0,
            1 if request.form.get("ai_generation_enabled") else 0,
            1 if request.form.get("show_internal_stock_publicly") else 0,
            1 if request.form.get("smtp_enabled") else 0,
            1 if request.form.get("smtp_authentication") else 0,
            1 if request.form.get("auto_language_detection") else 0,
            1 if request.form.get("browser_language_fallback") else 0,
            1 if request.form.get("remember_language_choice") else 0,
            1 if request.form.get("free_delivery_promo_enabled") else 0,
            1 if request.form.get("free_delivery_promo_anim_entrance") else 0,
            1 if request.form.get("free_delivery_promo_anim_shimmer") else 0,
            1 if request.form.get("free_delivery_promo_anim_pulse") else 0,
            1 if request.form.get("free_delivery_promo_anim_slide_text") else 0,
            1 if request.form.get("free_delivery_promo_anim_countdown") else 0,
            1 if request.form.get("carfax_button_enabled") else 0,
            1 if request.form.get("savings_banner_enabled") else 0,
            1 if request.form.get("inspection_button_enabled") else 0,
        )
        get_db().execute(
            f"""
            UPDATE site_settings
            SET {", ".join(f"{field} = ?" for field in fields)},
                enable_call_button = ?, enable_whatsapp_button = ?,
                enable_email_button = ?, enable_sticky_contact_form = ?,
                enable_currency_conversion = ?, ai_generation_enabled = ?,
                show_internal_stock_publicly = ?,
                smtp_enabled = ?, smtp_authentication = ?,
                auto_language_detection = ?, browser_language_fallback = ?, remember_language_choice = ?,
                free_delivery_promo_enabled = ?,
                free_delivery_promo_anim_entrance = ?, free_delivery_promo_anim_shimmer = ?,
                free_delivery_promo_anim_pulse = ?, free_delivery_promo_anim_slide_text = ?,
                free_delivery_promo_anim_countdown = ?,
                carfax_button_enabled = ?, savings_banner_enabled = ?,
                inspection_button_enabled = ?,
                logo_path = ?, favicon_path = ?, openai_api_key_encrypted = ?,
                fireworks_api_key_encrypted = ?, zai_api_key_encrypted = ?, smtp_password_encrypted = ?,
                imap_password_encrypted = ?,
                openai_logo_file_id = ?, shop_logo_path = ?, updated_at = ?
            WHERE id = 1
            """,
            (
                *values,
                *toggles,
                logo_path,
                favicon_path,
                encrypted_key,
                encrypted_fireworks_key,
                encrypted_zai_key,
                encrypted_smtp_password,
                encrypted_imap_password,
                "" if uploaded_logo else (settings["openai_logo_file_id"] or ""),
                shop_logo_path,
                now_utc(),
            ),
        )
        get_db().commit()
        current_settings_snapshot = dict(get_settings())
        tracked_contact_fields = ("company_name", "phone", "whatsapp", "email", "address", "footer_text")
        old_contact = {key: old_settings_snapshot.get(key) for key in tracked_contact_fields}
        new_contact = {key: current_settings_snapshot.get(key) for key in tracked_contact_fields}
        if old_contact != new_contact:
            trust_audit("company/contact information changed", "site_settings", 1, old_contact, new_contact)
            get_db().commit()
        if request.form.get("settings_action") == "test_smtp_email":
            current_settings = get_settings()
            recipient = request.form.get("smtp_test_recipient", "").strip() or settings_value(current_settings, "smtp_recipient_email", "") or settings_value(current_settings, "email", "")
            ok, message = smtp_send_message(
                current_settings,
                "Canada Auto Sales SMTP test",
                "This is a test email from your Canada Auto Sales website SMTP settings.",
                "<p>This is a test email from your <strong>Canada Auto Sales</strong> website SMTP settings.</p>",
                recipient=recipient,
            )
            flash(message, "success" if ok else "warning")
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "regenerate_image_names":
            summary = regenerate_image_filenames()
            get_db().commit()
            flash(
                f"Image filenames regenerated: {summary['renamed']} files renamed, {summary['updated_rows']} image rows updated.",
                "success",
            )
            if summary["messages"]:
                flash("Some files were skipped: " + " | ".join(summary["messages"]), "warning")
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "archive_unused_listing_files":
            summary = archive_unused_listing_files()
            flash(
                f"Archived {summary['moved']} unused listing image file{'s' if summary['moved'] != 1 else ''}.",
                "success",
            )
            if summary["skipped"]:
                flash(f"{summary['skipped']} unused file{'s' if summary['skipped'] != 1 else ''} could not be moved.", "warning")
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "migrate_ai_edit_folders":
            summary = migrate_ai_edits_to_stock_folders()
            get_db().commit()
            flash(
                f"AI edit folders migrated: {summary['moved']} files moved, {summary['updated_rows']} image rows updated.",
                "success",
            )
            if summary["messages"]:
                flash("Some AI files were skipped: " + " | ".join(summary["messages"]), "warning")
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "sanitize_public_content":
            changed = sanitize_public_listing_content(regenerate_slugs=True)
            summary = regenerate_image_filenames()
            archive_unused_listing_files()
            get_db().commit()
            flash(
                f"Public listing cleanup complete: {changed} listings updated and {summary['renamed']} image files renamed.",
                "success",
            )
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "generate_missing_image_variants":
            started, message = start_image_variant_background_job(regenerate=False)
            flash(message, "success" if started else "warning")
            return redirect(url_for("admin_settings"))
        if request.form.get("settings_action") == "regenerate_all_image_variants":
            started, message = start_image_variant_background_job(regenerate=True)
            flash(("Background full regeneration started." if started else message), "success" if started else "warning")
            if started:
                flash("This will rebuild all thumb/gallery files in the background using the current originals. You can leave the page; it will keep running.", "success")
            return redirect(url_for("admin_settings"))
        flash("Settings updated.", "success")
        return redirect(url_for("admin_settings"))
    return render_template(
        "admin/settings.html",
        settings=settings,
        languages=get_active_languages(),
        openai_key=openai_key_status(settings),
        ai_status=ai_provider_status(settings),
        smtp_status=smtp_status(settings),
        fireworks_models=FIREWORKS_MODELS,
        fireworks_image_review_models=FIREWORKS_IMAGE_REVIEW_MODELS,
        image_variant_job=read_image_variant_job_status(),
        image_variant_log=image_variant_log_tail(),
    )


@app.route(f"{ADMIN_URL_PREFIX}/import", methods=("GET", "POST"))
@login_required
def admin_import():
    summary = None
    preview = None
    source_db = request.form.get("source_db", "").strip() if request.method == "POST" else request.args.get("source_db", "").strip()
    source_search = request.form.get("source_search", "").strip() if request.method == "POST" else request.args.get("source_search", "").strip()
    if request.method == "POST":
        action = request.form.get("action", "preview")
        if action == "repair_all":
            repaired = repair_all_image_orders()
            summary = {
                "total_source_listings": 0,
                "listings_selected": 0,
                "listings_created": 0,
                "listings_updated": 0,
                "listings_skipped": 0,
                "images_found": 0,
                "images_imported": 0,
                "images_copied": 0,
                "missing_image_files": 0,
                "primary_images_set": 0,
                "featured_images_updated": 0,
                "repaired_listings": repaired,
                "image_variants_generated": 0,
                "errors": [],
                "dry_run": False,
            }
            flash("Image order and primary images repaired.", "success")
        elif not source_db:
            flash("Enter a source database path.", "warning")
        elif action == "preview":
            preview = load_source_preview(source_db, source_search)
            if preview["error"]:
                flash(preview["error"], "warning")
        else:
            selected_stocks = request.form.getlist("selected_stocks")
            if not selected_stocks and not request.form.get("missing_only"):
                flash("Select at least one listing, or choose Import missing only.", "warning")
                preview = load_source_preview(source_db, source_search)
            else:
                summary = import_inventory_from_source(
                    source_db=source_db,
                    copy_images=bool(request.form.get("copy_images")),
                    overwrite=bool(request.form.get("overwrite")),
                    publish=bool(request.form.get("publish")),
                    default_status=request.form.get("default_status", "Available").strip() or "Available",
                    dry_run=bool(request.form.get("dry_run")),
                    selected_stocks=selected_stocks,
                    missing_only=bool(request.form.get("missing_only")),
                    repair_after_import=bool(request.form.get("repair_after_import")),
                    generated_image_mode=request.form.get("generated_image_mode", "source"),
                    generate_variants_after_import=bool(request.form.get("generate_variants_after_import")),
                )
                preview = load_source_preview(source_db, source_search)
                if summary["errors"]:
                    flash("Import finished with messages. Review the summary below.", "warning")
                else:
                    flash("Import finished.", "success")
    elif source_db:
        preview = load_source_preview(source_db, source_search)
    return render_template(
        "admin/import.html",
        summary=summary,
        preview=preview,
        source_db=source_db,
        source_search=source_search,
        statuses=listing_statuses(),
        summary_fields=IMPORT_SUMMARY_FIELDS,
    )


@app.route(f"{ADMIN_URL_PREFIX}/menus", methods=("GET", "POST"))
@login_required
def admin_menus():
    db = get_db()
    if request.method == "POST":
        timestamp = now_utc()
        item_ids = request.form.getlist("item_id")
        for item_id in item_ids:
            db.execute(
                """
                UPDATE menu_items
                SET label_en = ?, url = ?, location = ?, sort_order = ?, is_active = ?, updated_at = ?
                WHERE id = ?
                """,
                (
                    request.form.get(f"label_en_{item_id}", "").strip(),
                    request.form.get(f"url_{item_id}", "").strip(),
                    request.form.get(f"location_{item_id}", "header"),
                    request.form.get(f"sort_order_{item_id}", "0") or 0,
                    1 if request.form.get(f"is_active_{item_id}") else 0,
                    timestamp,
                    item_id,
                ),
            )
        new_label = request.form.get("new_label_en", "").strip()
        new_url = request.form.get("new_url", "").strip()
        if new_label and new_url:
            db.execute(
                """
                INSERT INTO menu_items (label_en, url, location, sort_order, is_active, created_at, updated_at)
                VALUES (?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    new_label,
                    new_url,
                    request.form.get("new_location", "header"),
                    request.form.get("new_sort_order", "100") or 100,
                    1,
                    timestamp,
                    timestamp,
                ),
            )
        db.commit()
        flash("Menus updated.", "success")
        return redirect(url_for("admin_menus"))
    items = db.execute("SELECT * FROM menu_items ORDER BY location, sort_order, id").fetchall()
    return render_template("admin/menus.html", items=items)


@app.route(f"{ADMIN_URL_PREFIX}/listings")
@login_required
def admin_listings():
    query, params, filters = build_listing_query(published_only=False)
    listings = get_db().execute(query, params).fetchall()
    total_count = get_db().execute(
        "SELECT COUNT(*) FROM listings WHERE COALESCE(is_deleted, 0) = 0"
    ).fetchone()[0]
    inspection_statuses = {
        row["listing_id"]: row["status"]
        for row in get_db().execute("SELECT listing_id, status FROM inspection_reports").fetchall()
    }
    return render_template(
        "admin/listings.html",
        listings=listings,
        total_count=total_count,
        shown_count=len(listings),
        filters=filters,
        filter_options=listing_filter_options(published_only=False),
        languages=get_active_languages(),
        listing_translation_summary=listing_translation_summary,
        inspection_statuses=inspection_statuses,
        trust_saved_locations=get_db().execute("SELECT * FROM saved_locations WHERE is_active=1 ORDER BY internal_name").fetchall(),
    )


# Crawler-DB import job state (admin upload). One at a time.
_import_inventory_jobs = {"current": None}


@app.route(f"{ADMIN_URL_PREFIX}/listings/import-crawler", methods=("POST",))
@login_required
def admin_listings_import_crawler():
    """Import crawler vehicles into the site inventory in the background. Two ways in:
      (1) paste a SERVER PATH to a .zip, a .sqlite3, or a folder containing both
          vehicles.sqlite3 + images/  (use this for big imports — SFTP the files over
          with WinSCP, which can resume, then point the import at the path).
      (2) upload a .sqlite3 or .zip via the browser (smaller imports).
    Image paths in the crawler DB are relative (images\\VIN\\000_xxx.jpg) and resolve
    against the DB's parent dir."""
    if _import_inventory_jobs.get("current") and not _import_inventory_jobs["current"].get("finished"):
        flash("An import is already running — wait for it to finish.", "warning")
        return redirect(url_for("admin_listings"))

    import zipfile
    copy_images = bool(request.form.get("copy_images"))
    overwrite = bool(request.form.get("overwrite"))
    publish = bool(request.form.get("publish"))
    missing_only = bool(request.form.get("missing_only"))
    images_only = bool(request.form.get("images_only"))

    # ── Server-path mode (preferred for large imports) ───────────────────────
    server_path = (request.form.get("server_path") or "").strip()
    db_path = None
    cleanup_dir = None  # a temp dir we created (zip extraction); None = don't delete user files
    if server_path:
        sp = Path(server_path).expanduser()
        if not sp.exists():
            flash(f"Server path not found: {server_path}", "warning")
            return redirect(url_for("admin_listings"))
        if sp.is_file() and sp.suffix.lower() == ".zip":
            work_dir = INSTANCE_DIR / "crawler_uploads" / datetime.utcnow().strftime("%Y%m%d%H%M%S")
            work_dir.mkdir(parents=True, exist_ok=True)
            try:
                with zipfile.ZipFile(str(sp)) as zf:
                    zf.extractall(str(work_dir))
            except Exception as e:
                flash(f"Could not extract zip: {e}", "warning")
                return redirect(url_for("admin_listings"))
            found = list(work_dir.rglob("vehicles.sqlite3")) or list(work_dir.rglob("*.sqlite3"))
            if not found:
                flash("No .sqlite3 file found inside the zip.", "warning")
                return redirect(url_for("admin_listings"))
            db_path = found[0]
            cleanup_dir = work_dir
        elif sp.is_file() and sp.suffix.lower() in (".sqlite3", ".sqlite", ".db"):
            db_path = sp  # images resolve against sp.parent (the user's folder — not deleted)
        elif sp.is_dir():
            found = list(sp.rglob("vehicles.sqlite3")) or list(sp.rglob("*.sqlite3"))
            if not found:
                flash(f"No .sqlite3 file found in folder: {server_path}", "warning")
                return redirect(url_for("admin_listings"))
            db_path = found[0]  # user's folder — not deleted
        else:
            flash("Server path must be a .zip, a .sqlite3, or a folder.", "warning")
            return redirect(url_for("admin_listings"))
    else:
        # ── Browser-upload mode ──────────────────────────────────────────────
        if "crawler_db" not in request.files:
            flash("Choose a file to upload, or enter a server path.", "warning")
            return redirect(url_for("admin_listings"))
        f = request.files["crawler_db"]
        if not f or not f.filename:
            flash("Choose a file to upload, or enter a server path.", "warning")
            return redirect(url_for("admin_listings"))
        fname = secure_filename(f.filename) or "crawler"
        is_zip = fname.lower().endswith(".zip")
        is_db = fname.lower().endswith((".sqlite3", ".sqlite", ".db"))
        if not (is_zip or is_db):
            flash("Upload a .sqlite3 (crawler DB) or a .zip (DB + images folder), or use a server path.", "warning")
            return redirect(url_for("admin_listings"))
        upload_dir = INSTANCE_DIR / "crawler_uploads"
        upload_dir.mkdir(parents=True, exist_ok=True)
        work_dir = upload_dir / datetime.utcnow().strftime("%Y%m%d%H%M%S")
        work_dir.mkdir(parents=True, exist_ok=True)
        target = work_dir / fname
        try:
            f.save(str(target))
        except Exception as e:
            flash(f"Could not save uploaded file: {e}", "warning")
            return redirect(url_for("admin_listings"))
        db_path = target
        cleanup_dir = work_dir
        if is_zip:
            try:
                with zipfile.ZipFile(str(target)) as zf:
                    zf.extractall(str(work_dir))
            except Exception as e:
                flash(f"Could not extract zip: {e}", "warning")
                return redirect(url_for("admin_listings"))
            found = list(work_dir.rglob("vehicles.sqlite3")) or list(work_dir.rglob("*.sqlite3"))
            if not found:
                flash("No .sqlite3 file found inside the zip.", "warning")
                return redirect(url_for("admin_listings"))
            db_path = found[0]

    _import_inventory_jobs["current"] = {"finished": False, "summary": {}, "error": None, "started_at": now_utc()}
    job_ref = _import_inventory_jobs

    def _run():
        with app.app_context():
            try:
                if images_only:
                    summary = repair_crawler_images_for_listings(source_db=str(db_path))
                else:
                    summary = import_cars_from_source(
                        source_db=str(db_path), copy_images=copy_images, overwrite=overwrite,
                        publish=publish, default_status="Available", dry_run=False,
                        selected_vins=None, limit=None, missing_only=missing_only,
                    )
                # Kick off thumbnail/_gallery variant generation in the background
                # so newly copied images get card thumbnails without a manual step.
                if summary.get("images_copied"):
                    started, _message = start_image_variant_background_job(regenerate=False)
                    summary["thumbnails_job_started"] = bool(started)
                job_ref["current"]["summary"] = summary
                job_ref["current"]["error"] = (summary.get("errors") or [None])[0] if summary.get("errors") else None
            except Exception as e:
                job_ref["current"]["error"] = f"{type(e).__name__}: {e}"
            finally:
                job_ref["current"]["finished"] = True
                # Only clean up a temp dir WE created (zip extraction / browser upload).
                # Never delete the user's server-path folder/files.
                if cleanup_dir:
                    try:
                        shutil.rmtree(cleanup_dir, ignore_errors=True)
                    except Exception:
                        pass

    threading.Thread(target=_run, daemon=True).start()
    if images_only:
        flash("Image repair started in the background. The page will show progress.", "success")
    else:
        flash("Import started in the background. The page will show progress.", "success")
    return redirect(url_for("admin_listings"))


@app.route(f"{ADMIN_URL_PREFIX}/listings/import-crawler/status")
@login_required
def admin_listings_import_crawler_status():
    return jsonify(_import_inventory_jobs.get("current") or {"finished": True, "summary": {}, "error": None})


def purge_listing_permanently(listing_id):
    """Delete a listing AND everything tied to it (images, AI edits, turbo tracking,
    inspection reports, redirects) + the image FILES on disk. Frees disk space (use for
    sold cars). Returns a small summary dict. Unlike the soft 'is_deleted=1' archive,
    this is irreversible."""
    db = get_db()
    listing = db.execute("SELECT id, slug, folder_path, featured_image FROM listings WHERE id=?", (listing_id,)).fetchone()
    if not listing:
        return {"deleted": False}
    files_removed = 0
    folders_removed = []
    # 1) Image files: the listing's folder under static/uploads/listings/<folder_path>
    folder = (listing["folder_path"] or "").strip()
    if folder:
        fdir = LISTING_UPLOADS_DIR / folder
        if fdir.exists() and fdir.is_dir():
            try:
                for f in fdir.iterdir():
                    try:
                        f.unlink()
                        files_removed += 1
                    except Exception:
                        pass
                shutil.rmtree(fdir, ignore_errors=True)
                folders_removed.append(folder)
            except Exception:
                pass
    # Also remove any image files referenced by local_path (covers non-folder layouts).
    for img in db.execute("SELECT local_path FROM listing_images WHERE listing_id=?", (listing_id,)).fetchall():
        lp = local_static_image_path(img["local_path"] or "")
        if lp:
            p = Path(lp.lstrip("/"))
            target = (BASE_DIR / p) if not Path(lp).is_absolute() else Path(lp)
            try:
                if target.is_file():
                    target.unlink()
                    files_removed += 1
            except Exception:
                pass
    # 2) AI-generated image variants (thumb/gallery) for this listing's images.
    for img in db.execute("SELECT local_path FROM listing_images WHERE listing_id=?", (listing_id,)).fetchall():
        for variant in ("_thumb.webp", "_gallery.webp"):
            base = local_static_image_path(img["local_path"] or "")
            if base:
                vp = Path(base.lstrip("/").rsplit(".", 1)[0] + variant)
                vt = (BASE_DIR / vp) if not vp.is_absolute() else vp
                try:
                    if vt.is_file():
                        vt.unlink()
                        files_removed += 1
                except Exception:
                    pass
    # 3) Related DB rows.
    for stmt in (
        "DELETE FROM ai_image_edits WHERE source_image_id IN (SELECT id FROM listing_images WHERE listing_id=?)",
        "DELETE FROM listing_images WHERE listing_id=?",
        "DELETE FROM turbo_posted WHERE listing_id=?",
        "DELETE FROM turbo_payloads WHERE listing_id=?",
        "DELETE FROM turbo_history WHERE listing_id=?",
        "DELETE FROM turbo_descriptions WHERE listing_id=?",
        "DELETE FROM turbo_campaign_vehicles WHERE listing_id=?",
        "DELETE FROM inspection_reports WHERE listing_id=?",
        "DELETE FROM redirects WHERE listing_id=?",
        "DELETE FROM leads WHERE listing_id=?",
        "DELETE FROM listings WHERE id=?",
    ):
        try:
            db.execute(stmt, (listing_id,))
        except Exception:
            pass
    db.commit()
    return {"deleted": True, "files_removed": files_removed, "folders_removed": folders_removed}


@app.route(f"{ADMIN_URL_PREFIX}/listings/bulk", methods=("POST",))
@login_required
def admin_listings_bulk():
    selected_ids = [int(value) for value in request.form.getlist("listing_ids") if value.isdigit()]
    action = request.form.get("bulk_action", "")
    if not selected_ids:
        flash("Select at least one listing.", "warning")
        return redirect(url_for("admin_listings"))
    placeholders = ", ".join("?" for _ in selected_ids)
    db = get_db()
    if action == "delete":
        db.execute(f"UPDATE listings SET is_deleted = 1, is_published = 0, updated_at = ? WHERE id IN ({placeholders})", (now_utc(), *selected_ids))
        flash("Selected listings were archived and hidden from inventory.", "success")
    elif action == "delete_permanent":
        # Permanently purge sold listings + all related data + image FILES (frees disk).
        files = 0
        for lid in selected_ids:
            res = purge_listing_permanently(lid)
            files += res.get("files_removed", 0)
        flash(f"Permanently deleted {len(selected_ids)} listing(s) + all related data and {files} image file(s). Disk space freed. Irreversible.", "success")
    elif action == "restore":
        db.execute(f"UPDATE listings SET is_deleted = 0, is_published = 0, updated_at = ? WHERE id IN ({placeholders})", (now_utc(), *selected_ids))
        flash(f"Restored {len(selected_ids)} listing{'s' if len(selected_ids) != 1 else ''} (now drafts — review and publish when ready).", "success")
    elif action == "publish":
        db.execute(f"UPDATE listings SET is_published = 1, is_deleted = 0, updated_at = ? WHERE id IN ({placeholders})", (now_utc(), *selected_ids))
        flash("Selected listings published.", "success")
    elif action == "unpublish":
        db.execute(f"UPDATE listings SET is_published = 0, updated_at = ? WHERE id IN ({placeholders})", (now_utc(), *selected_ids))
        flash("Selected listings unpublished.", "success")
    elif action in ("Available", "Reserved", "Awaiting Confirmation", "Sold", "Unavailable", "Archived"):
        old_statuses = [dict(row) for row in db.execute(f"SELECT id,status FROM listings WHERE id IN ({placeholders})", selected_ids).fetchall()]
        db.execute(f"UPDATE listings SET status = ?, updated_at = ? WHERE id IN ({placeholders})", (action, now_utc(), *selected_ids))
        trust_audit("bulk availability changed", "listing", None, old_statuses, {"listing_ids": selected_ids, "status": action})
        flash("Selected listing statuses updated.", "success")
    elif action == "set_category":
        category = request.form.get("bulk_category", "").strip()
        if category:
            db.execute(f"UPDATE listings SET category = ?, updated_at = ? WHERE id IN ({placeholders})", (category, now_utc(), *selected_ids))
            flash("Selected listing categories updated.", "success")
    elif action == "set_make":
        make = request.form.get("bulk_make", "").strip()
        if make:
            db.execute(f"UPDATE listings SET make = ?, updated_at = ? WHERE id IN ({placeholders})", (make, now_utc(), *selected_ids))
            flash("Selected listing makes updated.", "success")
    elif action == "repair_images":
        repaired = repair_all_image_orders(selected_ids)
        flash(f"Repaired image order for {repaired} listings.", "success")
        return redirect(url_for("admin_listings"))
    elif action == "rename_images":
        summary = regenerate_image_filenames(selected_ids)
        flash(
            f"Renamed {summary['renamed']} image file{'s' if summary['renamed'] != 1 else ''} and updated {summary['updated_rows']} image row{'s' if summary['updated_rows'] != 1 else ''}.",
            "success",
        )
        if summary["messages"]:
            flash("Some files were skipped: " + " | ".join(summary["messages"]), "warning")
    elif action == "regen_refs":
        for listing_id in selected_ids:
            ensure_public_stock_number(listing_id)
        flash("Public references regenerated where missing.", "success")
    elif action == "reset_refs":
        changed = reset_public_stock_numbers(listing_ids=selected_ids)
        flash(f"Reset {changed} public reference{'s' if changed != 1 else ''}.", "success")
    elif action == "set_carfax_url":
        carfax_url = request.form.get("bulk_carfax_url", "").strip()
        if carfax_url:
            db.execute(f"UPDATE listings SET carfax_url = ?, updated_at = ? WHERE id IN ({placeholders})", (carfax_url, now_utc(), *selected_ids))
            flash(f"Carfax URL set for {len(selected_ids)} listing{'s' if len(selected_ids) != 1 else ''}.", "success")
        else:
            db.execute(f"UPDATE listings SET carfax_url = NULL, updated_at = ? WHERE id IN ({placeholders})", (now_utc(), *selected_ids))
            flash(f"Carfax URL cleared for {len(selected_ids)} listing{'s' if len(selected_ids) != 1 else ''}.", "success")
    elif action in ("set_carfax_on", "set_carfax_off"):
        enabled = 1 if action == "set_carfax_on" else 0
        db.execute(f"UPDATE listings SET carfax_enabled = ?, updated_at = ? WHERE id IN ({placeholders})", (enabled, now_utc(), *selected_ids))
        flash(f"Carfax button {'enabled' if enabled else 'disabled'} for {len(selected_ids)} listing{'s' if len(selected_ids) != 1 else ''}.", "success")
    else:
        flash("Choose a valid bulk action.", "warning")
        return redirect(url_for("admin_listings"))
    db.commit()
    return redirect(url_for("admin_listings"))


@app.route(f"{ADMIN_URL_PREFIX}/carfax-vin-update", methods=("GET", "POST"))
@login_required
def admin_carfax_vin_update():
    db = get_db()
    if request.method == "POST":
        action = request.form.get("action", "")
        if action == "update":
            raw = request.form.get("vin_pairs", "")
            pairs = []
            for line in raw.replace(",", "\n").split("\n"):
                line = line.strip()
                if ":" in line:
                    old, new = line.split(":", 1)
                    old, new = old.strip(), new.strip()
                    if old and new and old != new:
                        pairs.append((old, new))
            results = {"updated": 0, "skipped": 0, "errors": []}
            for old_vin, new_vin in pairs:
                listing = db.execute("SELECT id, stock_number FROM listings WHERE vin = ?", (old_vin,)).fetchone()
                if not listing:
                    results["skipped"] += 1
                    results["errors"].append(f"{old_vin}: no listing found")
                    continue
                # Rename the HTML file + replace VIN inside
                old_path = os.path.join(CARFAX_REPORTS_DIR, f"{old_vin}.html")
                new_path = os.path.join(CARFAX_REPORTS_DIR, f"{new_vin}.html")
                if os.path.isfile(old_path):
                    with open(old_path, "r", encoding="utf-8") as f:
                        content = f.read()
                    content = content.replace(old_vin, new_vin)
                    with open(new_path, "w", encoding="utf-8") as f:
                        f.write(content)
                    os.remove(old_path)
                # Update ONLY the vin column — NOT stock_number, NOT id
                db.execute("UPDATE listings SET vin = ?, updated_at = ? WHERE id = ?", (new_vin, now_utc(), listing["id"]))
                db.execute("INSERT INTO carfax_vin_changes (listing_id, old_vin, new_vin, changed_at, reverted) VALUES (?, ?, ?, ?, 0)",
                           (listing["id"], old_vin, new_vin, now_utc()))
                results["updated"] += 1
            db.commit()
            flash(f"Updated {results['updated']} VIN(s). Skipped {results['skipped']}.", "success" if results["updated"] else "warning")
            if results["errors"]:
                flash("Issues: " + " | ".join(results["errors"][:5]), "warning")
        elif action == "revert":
            changes = db.execute("SELECT * FROM carfax_vin_changes WHERE reverted = 0 ORDER BY id DESC").fetchall()
            reverted_count = 0
            for change in changes:
                old_vin, new_vin = change["old_vin"], change["new_vin"]
                new_path = os.path.join(CARFAX_REPORTS_DIR, f"{new_vin}.html")
                old_path = os.path.join(CARFAX_REPORTS_DIR, f"{old_vin}.html")
                if os.path.isfile(new_path):
                    with open(new_path, "r", encoding="utf-8") as f:
                        content = f.read()
                    content = content.replace(new_vin, old_vin)
                    with open(old_path, "w", encoding="utf-8") as f:
                        f.write(content)
                    os.remove(new_path)
                db.execute("UPDATE listings SET vin = ?, updated_at = ? WHERE id = ?", (old_vin, now_utc(), change["listing_id"]))
                db.execute("UPDATE carfax_vin_changes SET reverted = 1 WHERE id = ?", (change["id"],))
                reverted_count += 1
            db.commit()
            flash(f"Reverted {reverted_count} VIN change(s).", "success")
        return redirect(url_for("admin_carfax_vin_update"))
    changes = db.execute("SELECT * FROM carfax_vin_changes ORDER BY id DESC LIMIT 50").fetchall()
    return render_template("admin/carfax_vin_update.html", changes=changes)


def listing_form_values(existing=None):
    title = request.form.get("title_en", "").strip()
    slug = request.form.get("slug", "").strip() or slugify(title)
    stock_number = request.form.get("stock_number", "").strip()
    if not stock_number and existing:
        stock_number = existing["stock_number"]
    if not stock_number:
        stock_number = f"MES-{datetime.utcnow().strftime('%H%M%S')}"
    return {
        "stock_number": stock_number,
        "public_stock_number": request.form.get("public_stock_number", "").strip() or (existing["public_stock_number"] if existing else ""),
        "slug": slug,
        "title_en": title,
        "short_description_en": request.form.get("short_description_en", "").strip(),
        "description_html_en": request.form.get("description_html_en", "").strip(),
        "seo_title_en": request.form.get("seo_title_en", "").strip() or title,
        "meta_description_en": request.form.get("meta_description_en", "").strip(),
        "year": request.form.get("year", "").strip(),
        "make": request.form.get("make", "").strip(),
        "model": request.form.get("model", "").strip(),
        "category": request.form.get("category", "").strip(),
        "hours": request.form.get("hours", "").strip(),
        "price": request.form.get("price", "").strip(),
        "odometer": request.form.get("odometer", "").strip(),
        "price_label": request.form.get("price_label", "").strip(),
        "status": request.form.get("status", "Available"),
        "source_url": request.form.get("source_url", "").strip(),
        "folder_path": request.form.get("folder_path", "").strip(),
        "featured_image": request.form.get("featured_image", "").strip(),
        "is_featured": 1 if request.form.get("is_featured") else 0,
        "is_published": 1 if request.form.get("is_published") else 0,
        "carfax_url": request.form.get("carfax_url", "").strip(),
        "carfax_enabled": 1 if request.form.get("carfax_enabled") else 0,
        "carfax_accident_notes": request.form.get("carfax_accident_notes", "").strip(),
    }


@app.route(f"{ADMIN_URL_PREFIX}/listings/create", methods=("GET", "POST"))
@login_required
def admin_listing_create():
    if request.method == "POST":
        values = listing_form_values()
        timestamp = now_utc()
        columns = list(values.keys()) + ["created_at", "updated_at"]
        get_db().execute(
            f"INSERT INTO listings ({', '.join(columns)}) VALUES ({', '.join('?' for _ in columns)})",
            (*values.values(), timestamp, timestamp),
        )
        listing_id = get_db().execute("SELECT last_insert_rowid()").fetchone()[0]
        ensure_public_stock_number(listing_id)
        get_db().commit()
        flash("Listing created.", "success")
        return redirect(url_for("admin_listings"))
    return render_template("admin/listing_form.html", listing=None, statuses=listing_statuses())


@app.route(f"{ADMIN_URL_PREFIX}/listings/edit/<int:id>/generate-angle", methods=("POST",))
@login_required
def admin_listing_generate_angle(id):
    listing = get_db().execute("SELECT * FROM listings WHERE id = ?", (id,)).fetchone()
    if listing is None:
        abort(404)
    reference_id = request.form.get("angle_reference_image_id", "")
    requested_angle = request.form.get("angle_name", "").strip()
    image = get_db().execute(
        "SELECT * FROM listing_images WHERE id = ? AND listing_id = ?", (reference_id, id)
    ).fetchone() if reference_id.isdigit() else None
    if not image:
        return jsonify({"ok": False, "error": "Reference image was not found."}), 404
    angle_name = angle_label(requested_angle)
    try:
        edit_id = generate_ai_image_edit(
            listing,
            image,
            "",
            edit_kind="angle",
            angle_name=angle_name,
        )
    except Exception as error:
        get_db().rollback()
        return jsonify({"ok": False, "error": str(error), "angle": angle_name}), 500
    return jsonify({"ok": True, "edit_id": edit_id, "angle": angle_name})


def strip_html_to_text(html_value):
    if not html_value:
        return ""
    text = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", str(html_value))
    text = re.sub(r"(?i)<br\s*/?>", "\n", text)
    text = re.sub(r"(?i)</p\s*>", "\n\n", text)
    text = re.sub(r"<[^>]+>", " ", text)
    text = unescape(text)
    text = re.sub(r"[ \t]+", " ", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def listing_description_export_html(listing, lang="en"):
    title = get_translated(listing, "title", lang) or listing["title_en"] or listing["slug"]
    short_description = get_translated(listing, "short_description", lang)
    description_html = get_translated(listing, "description_html", lang)
    specs = [
        ("Stock Reference", listing["public_stock_number"] or listing["stock_number"] or ""),
        ("Year", listing["year"] or ""),
        ("Make", listing["make"] or ""),
        ("Model", listing["model"] or ""),
        ("Category", listing["category"] or ""),
        ("Hours", listing["hours"] or ""),
        ("Price", listing["price_label"] or listing["price"] or ""),
        ("Status", listing["status"] or ""),
    ]
    rows = "".join(
        f"<tr><th style='text-align:left;padding:6px 10px;border:1px solid #ddd;'>{escape(label)}</th>"
        f"<td style='padding:6px 10px;border:1px solid #ddd;'>{escape(str(value))}</td></tr>"
        for label, value in specs if value
    )
    return f"""<!doctype html>
<html lang="en">
<head><meta charset="utf-8"><title>{escape(title)}</title></head>
<body>
<h1>{escape(title)}</h1>
{f'<p><strong>{escape(short_description)}</strong></p>' if short_description else ''}
<table style="border-collapse:collapse;margin:16px 0;">{rows}</table>
<h2>Description</h2>
{description_html or '<p>No description saved.</p>'}
</body>
</html>
"""


def listing_description_export_text(listing, lang="en"):
    title = get_translated(listing, "title", lang) or listing["title_en"] or listing["slug"]
    lines = [title, "=" * len(title), ""]
    specs = [
        ("Stock Reference", listing["public_stock_number"] or listing["stock_number"] or ""),
        ("Year", listing["year"] or ""),
        ("Make", listing["make"] or ""),
        ("Model", listing["model"] or ""),
        ("Category", listing["category"] or ""),
        ("Hours", listing["hours"] or ""),
        ("Price", listing["price_label"] or listing["price"] or ""),
        ("Status", listing["status"] or ""),
    ]
    for label, value in specs:
        if value:
            lines.append(f"{label}: {value}")
    short_description = get_translated(listing, "short_description", lang)
    if short_description:
        lines.extend(["", "Short Summary:", short_description])
    description_text = strip_html_to_text(get_translated(listing, "description_html", lang))
    if description_text:
        lines.extend(["", "Description:", description_text])
    return "\n".join(lines).strip() + "\n"


def current_gallery_image_rows(listing_id):
    return get_db().execute(
        """
        SELECT * FROM listing_images
        WHERE listing_id = ?
          AND COALESCE(is_public, 1) = 1
        ORDER BY sort_order, id
        """,
        (listing_id,),
    ).fetchall()


@app.route(f"{ADMIN_URL_PREFIX}/listings/<int:id>/download-gallery")
@login_required
def admin_listing_download_gallery(id):
    listing = get_db().execute("SELECT * FROM listings WHERE id = ?", (id,)).fetchone()
    if listing is None:
        abort(404)

    buffer = io.BytesIO()
    zip_slug = slugify(listing["slug"] or listing["title_en"] or listing["public_stock_number"] or f"listing-{id}") or f"listing-{id}"
    used_names = set()
    missing = []

    with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive:
        archive.writestr("description.html", listing_description_export_html(listing, "en"))
        archive.writestr("description.txt", listing_description_export_text(listing, "en"))

        for index, image in enumerate(current_gallery_image_rows(id), start=1):
            display_path = get_image_display_path(image)
            file_path = static_url_to_path(display_path)
            if not file_path:
                missing.append(display_path or image["local_path"] or image["generated_local_path"] or f"image-row-{image['id']}")
                continue
            original_name = secure_filename(file_path.name) or f"image-{index:03d}{file_path.suffix.lower() or '.jpg'}"
            archive_name = f"images/{index:03d}-{original_name}"
            counter = 2
            while archive_name in used_names:
                archive_name = f"images/{index:03d}-{Path(original_name).stem}-{counter}{Path(original_name).suffix}"
                counter += 1
            used_names.add(archive_name)
            archive.write(file_path, archive_name)

        manifest = [
            f"Listing: {listing['title_en'] or listing['slug']}",
            f"Slug: {listing['slug']}",
            f"Stock number: {listing['stock_number'] or ''}",
            f"Public reference: {listing['public_stock_number'] or ''}",
            f"Exported images: {len(used_names)}",
        ]
        if missing:
            manifest.extend(["", "Missing files skipped:", *missing])
        archive.writestr("manifest.txt", "\n".join(manifest).strip() + "\n")

    buffer.seek(0)
    return send_file(
        buffer,
        mimetype="application/zip",
        as_attachment=True,
        download_name=f"{zip_slug}.zip",
    )


@app.route(f"{ADMIN_URL_PREFIX}/listings/edit/<int:id>", methods=("GET", "POST"))
@login_required
def admin_listing_edit(id):
    listing = get_db().execute("SELECT * FROM listings WHERE id = ?", (id,)).fetchone()
    if listing is None:
        abort(404)
    if request.method == "POST":
        if request.form.get("image_action") == "upload_images":
            files = [item for item in request.files.getlist("gallery_images") if item and item.filename]
            if not files:
                flash("Choose at least one image to upload.", "warning")
                return redirect(url_for("admin_listing_edit", id=id))
            existing_count = get_db().execute(
                "SELECT COUNT(*) FROM listing_images WHERE listing_id = ?", (id,)
            ).fetchone()[0]
            next_order = get_db().execute(
                "SELECT COALESCE(MAX(sort_order), 0) + 1 FROM listing_images WHERE listing_id = ?", (id,)
            ).fetchone()[0]
            uploaded = 0
            warnings = []
            timestamp = now_utc()
            for file_storage in files:
                image_data, message = save_listing_gallery_upload(listing, file_storage, next_order)
                if not image_data:
                    warnings.append(message)
                    continue
                get_db().execute(
                    """
                    INSERT INTO listing_images (
                        listing_id, image_url, local_path, alt_text, sort_order, is_primary,
                        file_size, checksum_sha256, download_status, openai_file_id,
                        generated_local_path, use_generated_image, is_ai_generated, is_public, created_at
                    )
                    VALUES (?, '', ?, ?, ?, ?, ?, ?, 'manual-upload', '', '', 0, 0, 1, ?)
                    """,
                    (
                        id,
                        image_data["local_path"],
                        make_image_alt(
                            listing["title_en"],
                            listing["year"] or "",
                            listing["make"] or "",
                            listing["model"] or "",
                            listing["category"] or "",
                            next_order,
                        ),
                        next_order,
                        1 if existing_count == 0 and uploaded == 0 else 0,
                        image_data["file_size"],
                        image_data["checksum_sha256"],
                        timestamp,
                    ),
                )
                generate_image_derivatives_for_path(image_data["local_path"], regenerate=False)
                uploaded += 1
                next_order += 1
            if uploaded:
                sync_listing_featured_image_from_primary(id)
                get_db().commit()
                flash(f"Uploaded {uploaded} gallery image{'s' if uploaded != 1 else ''}.", "success")
            else:
                get_db().rollback()
                flash("No images were uploaded.", "warning")
            if warnings:
                flash("Some files were skipped: " + " | ".join(warnings[:4]), "warning")
            return redirect(url_for("admin_listing_edit", id=id))
        if request.form.get("image_action") == "reorder":
            ordered_ids = [int(value) for value in request.form.get("ordered_image_ids", "").split(",") if value.strip().isdigit()]
            if not ordered_ids:
                flash("Drag images into order before saving.", "warning")
            else:
                for sort_order, image_id in enumerate(ordered_ids, start=1):
                    get_db().execute(
                        "UPDATE listing_images SET sort_order = ? WHERE id = ? AND listing_id = ?",
                        (sort_order, image_id, id),
                    )
                sync_listing_featured_image_from_primary(id)
                get_db().commit()
                flash("Image order updated.", "success")
            return redirect(url_for("admin_listing_edit", id=id))
        if request.form.get("image_action") == "bulk_delete_images":
            selected_image_ids = [int(value) for value in request.form.getlist("selected_gallery_image_ids") if value.isdigit()]
            if not selected_image_ids:
                flash("Select at least one image to delete.", "warning")
            else:
                placeholders = ", ".join("?" for _ in selected_image_ids)
                get_db().execute(
                    f"DELETE FROM listing_images WHERE listing_id = ? AND id IN ({placeholders})",
                    (id, *selected_image_ids),
                )
                repair_image_order_for_listing(id)
                get_db().commit()
                flash(f"Deleted {len(selected_image_ids)} selected image row{'s' if len(selected_image_ids) != 1 else ''}.", "success")
            return redirect(url_for("admin_listing_edit", id=id))
        if request.form.get("image_action") == "sync_folder_images":
            summary = sync_missing_folder_images_for_listing(id)
            get_db().commit()
            flash(f"Synced {summary['added']} missing folder image row{'s' if summary['added'] != 1 else ''}. Repair order was applied after sync.", "success")
            return redirect(url_for("admin_listing_edit", id=id))
        if request.form.get("image_action") in ("generate_variants", "regenerate_variants"):
            regenerate = request.form.get("image_action") == "regenerate_variants"
            summary = generate_image_derivatives_for_listing(id, regenerate=regenerate)
            flash(
                f"{'Regenerated' if regenerate else 'Generated'} {summary['written']} thumb/gallery files for this listing.",
                "success",
            )
            if summary["errors"]:
                flash("Some images were skipped: " + " | ".join(summary["errors"][:4]), "warning")
            return redirect(url_for("admin_listing_edit", id=id))
        ai_action = request.form.get("ai_action", "")
        if ai_action:
            if ai_action == "generate":
                selected_ids = [int(value) for value in request.form.getlist("ai_image_ids") if value.isdigit()]
                instruction = request.form.get("ai_instruction", "").strip()
                if not selected_ids:
                    flash("Select at least one image for AI editing.", "warning")
                else:
                    created = 0
                    for image_id in selected_ids:
                        image = get_db().execute(
                            "SELECT * FROM listing_images WHERE id = ? AND listing_id = ?", (image_id, id)
                        ).fetchone()
                        if not image:
                            continue
                        try:
                            generate_ai_image_edit(listing, image, instruction)
                            created += 1
                        except Exception as error:
                            flash(f"Image {image_id}: {error}", "warning")
                    if created:
                        flash(f"Generated {created} AI edit preview{'s' if created != 1 else ''}.", "success")
                return redirect(url_for("admin_listing_edit", id=id))
            if ai_action == "generate_angles":
                reference_id = request.form.get("angle_reference_image_id", "")
                requested_angle = request.form.get("angle_name", "").strip()
                image = get_db().execute(
                    "SELECT * FROM listing_images WHERE id = ? AND listing_id = ?", (reference_id, id)
                ).fetchone() if reference_id.isdigit() else None
                if not image:
                    flash("Select one reference image for angle generation.", "warning")
                else:
                    angle_name = angle_label(requested_angle)
                    try:
                        generate_ai_image_edit(
                            listing,
                            image,
                            "",
                            edit_kind="angle",
                            angle_name=angle_name,
                        )
                        flash(f"Generated additional angle preview: {angle_name}.", "success")
                    except Exception as error:
                        flash(f"Angle generation failed: {error}", "warning")
                return redirect(url_for("admin_listing_edit", id=id))
            if ai_action == "edit_again":
                edit_id = request.form.get("edit_id", "")
                correction = request.form.get("ai_correction", "").strip()
                edit = get_db().execute(
                    "SELECT * FROM ai_image_edits WHERE id = ? AND listing_id = ? AND status = 'pending'",
                    (edit_id, id),
                ).fetchone()
                if not edit:
                    flash("AI edit preview was not found.", "warning")
                else:
                    image = get_db().execute(
                        "SELECT * FROM listing_images WHERE id = ? AND listing_id = ?",
                        (edit["source_image_id"], id),
                    ).fetchone()
                    try:
                        if not correction:
                            correction = "Revise the previous AI output so it follows the original admin instructions more closely while preserving the source vehicle and scene."
                        generate_ai_image_edit(listing, image, correction, source_edit=edit)
                        get_db().execute("UPDATE ai_image_edits SET status = 'superseded', updated_at = ? WHERE id = ?", (now_utc(), edit["id"]))
                        get_db().commit()
                        flash("Generated a revised AI edit preview.", "success")
                    except Exception as error:
                        flash(str(error), "warning")
                return redirect(url_for("admin_listing_edit", id=id))
            if ai_action in ("accept", "keep", "cancel"):
                edit_id = request.form.get("edit_id", "")
                edit = get_db().execute(
                    "SELECT * FROM ai_image_edits WHERE id = ? AND listing_id = ? AND status = 'pending'",
                    (edit_id, id),
                ).fetchone()
                if not edit:
                    flash("AI edit preview was not found.", "warning")
                    return redirect(url_for("admin_listing_edit", id=id))
                if ai_action == "accept":
                    image = get_db().execute(
                        "SELECT * FROM listing_images WHERE id = ? AND listing_id = ?",
                        (edit["source_image_id"], id),
                    ).fetchone()
                    if edit["edit_kind"] == "angle":
                        next_order = get_db().execute(
                            "SELECT COALESCE(MAX(sort_order), 0) + 1 FROM listing_images WHERE listing_id = ?",
                            (id,),
                        ).fetchone()[0]
                        get_db().execute(
                            """
                            INSERT INTO listing_images (
                                listing_id, image_url, local_path, alt_text, sort_order, is_primary,
                                download_status, generated_local_path, use_generated_image,
                                is_ai_generated, is_public, created_at
                            )
                            VALUES (?, '', ?, ?, ?, 0, 'ai-generated-angle', ?, 1, 1, 1, ?)
                            """,
                            (
                                id,
                                edit["result_path"],
                                f"{listing['title_en'] or listing['stock_number']} - AI generated angle",
                                next_order,
                                edit["result_path"],
                                now_utc(),
                            ),
                        )
                        generate_image_derivatives_for_path(edit["result_path"], regenerate=False)
                    else:
                        get_db().execute(
                            """
                            UPDATE listing_images
                            SET generated_local_path = ?, use_generated_image = 1,
                                openai_file_id = '', download_status = 'ai-edited'
                            WHERE id = ? AND listing_id = ?
                            """,
                            (edit["result_path"], edit["source_image_id"], id),
                        )
                        generate_image_derivatives_for_path(edit["result_path"], regenerate=False)
                        if image and image["is_primary"]:
                            get_db().execute(
                                "UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ?",
                                (edit["result_path"], now_utc(), id),
                            )
                    get_db().execute("UPDATE ai_image_edits SET status = 'accepted', updated_at = ? WHERE id = ?", (now_utc(), edit["id"]))
                    flash("AI edit accepted and set as the listing image.", "success")
                else:
                    new_status = "kept_original" if ai_action == "keep" else "cancelled"
                    get_db().execute("UPDATE ai_image_edits SET status = ?, updated_at = ? WHERE id = ?", (new_status, now_utc(), edit["id"]))
                    flash("Original image kept.", "success")
                get_db().commit()
                return redirect(url_for("admin_listing_edit", id=id))
        image_action = request.form.get("image_action", "")
        image_id = request.form.get("image_id", "")
        if image_action and image_id.isdigit():
            if image_action == "set_primary":
                get_db().execute("UPDATE listing_images SET is_primary = 0 WHERE listing_id = ?", (id,))
                get_db().execute("UPDATE listing_images SET is_primary = 1 WHERE id = ? AND listing_id = ?", (image_id, id))
                image = get_db().execute("SELECT * FROM listing_images WHERE id = ? AND listing_id = ?", (image_id, id)).fetchone()
                if image:
                    get_db().execute("UPDATE listings SET featured_image = ?, updated_at = ? WHERE id = ?", (get_image_display_path(image), now_utc(), id))
                flash("Primary image updated.", "success")
            elif image_action == "delete_image":
                get_db().execute("DELETE FROM listing_images WHERE id = ? AND listing_id = ?", (image_id, id))
                repair_image_order_for_listing(id)
                flash("Image row deleted.", "success")
            elif image_action == "use_generated":
                get_db().execute(
                    "UPDATE listing_images SET use_generated_image = 1 WHERE id = ? AND listing_id = ? AND generated_local_path IS NOT NULL AND generated_local_path != ''",
                    (image_id, id),
                )
                repair_image_order_for_listing(id)
                flash("Generated image version is now active.", "success")
            elif image_action == "use_original":
                get_db().execute("UPDATE listing_images SET use_generated_image = 0 WHERE id = ? AND listing_id = ?", (image_id, id))
                repair_image_order_for_listing(id)
                flash("Original image version is now active.", "success")
            elif image_action == "toggle_public":
                image = get_db().execute("SELECT is_public FROM listing_images WHERE id = ? AND listing_id = ?", (image_id, id)).fetchone()
                if image:
                    get_db().execute(
                        "UPDATE listing_images SET is_public = ? WHERE id = ? AND listing_id = ?",
                        (0 if image["is_public"] else 1, image_id, id),
                    )
                    flash("Public gallery visibility updated.", "success")
            get_db().commit()
            return redirect(url_for("admin_listing_edit", id=id))
        if request.form.get("repair_images"):
            repair_image_order_for_listing(id)
            get_db().commit()
            flash("Image order repaired.", "success")
            return redirect(url_for("admin_listing_edit", id=id))
        values = listing_form_values(listing)
        get_db().execute(
            f"""
            UPDATE listings
            SET {", ".join(f"{key} = ?" for key in values.keys())}, updated_at = ?
            WHERE id = ?
            """,
            (*values.values(), now_utc(), id),
        )
        if (listing["status"] or "") != (values.get("status") or ""):
            trust_audit("listing availability changed", "listing", id, {"status": listing["status"]}, {"status": values.get("status")})
        get_db().commit()
        flash("Listing updated.", "success")
        return redirect(url_for("admin_listing_edit", id=id))
    images = get_db().execute(
        "SELECT * FROM listing_images WHERE listing_id = ? ORDER BY sort_order, id", (id,)
    ).fetchall()
    inspection_report = inspection_reports.get_report_for_listing(get_db(), id)
    inspection_data = {}
    if inspection_report:
        try:
            inspection_data = json.loads(inspection_report["report_json"] or "{}")
        except Exception:
            inspection_data = {}
    return render_template(
        "admin/listing_form.html",
        listing=listing,
        statuses=listing_statuses(),
        images=images,
        inspection_report=inspection_report,
        inspection_data=inspection_data,
        pending_ai_edits=pending_ai_edits_for_listing(id),
        angle_choices=ai_angle_choices(),
        openai_key=openai_key_status(),
        ai_status=ai_provider_status(),
        trust_locations=get_db().execute("SELECT * FROM saved_locations WHERE is_active=1 ORDER BY internal_name").fetchall(),
        trust_location_types=("owned warehouse", "partner warehouse", "supplier location", "secure storage", "other"),
        trust_visibility_modes=("full", "city", "country", "custom", "hidden"),
        trust_inspection_statuses=("not checked", "inspection pending", "visually checked", "mechanically inspected", "third-party inspected", "documentation only"),
        trust_serial_statuses=("available publicly", "available on request", "recorded internally", "pending verification", "unavailable"),
        trust_condition_statuses=("not checked", "visually checked", "mechanically checked", "third-party checked", "documentation only", "pending"),
        trust_today=date.today().isoformat(),
    )


def listing_statuses():
    return ("Available", "Reserved", "Awaiting Confirmation", "Sold", "Unavailable", "Archived", "Coming Soon")


@app.route(f"{ADMIN_URL_PREFIX}/ai")
@login_required
def admin_ai():
    status = ai_provider_status()
    missing_count = get_db().execute(
        """
        SELECT COUNT(*) FROM listings
        WHERE COALESCE(is_deleted, 0) = 0
          AND (
              description_html_en IS NULL OR description_html_en = ''
              OR id NOT IN (
                  SELECT DISTINCT listing_id
                  FROM ai_generations
                  WHERE status = 'success'
              )
          )
        """
    ).fetchone()[0]
    generated_count = get_db().execute(
        "SELECT COUNT(DISTINCT listing_id) FROM ai_generations WHERE status = 'success'"
    ).fetchone()[0]
    generated_listing_ids = {
        row["listing_id"]
        for row in get_db().execute(
            "SELECT DISTINCT listing_id FROM ai_generations WHERE status = 'success'"
        ).fetchall()
    }
    recent_generations = get_db().execute(
        """
        SELECT ai_generations.*, listings.title_en, listings.public_stock_number, listings.stock_number
        FROM ai_generations
        LEFT JOIN listings ON listings.id = ai_generations.listing_id
        ORDER BY ai_generations.created_at DESC, ai_generations.id DESC
        LIMIT 12
        """
    ).fetchall()
    query, params, filters = build_listing_query(published_only=False)
    listings = get_db().execute(query, params).fetchall()
    return render_template(
        "admin/ai.html",
        ai_status=status,
        missing_count=missing_count,
        generated_count=generated_count,
        recent_generations=recent_generations,
        listings=listings,
        generated_listing_ids=generated_listing_ids,
        filters=filters,
        filter_options=listing_filter_options(published_only=False),
        fireworks_models=FIREWORKS_MODELS,
    )


@app.route(f"{ADMIN_URL_PREFIX}/listings/<int:id>/generate-ai", methods=("POST",))
@login_required
def admin_listing_generate_ai(id):
    mode = request.form.get("ai_content_mode", "full")
    allow_overwrite = bool(request.form.get("allow_ai_overwrite"))
    try:
        generate_listing_ai_content(id, allow_overwrite=allow_overwrite, mode=mode)
        flash("AI listing content generated.", "success")
    except Exception as error:
        settings = get_settings()
        status = ai_provider_status(settings)
        listing = get_db().execute("SELECT * FROM listings WHERE id = ?", (id,)).fetchone()
        prompt = ai_generation_prompt(listing) if listing else ""
        model = status["openai_model"] if status["provider"] == "openai" else status["fireworks_model"]
        record_ai_generation_error(id, status["provider"], model, prompt, error)
        flash(f"AI generation failed: {error}", "warning")
    return redirect(url_for("admin_listing_edit", id=id))


@app.route(f"{ADMIN_URL_PREFIX}/listings/<int:id>/generate-description", methods=("POST",))
@login_required
def admin_generate_description(id):
    """Generate a compact AI description for a listing using the two-pass engine."""
    try:
        settings = get_settings()
        fstatus, ai_adapter = make_text_adapter(settings, "descriptions")
        if not fstatus["active_configured"]:
            return jsonify({"ok": False, "error": f"{fstatus['provider'].title()} API key is missing."}), 400

        result = ai_descriptions.generate_vehicle_description(id, get_db(), ai_adapter)

        if result.get("status") == "error":
            return jsonify({"ok": False, "error": result.get("error", "Unknown error")}), 400

        # Return the generated HTML so the admin UI can populate the textarea
        return jsonify({
            "ok": True,
            "html": result.get("html", ""),
            "status": result.get("status"),
            "score": result.get("score") or result.get("best_score"),
            "attempts": result.get("attempts"),
        })

    except Exception as error:
        return jsonify({"ok": False, "error": str(error)}), 500


@app.route(f"{ADMIN_URL_PREFIX}/listings/<int:id>/generate-inspection", methods=("POST",))
@login_required
def admin_generate_inspection(id):
    """Generate (or regenerate) an AI mechanical inspection report for a listing.
    Regenerating an approved report resets its status to pending (shop must re-approve)."""
    try:
        settings = get_settings()
        fstatus, ai_adapter = make_text_adapter(settings, "inspection")
        if not fstatus["active_configured"]:
            return jsonify({"ok": False, "error": f"{fstatus['provider'].title()} API key is missing."}), 400

        prior = inspection_reports.get_report_for_listing(get_db(), id)
        was_approved = bool(prior and prior["status"] == "approved")

        result = inspection_reports.generate_inspection_report(id, get_db(), ai_adapter, carfax_dir=CARFAX_REPORTS_DIR)
        if result.get("status") == "error":
            return jsonify({"ok": False, "error": result.get("error", "Unknown error")}), 400

        provider = fstatus["provider"]
        model = settings_value(settings, "openai_model", "") if provider == "openai" else (settings_value(settings, "fireworks_model", "") or settings_value(settings, "fireworks_custom_model", ""))
        row = inspection_reports._save_report(
            get_db(), id, result["powertrain"], result["report_json"], provider, model
        )
        return jsonify({
            "ok": True,
            "powertrain": result["powertrain"],
            "status": row["status"],
            "report_token": row["report_token"],
            "was_approved": was_approved,
        })
    except Exception as error:
        return jsonify({"ok": False, "error": str(error)}), 500


@app.route(f"{ADMIN_URL_PREFIX}/listings/<int:id>/save-inspection", methods=("POST",))
@login_required
def admin_save_inspection(id):
    """Save admin edits to a report's sections (rating/findings/notes) + overall verdict."""
    try:
        row = inspection_reports.get_report_for_listing(get_db(), id)
        if not row:
            return jsonify({"ok": False, "error": "No inspection report exists for this listing."}), 404
        try:
            data = json.loads(row["report_json"] or "{}")
        except Exception:
            data = {}
        sections = []
        for section in data.get("sections", []):
            sid = section.get("id")
            if not sid:
                continue
            rating = (request.form.get(f"section_{sid}_rating") or "pass").strip()
            if rating not in inspection_reports.RATINGS:
                rating = "pass"
            findings_raw = request.form.get(f"section_{sid}_findings") or ""
            findings = [f.strip() for f in findings_raw.splitlines() if f.strip()]
            notes = (request.form.get(f"section_{sid}_notes") or "").strip()
            sections.append({"id": sid, "rating": rating, "findings": findings, "notes": notes})
        overall_rating = (request.form.get("overall_rating") or "pass").strip()
        if overall_rating not in inspection_reports.RATINGS:
            overall_rating = "pass"
        overall_summary = (request.form.get("overall_summary") or "").strip()
        inspection_reports.update_report_sections(
            get_db(), id, sections,
            overall_verdict={"rating": overall_rating, "summary": overall_summary},
        )
        return jsonify({"ok": True})
    except Exception as error:
        return jsonify({"ok": False, "error": str(error)}), 500


@app.route(f"{ADMIN_URL_PREFIX}/settings/regenerate-shop-token", methods=("POST",))
@login_required
def admin_regenerate_shop_token():
    """Regenerate the shop portal token (invalidates the old portal URL)."""
    get_db().execute(
        "UPDATE site_settings SET shop_portal_token = ?, updated_at = ? WHERE id = 1",
        (uuid.uuid4().hex, now_utc()),
    )
    get_db().commit()
    flash("Shop portal token regenerated. Provide the shop the new portal URL.", "success")
    return redirect(url_for("admin_settings") + "#shop-inspection")


@app.route(f"{ADMIN_URL_PREFIX}/listings/bulk-generate-ai", methods=("POST",))
@login_required
def admin_bulk_generate_ai():
    selected_ids = [int(value) for value in request.form.getlist("listing_ids") if value.isdigit()]
    limit = min(max(int(request.form.get("limit", "5") or 5), 1), 25)
    mode = request.form.get("ai_content_mode", "full")
    allow_overwrite = bool(request.form.get("allow_ai_overwrite"))
    missing_only = bool(request.form.get("missing_description_only"))
    imported_only = bool(request.form.get("imported_only"))
    params = []
    where = ["COALESCE(is_deleted, 0) = 0"]
    if selected_ids:
        where.append(f"id IN ({', '.join('?' for _ in selected_ids)})")
        params.extend(selected_ids)
        if missing_only:
            where.append(
                """
                (
                    description_html_en IS NULL OR description_html_en = ''
                    OR id NOT IN (
                        SELECT DISTINCT listing_id
                        FROM ai_generations
                        WHERE status = 'success'
                    )
                )
                """
            )
        if imported_only:
            where.append("(source_url IS NOT NULL AND source_url != '')")
        category = request.form.get("category", "").strip()
        if category:
            where.append("category = ?")
            params.append(category)
    else:
        if missing_only:
            where.append(
                """
                (
                    description_html_en IS NULL OR description_html_en = ''
                    OR id NOT IN (
                        SELECT DISTINCT listing_id
                        FROM ai_generations
                        WHERE status = 'success'
                    )
                )
                """
            )
        if imported_only:
            where.append("(source_url IS NOT NULL AND source_url != '')")
        category = request.form.get("category", "").strip()
        if category:
            where.append("category = ?")
            params.append(category)
    if selected_ids:
        rows = get_db().execute(
            f"SELECT id FROM listings WHERE {' AND '.join(where)}",
            params,
        ).fetchall()
        selected_order = {listing_id: index for index, listing_id in enumerate(selected_ids)}
        rows = sorted(rows, key=lambda row: selected_order.get(row["id"], len(selected_order)))[:limit]
    else:
        rows = get_db().execute(
            f"SELECT id FROM listings WHERE {' AND '.join(where)} ORDER BY updated_at DESC, id DESC LIMIT ?",
            (*params, limit),
        ).fetchall()
    ok = 0
    errors = []
    for row in rows:
        try:
            generate_listing_ai_content(row["id"], allow_overwrite=allow_overwrite, mode=mode)
            ok += 1
        except Exception as error:
            errors.append(f"Listing {row['id']}: {error}")
            settings = get_settings()
            status = ai_provider_status(settings)
            listing = get_db().execute("SELECT * FROM listings WHERE id = ?", (row["id"],)).fetchone()
            prompt = ai_generation_prompt(listing) if listing else ""
            model = status["openai_model"] if status["provider"] == "openai" else status["fireworks_model"]
            record_ai_generation_error(row["id"], status["provider"], model, prompt, error)
    if ok:
        flash(f"Generated AI content for {ok} listing{'s' if ok != 1 else ''}.", "success")
    if errors:
        flash("Some AI generations failed: " + " | ".join(errors[:3]), "warning")
    if not rows:
        flash("No listings matched the AI generation selection.", "warning")
    return redirect(url_for("admin_ai"))


def pricing_candidates_from_form():
    filters = {
        "category": request.values.get("category", "").strip(),
        "make": request.values.get("make", "").strip(),
        "status": request.values.get("status", "").strip(),
        "published": request.values.get("published", "").strip(),
        "price_min": request.values.get("price_min", "").strip(),
        "price_max": request.values.get("price_max", "").strip(),
    }
    selected_ids = [int(value) for value in request.values.getlist("listing_ids") if value.isdigit()]
    where = ["COALESCE(is_deleted, 0) = 0"]
    params = []
    if selected_ids:
        where.append(f"id IN ({', '.join('?' for _ in selected_ids)})")
        params.extend(selected_ids)
    for field in ("category", "make", "status"):
        if filters[field]:
            where.append(f"{field} = ?")
            params.append(filters[field])
    if filters["published"] in ("0", "1"):
        where.append("is_published = ?")
        params.append(int(filters["published"]))
    price_expr = price_sort_expression()
    if filters["price_min"]:
        where.append(f"{price_expr} >= ?")
        params.append(filters["price_min"])
    if filters["price_max"]:
        where.append(f"{price_expr} <= ?")
        params.append(filters["price_max"])
    rows = get_db().execute(
        f"SELECT * FROM listings WHERE {' AND '.join(where)} ORDER BY updated_at DESC, id DESC LIMIT 250",
        params,
    ).fetchall()
    return rows, filters, selected_ids


def price_update_preview(rows, percentage, direction, rounding):
    multiplier = 1 + ((percentage / 100.0) * (-1 if direction == "decrease" else 1))
    preview = []
    for row in rows:
        amount = parse_price_amount(row["price"])
        if amount is None:
            preview.append({"listing": row, "old_price": row["price"] or "POA", "new_price": "", "skip": "POA or unparsable"})
            continue
        new_amount = max(0, rounded_amount(amount * multiplier, rounding))
        preview.append(
            {
                "listing": row,
                "old_price": row["price"],
                "new_price": clean_price_text(row["price"], new_amount),
                "skip": "",
            }
        )
    return preview


@app.route(f"{ADMIN_URL_PREFIX}/pricing", methods=("GET", "POST"))
@login_required
def admin_pricing():
    rows, filters, selected_ids = pricing_candidates_from_form()
    percentage = float(request.values.get("percentage", "0") or 0)
    direction = request.values.get("direction", "increase")
    rounding = request.values.get("rounding", "none")
    preview = price_update_preview(rows, abs(percentage), direction, rounding) if request.values.get("percentage") else []
    if request.method == "POST" and request.form.get("apply") == "1":
        applied = 0
        for item in preview:
            if item["skip"]:
                continue
            listing = item["listing"]
            get_db().execute(
                "UPDATE listings SET price = ?, updated_at = ? WHERE id = ?",
                (item["new_price"], now_utc(), listing["id"]),
            )
            get_db().execute(
                """
                INSERT INTO price_history (listing_id, old_price, new_price, change_type, percentage, changed_by, created_at)
                VALUES (?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    listing["id"],
                    item["old_price"],
                    item["new_price"],
                    direction,
                    abs(percentage),
                    session.get("username", "admin"),
                    now_utc(),
                ),
            )
            applied += 1
        get_db().commit()
        flash(f"Applied price update to {applied} listing{'s' if applied != 1 else ''}.", "success")
        return redirect(url_for("admin_pricing"))
    history = get_db().execute(
        """
        SELECT price_history.*, listings.title_en, listings.public_stock_number, listings.stock_number
        FROM price_history
        LEFT JOIN listings ON listings.id = price_history.listing_id
        ORDER BY price_history.created_at DESC, price_history.id DESC
        LIMIT 20
        """
    ).fetchall()
    return render_template(
        "admin/pricing.html",
        rows=rows,
        filters=filters,
        selected_ids=selected_ids,
        preview=preview,
        history=history,
        percentage=percentage,
        direction=direction,
        rounding=rounding,
        filter_options=listing_filter_options(published_only=False),
    )


def analytics_show_full_ip():
    return bool(settings_value(get_settings(), "analytics_show_full_ip", 0))


def analytics_include_zero_seconds():
    return bool(settings_value(get_settings(), "analytics_include_zero_seconds", 1))


def analytics_ip_value(row):
    if analytics_show_full_ip() and "ip_full" in row.keys() and row["ip_full"]:
        return row["ip_full"]
    return row["ip_masked"] if "ip_masked" in row.keys() and row["ip_masked"] else (row["ip_hash"][:10] if row["ip_hash"] else "Unknown")


def analytics_source_exclusion_sql(session_alias="s"):
    # Filters sessions whose first referrer or landing page matches an admin-excluded internal/test source.
    # Matching is intentionally contains-based so a saved value like /admin/listings or canadaautosales.ca/admin/listings
    # excludes both absolute and relative forms.
    return f"""
        AND NOT EXISTS (
            SELECT 1 FROM analytics_source_exclusions ax
            WHERE COALESCE(ax.is_enabled,1)=1
              AND ax.source_value IS NOT NULL AND ax.source_value != ''
              AND (
                    LOWER(COALESCE({session_alias}.first_referrer,'')) LIKE '%' || LOWER(ax.source_value) || '%'
                 OR LOWER(COALESCE({session_alias}.landing_page,'')) LIKE '%' || LOWER(ax.source_value) || '%'
              )
        )
    """


def analytics_period_days(value):
    value = str(value or "30").strip().lower()
    if value == "all":
        return None, "all", "All time"
    days = 7 if value == "7" else 90 if value == "90" else 30
    return days, str(days), f"{days} days"


def analytics_since_sql(days):
    return "" if days is None else "AND {field} >= datetime('now', ?)"


def analytics_since_param(days):
    return [] if days is None else [f"-{days} days"]


def analytics_source_label_expr(column="first_referrer"):
    return f"""CASE WHEN {column} IS NULL OR {column}='' OR {column}='direct' THEN 'Direct'
        WHEN lower({column}) LIKE '%google.%' THEN 'Google' WHEN lower({column}) LIKE '%bing.%' THEN 'Bing'
        WHEN lower({column}) LIKE '%facebook.%' OR lower({column}) LIKE '%fb.%' THEN 'Facebook' ELSE substr({column},1,140) END"""


def fetch_analytics_sessions_for_metric(metric, days, include_zero):
    db = get_db()
    params = analytics_since_param(days)
    date_filter = "" if days is None else "AND s.started_at >= datetime('now', ?)"
    zero_filter = "" if include_zero else "AND COALESCE(s.engaged_seconds,0)>0"
    extra = ""
    if metric == "human_today":
        date_filter = "AND date(s.started_at)=date('now')"
        params = []
    elif metric == "visitors":
        pass
    elif metric == "sessions":
        pass
    elif metric == "zero_seconds":
        extra = "AND COALESCE(s.engaged_seconds,0)=0"
        zero_filter = ""
    else:
        pass
    return db.execute(f"""
        SELECT s.* FROM analytics_sessions s
        WHERE s.is_bot=0 {date_filter} {zero_filter} {extra} {analytics_source_exclusion_sql('s')}
        ORDER BY s.started_at DESC LIMIT 1000
    """, params).fetchall()


def analytics_table_rows(table, where_sql="", params=()):
    db = get_db()
    return [dict(row) for row in db.execute(f"SELECT * FROM {table} {where_sql}", params).fetchall()]


@app.route(f"{ADMIN_URL_PREFIX}/analytics", methods=("GET", "POST"))
@login_required
def admin_analytics():
    db = get_db()
    if request.method == "POST":
        action = request.form.get("analytics_action", "preferences")
        if action == "preferences":
            db.execute(
                "UPDATE site_settings SET analytics_show_full_ip=?, analytics_include_zero_seconds=?, updated_at=? WHERE id=1",
                (1 if request.form.get("analytics_show_full_ip") else 0,
                 1 if request.form.get("analytics_include_zero_seconds") else 0,
                 now_utc()),
            )
            db.commit()
            flash("Analytics display preferences updated.", "success")
        elif action == "exclude_sources":
            selected = [v.strip() for v in request.form.getlist("source_values") if v.strip()]
            saved = 0
            for value in selected:
                db.execute("""
                    INSERT INTO analytics_source_exclusions(source_value, reason, is_enabled, created_at, updated_at)
                    VALUES(?, 'Excluded from Traffic sources panel', 1, ?, ?)
                    ON CONFLICT(source_value) DO UPDATE SET is_enabled=1, updated_at=excluded.updated_at
                """, (value, now_utc(), now_utc()))
                saved += 1
            db.commit()
            flash(f"Excluded {saved} traffic source{'s' if saved != 1 else ''} from analytics counts.", "success" if saved else "warning")
        elif action in ("enable_source_filters", "disable_source_filters", "delete_source_filters"):
            ids = [int(v) for v in request.form.getlist("source_filter_ids") if v.isdigit()]
            if ids:
                ph = ",".join("?" for _ in ids)
                if action == "delete_source_filters":
                    db.execute(f"DELETE FROM analytics_source_exclusions WHERE id IN ({ph})", ids)
                else:
                    db.execute(f"UPDATE analytics_source_exclusions SET is_enabled=?, updated_at=? WHERE id IN ({ph})", (1 if action == "enable_source_filters" else 0, now_utc(), *ids))
                db.commit()
                flash("Selected traffic-source filters updated.", "success")
        return redirect(url_for("admin_analytics", period=request.form.get("period", "30")))

    days, period, period_label = analytics_period_days(request.args.get("period", "30"))
    settings = get_settings()
    reliable_since = settings_value(settings, "analytics_reliable_since", "")
    include_zero = bool(settings_value(settings, "analytics_include_zero_seconds", 1))
    show_full_ip = bool(settings_value(settings, "analytics_show_full_ip", 0))
    zero_filter = "" if include_zero else "AND COALESCE(s.engaged_seconds,0) > 0"
    session_date = "" if days is None else "AND s.started_at >= datetime('now', ?)"
    event_date = "" if days is None else "AND e.created_at >= datetime('now', ?)"
    lead_date = "" if days is None else "WHERE created_at >= datetime('now', ?)"
    session_params = analytics_since_param(days)
    event_params = analytics_since_param(days)
    lead_params = analytics_since_param(days)
    excl = analytics_source_exclusion_sql('s')

    humans_today = db.execute(f"""SELECT COUNT(DISTINCT session_token) FROM analytics_sessions s
        WHERE is_bot=0 AND date(started_at)=date('now') {analytics_source_exclusion_sql('s')}""").fetchone()[0]
    sessions_count = db.execute(f"SELECT COUNT(*) FROM analytics_sessions s WHERE is_bot=0 {session_date} {zero_filter} {excl}", session_params).fetchone()[0]
    unique_visitors = db.execute(f"SELECT COUNT(DISTINCT visitor_id) FROM analytics_sessions s WHERE is_bot=0 {session_date} {zero_filter} {excl}", session_params).fetchone()[0]
    page_views = db.execute(f"""SELECT COUNT(*) FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token
        WHERE e.is_bot=0 AND e.event_type='page_view' {event_date} {zero_filter} {excl}""", event_params).fetchone()[0]
    zero_second_views = db.execute(f"""SELECT COUNT(*) FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token
        WHERE e.is_bot=0 AND e.event_type='page_view' {event_date} AND COALESCE(s.engaged_seconds,0)=0 {excl}""", event_params).fetchone()[0]
    leads_count = db.execute(f"SELECT COUNT(*) FROM leads {lead_date}", lead_params).fetchone()[0]
    conversion = round((leads_count / unique_visitors * 100), 2) if unique_visitors else 0
    bounced = db.execute(f"SELECT COUNT(*) FROM analytics_sessions s WHERE is_bot=0 {session_date} AND pageview_count<=1 AND engaged_seconds<15 {zero_filter} {excl}", session_params).fetchone()[0]
    bounce_rate = round((bounced / sessions_count * 100), 1) if sessions_count else 0
    returning = db.execute(f"SELECT COUNT(DISTINCT visitor_id) FROM analytics_sessions s WHERE is_bot=0 {session_date} AND visit_number>1 {zero_filter} {excl}", session_params).fetchone()[0]
    return_rate = round((returning / unique_visitors * 100), 1) if unique_visitors else 0
    avg_engaged = db.execute(f"SELECT COALESCE(AVG(engaged_seconds),0) FROM analytics_sessions s WHERE is_bot=0 {session_date} {zero_filter} {excl}", session_params).fetchone()[0]
    bot_sessions = db.execute(f"SELECT COUNT(*) FROM analytics_sessions s WHERE is_bot=1 {session_date}", session_params).fetchone()[0]
    all_sessions = sessions_count + bot_sessions
    metrics = {"humans_today":humans_today,"sessions":sessions_count,"visitors":unique_visitors,"page_views":page_views,
        "zero_second_views":zero_second_views,"leads":leads_count,"conversion":conversion,"bounce_rate":bounce_rate,
        "return_rate":return_rate,"avg_engaged":round(avg_engaged or 0),"bot_rate":round((bot_sessions/all_sessions*100),1) if all_sessions else 0}
    countries = db.execute(f"""SELECT COALESCE(NULLIF(country_code,''),'Unknown') country, COUNT(*) count,
        SUM(CASE WHEN lead_count>0 THEN 1 ELSE 0 END) converted FROM analytics_sessions s
        WHERE is_bot=0 {session_date} {zero_filter} {excl}
        GROUP BY country ORDER BY count DESC LIMIT 20""", session_params).fetchall()
    devices = db.execute(f"SELECT COALESCE(NULLIF(device_type,''),'Unknown') device_type, COUNT(*) count FROM analytics_sessions s WHERE is_bot=0 {session_date} {zero_filter} {excl} GROUP BY device_type ORDER BY count DESC", session_params).fetchall()
    engagement_having = "" if include_zero else "HAVING engaged > 0"
    top_pages = db.execute(f"""WITH page_sessions AS (
        SELECT e.path,e.session_token,e.visitor_id,COUNT(e.id) views,
          COALESCE((SELECT SUM(ee.engaged_seconds) FROM analytics_events ee WHERE ee.session_token=e.session_token AND ee.path=e.path AND ee.event_type='engagement'),0) engaged
        FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token
        WHERE e.is_bot=0 AND e.event_type='page_view' {event_date} {excl}
        GROUP BY e.path,e.session_token)
        SELECT path,SUM(views) count,COUNT(DISTINCT session_token) visitors,SUM(engaged) engaged
        FROM page_sessions GROUP BY path {engagement_having} ORDER BY count DESC LIMIT 15""", event_params).fetchall()
    top_listings = db.execute(f"""WITH listing_sessions AS (
        SELECT e.listing_id,e.session_token,e.visitor_id,COUNT(CASE WHEN e.event_type='page_view' THEN 1 END) views,
          COALESCE(SUM(CASE WHEN e.event_type='engagement' THEN e.engaged_seconds ELSE 0 END),0) engaged
        FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token
        WHERE e.is_bot=0 AND e.listing_id IS NOT NULL {event_date} {excl}
        GROUP BY e.listing_id,e.session_token)
        SELECT l.id,l.title_en,l.public_stock_number,l.stock_number,COUNT(DISTINCT ls.session_token) visitors,SUM(ls.views) views,SUM(ls.engaged) engaged,
          (SELECT COUNT(*) FROM leads ld WHERE ld.listing_id=l.id {'' if days is None else "AND ld.created_at >= datetime('now', ?)"}) leads,
          (SELECT COUNT(*) FROM analytics_events ce JOIN analytics_sessions cs ON cs.session_token=ce.session_token WHERE ce.listing_id=l.id {'' if days is None else "AND ce.created_at >= datetime('now', ?)"} AND ce.is_bot=0 AND ce.event_type IN ('whatsapp_click','phone_click','email_click','cta_click') {analytics_source_exclusion_sql('cs')}) cta_clicks
        FROM listing_sessions ls JOIN listings l ON l.id=ls.listing_id
        {"WHERE ls.engaged>0" if not include_zero else ""}
        GROUP BY l.id ORDER BY leads DESC,visitors DESC LIMIT 20""", event_params + analytics_since_param(days) + analytics_since_param(days)).fetchall()
    source_expr = analytics_source_label_expr('s.first_referrer')
    sources = db.execute(f"""SELECT {source_expr} source,
        COALESCE(NULLIF(s.first_referrer,''),'direct') source_value,
        COUNT(*) count,SUM(CASE WHEN lead_count>0 THEN 1 ELSE 0 END) leads FROM analytics_sessions s
        WHERE is_bot=0 {session_date} {zero_filter} {excl} GROUP BY source, source_value ORDER BY count DESC LIMIT 20""", session_params).fetchall()
    funnel_rows = db.execute(f"""SELECT event_type,COUNT(DISTINCT e.session_token) count FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token WHERE e.is_bot=0 {event_date}
        {excl} AND event_type IN ('page_view','cta_click','form_start','chatbot_open','chatbot_message','lead_submit','whatsapp_click','phone_click','email_click') GROUP BY event_type""", event_params).fetchall()
    funnel={row['event_type']:row['count'] for row in funnel_rows}
    latest_leads=db.execute("""SELECT leads.*,listings.title_en,listings.public_stock_number,listings.stock_number FROM leads LEFT JOIN listings ON listings.id=leads.listing_id ORDER BY leads.created_at DESC,leads.id DESC LIMIT 10""").fetchall()
    excluded_sources=db.execute("SELECT * FROM analytics_source_exclusions ORDER BY is_enabled DESC, updated_at DESC, id DESC").fetchall()
    return render_template("admin/analytics.html",metrics=metrics,countries=countries,devices=devices,top_pages=top_pages,
        top_listings=top_listings,sources=sources,funnel=funnel,latest_leads=latest_leads,period=period,period_label=period_label,reliable_since=reliable_since,
        include_zero=include_zero,show_full_ip=show_full_ip,excluded_sources=excluded_sources)

@app.route(f"{ADMIN_URL_PREFIX}/analytics/export")
@login_required
def admin_analytics_export():
    days, period, period_label = analytics_period_days(request.args.get("period", "all"))
    event_where = "" if days is None else "WHERE created_at >= datetime('now', ?)"
    session_where = "" if days is None else "WHERE started_at >= datetime('now', ?)"
    lead_where = "" if days is None else "WHERE created_at >= datetime('now', ?)"
    event_params = analytics_since_param(days)
    payload = {
        "exported_at": now_utc(),
        "period": period,
        "period_label": period_label,
        "tables": {
            "analytics_sessions": analytics_table_rows("analytics_sessions", session_where, event_params),
            "analytics_events": analytics_table_rows("analytics_events", event_where, event_params),
            "security_events": analytics_table_rows("security_events", event_where, event_params),
            "analytics_source_exclusions": analytics_table_rows("analytics_source_exclusions"),
            "security_rules": analytics_table_rows("security_rules"),
            "leads": analytics_table_rows("leads", lead_where, event_params),
        },
    }
    raw = json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8")
    label = "all" if days is None else f"{days}d"
    return send_file(io.BytesIO(raw), mimetype="application/json", as_attachment=True, download_name=f"cleanagrisales_analytics_export_{label}_{date.today().isoformat()}.json")

@app.route(f"{ADMIN_URL_PREFIX}/analytics/import", methods=("POST",))
@login_required
def admin_analytics_import():
    upload = request.files.get("analytics_file")
    if not upload or not upload.filename:
        flash("Choose an analytics JSON export file first.", "warning")
        return redirect(url_for("admin_analytics"))
    try:
        payload = json.loads(upload.read().decode("utf-8"))
    except Exception as error:
        flash(f"Could not read analytics import: {error}", "warning")
        return redirect(url_for("admin_analytics"))
    tables = payload.get("tables", {}) if isinstance(payload, dict) else {}
    db = get_db()
    imported = 0
    for table in ("analytics_sessions", "analytics_events", "security_events", "analytics_source_exclusions", "security_rules", "leads"):
        rows = tables.get(table, [])
        if not isinstance(rows, list) or not rows:
            continue
        existing_cols = [r[1] for r in db.execute(f"PRAGMA table_info({table})").fetchall()]
        for row in rows:
            if not isinstance(row, dict):
                continue
            cols = [c for c in existing_cols if c in row]
            if not cols:
                continue
            placeholders = ",".join("?" for _ in cols)
            db.execute(f"INSERT OR IGNORE INTO {table} ({','.join(cols)}) VALUES ({placeholders})", [row.get(c) for c in cols])
            imported += 1
    db.commit()
    flash(f"Analytics import completed. Processed {imported} row{'s' if imported != 1 else ''}.", "success")
    return redirect(url_for("admin_analytics"))

@app.route(f"{ADMIN_URL_PREFIX}/analytics/metric/<metric>")
@login_required
def admin_analytics_metric_detail(metric):
    db = get_db()
    days, period, period_label = analytics_period_days(request.args.get("period", "30"))
    include_zero = analytics_include_zero_seconds()
    metric_labels = {
        "human_today": "Human visitors today",
        "visitors": "Unique visitors",
        "sessions": "Human sessions",
        "page_views": "Human page views",
        "zero_seconds": "0-second views",
        "leads": "Leads",
    }
    if metric not in metric_labels:
        abort(404)
    sessions=[]; events=[]; leads=[]
    if metric in ("human_today", "visitors", "sessions", "zero_seconds"):
        sessions = fetch_analytics_sessions_for_metric(metric, days, include_zero)
    elif metric == "page_views":
        event_date = "" if days is None else "AND e.created_at >= datetime('now', ?)"
        events = db.execute(f"""SELECT e.*,s.started_at,s.first_referrer,s.landing_page,s.engaged_seconds session_engaged,s.ip_full,s.ip_masked,s.ip_hash,s.country_code,s.device_type
            FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token
            WHERE e.is_bot=0 AND e.event_type='page_view' {event_date} {analytics_source_exclusion_sql('s')}
            ORDER BY e.created_at DESC LIMIT 1000""", analytics_since_param(days)).fetchall()
    elif metric == "leads":
        where = "" if days is None else "WHERE leads.created_at >= datetime('now', ?)"
        leads = db.execute(f"""SELECT leads.*,listings.public_stock_number,listings.stock_number,listings.title_en
            FROM leads LEFT JOIN listings ON listings.id=leads.listing_id {where}
            ORDER BY leads.created_at DESC LIMIT 1000""", analytics_since_param(days)).fetchall()
    return render_template("admin/analytics_metric_detail.html",metric=metric,title=metric_labels[metric],period=period,period_label=period_label,sessions=sessions,events=events,leads=leads,ip_value=analytics_ip_value)

@app.route(f"{ADMIN_URL_PREFIX}/analytics/listing/<int:listing_id>")
@login_required
def admin_analytics_listing_detail(listing_id):
    db=get_db(); listing=db.execute("SELECT * FROM listings WHERE id=?",(listing_id,)).fetchone()
    if not listing: abort(404)
    days, period, period_label = analytics_period_days(request.args.get("period","30"))
    event_date = "" if days is None else "AND e.created_at>=datetime('now',?)"
    include_zero=analytics_include_zero_seconds(); having="" if include_zero else "HAVING listing_engaged>0"
    sessions=db.execute(f"""SELECT s.*,COUNT(CASE WHEN e.event_type='page_view' THEN 1 END) listing_views,
      COALESCE(SUM(CASE WHEN e.event_type='engagement' THEN e.engaged_seconds ELSE 0 END),0) listing_engaged,
      MAX(CASE WHEN e.event_type='scroll' THEN e.scroll_depth ELSE 0 END) listing_scroll,
      SUM(CASE WHEN e.event_type IN ('whatsapp_click','phone_click','email_click','cta_click') THEN 1 ELSE 0 END) listing_cta,
      SUM(CASE WHEN e.event_type='lead_submit' THEN 1 ELSE 0 END) listing_leads
      FROM analytics_sessions s JOIN analytics_events e ON e.session_token=s.session_token AND e.listing_id=?
      WHERE s.is_bot=0 {event_date} {analytics_source_exclusion_sql('s')} GROUP BY s.session_token {having} ORDER BY MAX(e.created_at) DESC""",(listing_id,*analytics_since_param(days))).fetchall()
    summary=db.execute(f"""SELECT COUNT(DISTINCT e.session_token) sessions,COUNT(DISTINCT e.visitor_id) visitors,
      COUNT(CASE WHEN e.event_type='page_view' THEN 1 END) views,COALESCE(SUM(CASE WHEN e.event_type='engagement' THEN e.engaged_seconds ELSE 0 END),0) engaged,
      SUM(CASE WHEN e.event_type IN ('whatsapp_click','phone_click','email_click','cta_click') THEN 1 ELSE 0 END) cta,SUM(CASE WHEN e.event_type='lead_submit' THEN 1 ELSE 0 END) leads
      FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token WHERE e.is_bot=0 AND e.listing_id=? {event_date} {analytics_source_exclusion_sql('s')}""",(listing_id,*analytics_since_param(days))).fetchone()
    return render_template("admin/analytics_listing_detail.html",listing=listing,sessions=sessions,summary=summary,period=period,include_zero=include_zero,ip_value=analytics_ip_value)

@app.route(f"{ADMIN_URL_PREFIX}/analytics/page")
@login_required
def admin_analytics_page_detail():
    db=get_db(); path_value=request.args.get("path","").strip()
    if not path_value: abort(404)
    days, period, period_label = analytics_period_days(request.args.get("period","30"))
    event_date = "" if days is None else "AND e.created_at>=datetime('now',?)"
    include_zero=analytics_include_zero_seconds(); having="" if include_zero else "HAVING page_engaged>0"
    sessions=db.execute(f"""SELECT s.*,COUNT(CASE WHEN e.event_type='page_view' THEN 1 END) views,
      COALESCE(SUM(CASE WHEN e.event_type='engagement' THEN e.engaged_seconds ELSE 0 END),0) page_engaged,
      MAX(CASE WHEN e.event_type='scroll' THEN e.scroll_depth ELSE 0 END) page_scroll
      FROM analytics_sessions s JOIN analytics_events e ON e.session_token=s.session_token
      WHERE s.is_bot=0 AND e.path=? {event_date} {analytics_source_exclusion_sql('s')} GROUP BY s.session_token {having} ORDER BY MAX(e.created_at) DESC""",(path_value,*analytics_since_param(days))).fetchall()
    return render_template("admin/analytics_page_detail.html",path_value=path_value,sessions=sessions,period=period,include_zero=include_zero,ip_value=analytics_ip_value)

@app.route(f"{ADMIN_URL_PREFIX}/analytics/session/<session_token>")
@login_required
def admin_analytics_session_detail(session_token):
    db=get_db(); session_row=db.execute("SELECT * FROM analytics_sessions WHERE session_token=?",(session_token,)).fetchone()
    if not session_row: abort(404)
    events=db.execute("""SELECT e.*,l.public_stock_number,l.stock_number,l.title_en FROM analytics_events e LEFT JOIN listings l ON l.id=e.listing_id WHERE e.session_token=? ORDER BY e.created_at,e.id""",(session_token,)).fetchall()
    leads=db.execute("""SELECT leads.*,listings.public_stock_number,listings.stock_number,listings.title_en FROM leads LEFT JOIN listings ON listings.id=leads.listing_id WHERE leads.analytics_session_token=? ORDER BY leads.created_at""",(session_token,)).fetchall()
    return render_template("admin/analytics_session_detail.html",session_row=session_row,events=events,leads=leads,ip_value=analytics_ip_value(session_row))

@app.route(f"{ADMIN_URL_PREFIX}/analytics/security", methods=("GET", "POST"))
@login_required
def admin_analytics_security():
    db=get_db()
    if request.method=="POST":
        action=request.form.get("security_action","")
        if action=="add_rule":
            rule_type=request.form.get("rule_type","").strip(); raw_values=request.form.get("rule_value",""); reason=request.form.get("reason","").strip(); expires=request.form.get("expires_at","").strip()
            values=[v.strip().lower() for v in re.split(r"[\n,;]+",raw_values) if v.strip()]
            saved=0; invalid=[]
            if rule_type in ("blocked_ip","blocked_domain","approved_host"):
                for value in values:
                    if rule_type=="blocked_ip":
                        try: ipaddress.ip_network(value,strict=False)
                        except ValueError: invalid.append(value); continue
                    else:
                        value=_normalized_request_host(value)
                        if not value: invalid.append(value); continue
                    db.execute("""INSERT INTO security_rules(rule_type,rule_value,reason,is_enabled,expires_at,created_at,updated_at)
                      VALUES(?,?,?,?,?,?,?) ON CONFLICT(rule_type,rule_value) DO UPDATE SET reason=excluded.reason,is_enabled=1,expires_at=excluded.expires_at,updated_at=excluded.updated_at""",
                      (rule_type,value,reason,1,expires,now_utc(),now_utc()))
                    saved+=1
                db.commit()
                if saved: flash(f"Saved {saved} security rule{'s' if saved != 1 else ''}.","success")
                if invalid: flash("Invalid values skipped: "+", ".join(invalid[:8]),"warning")
        elif action in ("enable_rules","disable_rules","delete_rules"):
            ids=[int(v) for v in request.form.getlist("rule_ids") if v.isdigit()]
            if ids:
                ph=','.join('?' for _ in ids)
                if action=="delete_rules": db.execute(f"DELETE FROM security_rules WHERE id IN ({ph})",ids)
                else: db.execute(f"UPDATE security_rules SET is_enabled=?,updated_at=? WHERE id IN ({ph})",(1 if action=="enable_rules" else 0,now_utc(),*ids))
                db.commit(); flash("Selected rules updated.","success")
        elif action in ("delete_events","block_event_ips","block_event_domains"):
            ids=[int(v) for v in request.form.getlist("event_ids") if v.isdigit()]
            if ids:
                ph=','.join('?' for _ in ids); rows=db.execute(f"SELECT * FROM security_events WHERE id IN ({ph})",ids).fetchall()
                if action=="delete_events": db.execute(f"DELETE FROM security_events WHERE id IN ({ph})",ids)
                elif action=="block_event_ips":
                    for r in rows:
                        value=r['source_ip_full'] if 'source_ip_full' in r.keys() else ''
                        if value: db.execute("INSERT OR IGNORE INTO security_rules(rule_type,rule_value,reason,is_enabled,created_at,updated_at) VALUES('blocked_ip',?,'Added from security event',1,?,?)",(value,now_utc(),now_utc()))
                else:
                    for r in rows:
                        for raw in (r['origin'],r['referrer'],r['forwarded_host']):
                            host=_host_from_url(raw) or _normalized_request_host(raw)
                            if host: db.execute("INSERT OR IGNORE INTO security_rules(rule_type,rule_value,reason,is_enabled,created_at,updated_at) VALUES('blocked_domain',?,'Added from security event',1,?,?)",(host,now_utc(),now_utc()))
                db.commit(); flash("Security action completed.","success")
        return redirect(url_for("admin_analytics_security"))
    rows=db.execute("SELECT * FROM security_events ORDER BY created_at DESC,id DESC LIMIT 500").fetchall()
    rules=db.execute("SELECT * FROM security_rules ORDER BY rule_type,rule_value").fetchall()
    return render_template("admin/analytics_security.html",rows=rows,rules=rules,show_full_ip=analytics_show_full_ip())


@app.route(f"{ADMIN_URL_PREFIX}/analytics/country/<country_code>")
@login_required
def admin_analytics_country_detail(country_code):
    db=get_db(); code='Unknown' if (country_code or '').lower()=='unknown' else (country_code or '').upper()
    days, period, period_label = analytics_period_days(request.args.get('period','30'))
    session_date = "" if days is None else "AND s.started_at>=datetime('now',?)"
    event_date = "" if days is None else "AND e.created_at>=datetime('now',?)"
    include_zero=analytics_include_zero_seconds(); z="" if include_zero else "AND COALESCE(s.engaged_seconds,0)>0"
    sparams=[code,*analytics_since_param(days)]
    sessions=db.execute(f"SELECT * FROM analytics_sessions s WHERE is_bot=0 AND COALESCE(NULLIF(country_code,''),'Unknown')=? {session_date} {z} {analytics_source_exclusion_sql('s')} ORDER BY started_at DESC LIMIT 500",sparams).fetchall()
    summary=db.execute(f"SELECT COUNT(*) sessions,COUNT(DISTINCT visitor_id) visitors,SUM(CASE WHEN lead_count>0 THEN 1 ELSE 0 END) converted,COALESCE(AVG(engaged_seconds),0) avg_engaged FROM analytics_sessions s WHERE is_bot=0 AND COALESCE(NULLIF(country_code,''),'Unknown')=? {session_date} {z} {analytics_source_exclusion_sql('s')}",sparams).fetchone()
    pages=db.execute(f"""SELECT e.path,COUNT(*) views,COUNT(DISTINCT e.session_token) visitors FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token WHERE e.is_bot=0 AND e.event_type='page_view' AND COALESCE(NULLIF(s.country_code,''),'Unknown')=? {event_date} {analytics_source_exclusion_sql('s')} GROUP BY e.path ORDER BY views DESC LIMIT 20""",[code,*analytics_since_param(days)]).fetchall()
    listings=db.execute(f"""SELECT l.id,l.public_stock_number,l.stock_number,l.title_en,COUNT(*) views,COUNT(DISTINCT e.session_token) visitors FROM analytics_events e JOIN analytics_sessions s ON s.session_token=e.session_token JOIN listings l ON l.id=e.listing_id WHERE e.is_bot=0 AND e.event_type='page_view' AND COALESCE(NULLIF(s.country_code,''),'Unknown')=? {event_date} {analytics_source_exclusion_sql('s')} GROUP BY l.id ORDER BY views DESC LIMIT 20""",[code,*analytics_since_param(days)]).fetchall()
    sources=db.execute(f"""SELECT {analytics_source_label_expr('s.first_referrer')} source,COUNT(*) count FROM analytics_sessions s WHERE is_bot=0 AND COALESCE(NULLIF(country_code,''),'Unknown')=? {session_date} {analytics_source_exclusion_sql('s')} GROUP BY source ORDER BY count DESC LIMIT 15""",sparams[:1]+analytics_since_param(days)).fetchall()
    devices=db.execute(f"SELECT COALESCE(NULLIF(device_type,''),'Unknown') device_type,COUNT(*) count FROM analytics_sessions s WHERE is_bot=0 AND COALESCE(NULLIF(country_code,''),'Unknown')=? {session_date} {analytics_source_exclusion_sql('s')} GROUP BY device_type ORDER BY count DESC",sparams[:1]+analytics_since_param(days)).fetchall()
    return render_template('admin/analytics_country_detail.html',country_code=code,period=period,summary=summary,sessions=sessions,pages=pages,listings=listings,sources=sources,devices=devices,ip_value=analytics_ip_value)

def qa_report():
    db = get_db()
    missing_translations = {}
    outdated_translations = {}
    for lang in LANGUAGES:
        if lang == "en":
            continue
        missing_translations[lang] = db.execute(
            f"SELECT COUNT(*) FROM site_pages WHERE is_published = 1 AND (title_{lang} IS NULL OR title_{lang} = '' OR translation_status_{lang} = 'missing')"
        ).fetchone()[0]
        outdated_translations[lang] = db.execute(
            f"SELECT COUNT(*) FROM site_pages WHERE translation_status_{lang} = 'outdated'"
        ).fetchone()[0]
    missing_meta = db.execute(
        "SELECT COUNT(*) FROM site_pages WHERE is_published = 1 AND (meta_description_en IS NULL OR meta_description_en = '')"
    ).fetchone()[0]
    missing_hero = db.execute(
        "SELECT COUNT(*) FROM site_pages WHERE is_published = 1 AND (hero_image_path IS NULL OR hero_image_path = '')"
    ).fetchone()[0]
    broken_images = 0
    for query in (
        "SELECT hero_image_path AS path FROM site_pages WHERE hero_image_path IS NOT NULL AND hero_image_path != ''",
        "SELECT local_path AS path FROM listing_images WHERE local_path IS NOT NULL AND local_path != ''",
        "SELECT generated_local_path AS path FROM listing_images WHERE generated_local_path IS NOT NULL AND generated_local_path != ''",
        "SELECT local_path AS path FROM generated_visuals WHERE local_path IS NOT NULL AND local_path != ''",
    ):
        for row in db.execute(query).fetchall():
            if row["path"].startswith("/static/") and not static_url_to_path(row["path"]):
                broken_images += 1
    hreflang_errors = 0
    page_slugs = db.execute("SELECT slug FROM site_pages WHERE is_published = 1 AND slug != 'home' ORDER BY id LIMIT 20").fetchall()
    listing_slugs = db.execute(
        "SELECT slug FROM listings WHERE is_published = 1 AND COALESCE(is_deleted,0)=0 ORDER BY id LIMIT 20"
    ).fetchall()
    for lang in LANGUAGES:
        for endpoint, values in (
            ("home", {"lang": lang}),
            ("inventory", {"lang": lang}),
        ):
            try:
                url_for(endpoint, **values)
            except Exception:
                hreflang_errors += 1
        for page in page_slugs:
            try:
                url_for("public_page", lang=lang, page_slug=page["slug"])
            except Exception:
                hreflang_errors += 1
        for listing in listing_slugs:
            try:
                url_for("listing_detail", lang=lang, slug=listing["slug"])
            except Exception:
                hreflang_errors += 1
    return {
        "missing_translations": missing_translations,
        "outdated_translations": outdated_translations,
        "missing_meta": missing_meta,
        "missing_hero": missing_hero,
        "broken_images": broken_images,
        "hreflang_errors": hreflang_errors,
        "sitemap_url": url_for("sitemap_xml", _external=True),
        "image_sitemap_url": url_for("image_sitemap_xml", _external=True),
        "robots_url": url_for("robots_txt", _external=True),
    }


@app.route(f"{ADMIN_URL_PREFIX}/qa")
@login_required
def admin_qa():
    return render_template("admin/qa.html", report=qa_report())


@app.route(f"{ADMIN_URL_PREFIX}/production-checklist")
@login_required
def admin_production_checklist():
    settings = get_settings()
    user = get_db().execute("SELECT * FROM users WHERE username = 'admin'").fetchone()
    checklist = [
        ("Debug mode off", not app.debug),
        ("Secret key configured", bool(app.config["SECRET_KEY"]) and app.config["SECRET_KEY"] != "phase-2-local-change-me"),
        ("Admin password changed", not session.get("default_password_active") and not (user and user["must_change_password"])),
        ("Fireworks API key configured", bool(fireworks_api_key(settings))),
        ("OpenAI image key configured if using image edits", bool(openai_content_api_key(settings))),
        ("SQLite database present", DATABASE.exists()),
        ("Static uploads folder present", UPLOADS_DIR.exists()),
        ("Sitemap route available", True),
        ("Robots route available", True),
        ("Contact forms tested manually", False),
        ("Chatbot enabled and tested", bool(settings["chatbot_enabled"] if "chatbot_enabled" in settings.keys() else 0)),
        ("Leads saving checked", get_db().execute("SELECT COUNT(*) FROM leads").fetchone()[0] > 0),
        ("Mobile public pages checked", False),
        ("Desktop public pages checked", False),
        ("No admin links in public navigation", True),
        ("Analytics excludes admin pages", True),
        ("Database and uploads backed up before deploy", False),
        ("Staging/subdomain tested before production", False),
    ]
    return render_template("admin/production_checklist.html", checklist=checklist)


@app.route(f"{ADMIN_URL_PREFIX}/leads")
@login_required
def admin_leads():
    leads = get_db().execute(
        """
        SELECT leads.*, listings.stock_number, listings.title_en
        FROM leads
        LEFT JOIN listings ON listings.id = leads.listing_id
        ORDER BY leads.created_at DESC, leads.id DESC
        """
    ).fetchall()
    return render_template("admin/leads.html", leads=leads)


# ---------------- Lead campaigns (redirect tracking + AI replies) ----------------

def _campaign_link_token():
    return uuid.uuid4().hex


def _campaign_vehicle_dict(listing):
    return {
        "year": listing["year"] or "",
        "make": listing["make"] or "",
        "model": listing["model"] or "",
        "trim": listing["trim"] or "",
        "price": str(listing["price"] or "").replace(",", ""),
        "stock": listing["public_stock_number"] or listing["stock_number"] or "",
        "body_style": listing["body_style"] or "",
        "fuel_type": listing["fuel_type"] or "",
        "odometer": listing["odometer"] or "",
        "carfax_url": (listing["carfax_url"] or "") if "carfax_url" in listing.keys() else "",
        "carfax_enabled": (listing["carfax_enabled"] if "carfax_enabled" in listing.keys() else 1),
    }


def _resolve_vehicle_search(query):
    """Resolve a typed vehicle string (from the filterable datalist) to a listing id.
    Tries VIN substring first, then a LIKE search on title/make/model. Returns id or None."""
    q = (query or "").strip()
    if not q:
        return None
    db = get_db()
    rows = db.execute(
        "SELECT id, vin FROM listings WHERE COALESCE(is_deleted,0)=0 AND vin IS NOT NULL AND vin != ''"
    ).fetchall()
    for r in rows:
        if r["vin"] and r["vin"] in q:
            return r["id"]
    like = f"%{q.lower()}%"
    rows = db.execute(
        "SELECT id FROM listings WHERE COALESCE(is_deleted,0)=0 AND "
        "(LOWER(COALESCE(title_en,'')) LIKE ? OR LOWER(COALESCE(make,'')||' '||COALESCE(model,'')) LIKE ?)",
        (like, like),
    ).fetchall()
    if len(rows) == 1:
        return rows[0]["id"]
    return None


def _parse_price(value):
    try:
        return float(re.sub(r"[^0-9.]", "", str(value or "")))
    except (ValueError, TypeError):
        return None


def _norm_title(text):
    return set(re.findall(r"[a-z0-9]+", (text or "").lower()))


def _parse_price_from_line(line):
    m = re.findall(r"\$?\s*([0-9][0-9,]*)", line or "")
    if not m:
        return None
    return _parse_price(m[-1])


def _strip_price_from_line(line):
    return re.sub(r"\$?\s*[0-9][0-9,]*\s*$", "", (line or "")).strip(" -—")


def _parse_lead_block(block):
    """Parse one pasted lead block into {car_title, price, cas_ref, vin, name, phone, email, message}.
    CAS ref + VIN are extracted from the WHOLE block first (they may sit anywhere, and
    their digits must be pulled before any price stripping eats them)."""
    lines = [ln.strip() for ln in (block or "").splitlines() if ln.strip()]
    if len(lines) < 2:
        return None
    car_line = lines[0]
    vin = _extract_vin(block)
    cas_ref = _extract_cas_ref(_VIN_RE.sub(" ", block) if vin else block)
    price = _parse_price_from_line(car_line)
    # clean title for notes: remove VIN / CAS token / price from the first line
    title = car_line
    if vin:
        title = _VIN_RE.sub(" ", title)
    if cas_ref:
        title = _CAS_REF_RE.sub(" ", title)
    title = re.sub(r"\$\s*\d[\d,]*", " ", title)          # $26,000 or $26000
    title = re.sub(r"\b\d{1,3}(?:,\d{3})+\b", " ", title)  # bare 26,000
    title = re.sub(r"\s+", " ", title).strip(" -—")
    car_title = title or car_line
    rest = lines[1:]
    email, phone, name = "", "", ""
    email_idx = None
    for i in range(len(rest) - 1, -1, -1):
        if re.match(r"^[^\s@]+@[^\s@]+\.[^\s@]+$", rest[i]):
            email_idx = i
            email = rest[i]
            break
    if email_idx is None:
        message_parts = rest
    else:
        pre = rest[:email_idx]
        phone_re = re.compile(r"^[\+\d\(\)\-\s]{7,}$")
        if pre and phone_re.match(pre[-1]) and any(c.isdigit() for c in pre[-1]) and "@" not in pre[-1]:
            phone = pre[-1]
            pre = pre[:-1]
        if pre:
            name = pre[-1]
            pre = pre[:-1]
        message_parts = pre
    return {
        "car_title": car_title,
        "price": price,
        "cas_ref": cas_ref or "",
        "vin": vin,
        "name": name,
        "phone": phone,
        "email": email,
        "message": " ".join(message_parts).strip(),
    }


_CAS_REF_RE = re.compile(r"\bCAS[-\s]?(\d{3,})\b", re.IGNORECASE)
_VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)  # VINs exclude I, O, Q


def _extract_cas_ref(text):
    """Extract the CAS stock number digits from a lead subject line, e.g.
    'CAS-1158', 'cas 1158' -> '1158'. Returns None when absent."""
    m = _CAS_REF_RE.search(text or "")
    return m.group(1) if m else None


def _extract_vin(text):
    """Extract a 17-char VIN from anywhere in the lead block. Returns '' when absent."""
    m = _VIN_RE.search(text or "")
    return m.group(1).upper() if m else ""


def _match_listing(cas_ref=None, vin=None):
    """Match a parsed lead to a listing by exact identifier only: CAS stock number
    first, then VIN. No fuzzy title/price matching (it misassigned leads between
    same-price vehicles). No identifier or no match -> None -> Unassigned inbox."""
    db = get_db()
    if cas_ref:
        row = db.execute(
            "SELECT id FROM listings WHERE COALESCE(is_deleted,0)=0 AND ("
            "UPPER(REPLACE(REPLACE(COALESCE(public_stock_number,''),'-',''),' ','')) IN (?, ?) "
            "OR UPPER(REPLACE(REPLACE(COALESCE(stock_number,''),'-',''),' ','')) IN (?, ?))",
            (f"CAS{cas_ref}", cas_ref, f"CAS{cas_ref}", cas_ref),
        ).fetchone()
        if row:
            return row["id"]
    if vin:
        row = db.execute(
            "SELECT id FROM listings WHERE COALESCE(is_deleted,0)=0 AND UPPER(COALESCE(vin,'')) = ?",
            (vin.upper(),),
        ).fetchone()
        if row:
            return row["id"]
    return None


def _normalize_title(t):
    """Normalize a vehicle title for matching: strip price/CAS/VIN tokens, collapse
    whitespace, lowercase. The lead parser already strips price/CAS/VIN from the
    stored car_title, and the admin-entered reference title may include the price —
    normalizing both the same way makes them match."""
    t = t or ""
    t = re.sub(r"\$\s*\d[\d,]*", " ", t)            # $26,000 / $26000
    t = re.sub(r"\b\d{1,3}(?:,\d{3})+\b", " ", t)   # bare 26,000
    t = re.sub(r"\bCAS[-\s]?\d{3,}\b", " ", t, flags=re.I)  # CAS-1158
    t = re.sub(r"\b[A-HJ-NPR-Z0-9]{17}\b", " ", t, flags=re.I)  # VIN
    t = re.sub(r"\s+", " ", t).strip(" -—")
    return t.lower().strip()


def _match_listing_by_title(car_title):
    """Fallback when a parsed lead has no CAS/VIN: look the cleaned title up in
    lead_title_map → stock# → find the listing. Most-recent mapping wins (so adding a
    corrected line overrides an older one). Returns listing id or None."""
    key = _normalize_title(car_title)
    if not key:
        return None
    db = get_db()
    row = db.execute(
        "SELECT stock_number FROM lead_title_map WHERE title_key=? ORDER BY id DESC LIMIT 1",
        (key,)).fetchone()
    if not row or not row["stock_number"]:
        return None
    stock = (row["stock_number"] or "").strip()
    lst = db.execute(
        "SELECT id FROM listings WHERE COALESCE(is_deleted,0)=0 AND "
        "(UPPER(COALESCE(public_stock_number,''))=? OR UPPER(COALESCE(stock_number,''))=?) LIMIT 1",
        (stock.upper(), stock.upper())).fetchone()
    return lst["id"] if lst else None


def _find_or_create_campaign_for_listing(db, listing_id, create_new):
    """Return a campaign id for a listing: latest existing, or a new one (always new if create_new)."""
    if not create_new:
        row = db.execute(
            "SELECT id FROM lead_campaigns WHERE listing_id = ? ORDER BY id DESC LIMIT 1", (listing_id,)
        ).fetchone()
        if row:
            return row["id"]
    cur = db.execute(
        "INSERT INTO lead_campaigns (listing_id, label, created_at) VALUES (?, ?, ?)",
        (listing_id, "", now_utc()),
    )
    return cur.lastrowid


def _unassigned_campaign_id(db):
    row = db.execute(
        "SELECT id FROM lead_campaigns WHERE listing_id IS NULL AND label = 'Unassigned Leads' LIMIT 1"
    ).fetchone()
    if row:
        return row["id"]
    cur = db.execute(
        "INSERT INTO lead_campaigns (listing_id, label, created_at) VALUES (NULL, 'Unassigned Leads', ?)",
        (now_utc(),),
    )
    return cur.lastrowid


def send_campaign_link_reply(link, settings):
    """Send one campaign link's stored (final) reply via SMTP. Returns (ok, msg).
    The body is expected to already contain the tracking link (appended at
    generation/schedule/save time — the send daemon has no request context)."""
    email = (link["lead_email"] or "").strip()
    body = (link["generated_reply"] or "").strip()
    if not email:
        return False, "Cannot send: missing recipient email."
    if not body:
        return False, "Cannot send: no reply text."
    subject = (link["reply_subject"] or "").strip() if "reply_subject" in link.keys() else ""
    if not subject:
        camp = get_db().execute("SELECT listing_id FROM lead_campaigns WHERE id=?", (link["campaign_id"],)).fetchone()
        listing = get_db().execute("SELECT year, make, model FROM listings WHERE id=?", (camp["listing_id"],)).fetchone() if camp and camp["listing_id"] else None
        subject = f"Re: your enquiry about {listing['year']} {listing['make']} {listing['model']}" if listing else "Your vehicle enquiry"
    return smtp_send_message(settings, subject, body, None, recipient=email)


def _mark_link_sent(link_id, body):
    db = get_db()
    db.execute(
        "UPDATE campaign_links SET generated_reply=?, reply_status='sent', sent_at=?, scheduled_at=NULL, send_error=NULL WHERE id=?",
        (body, now_utc(), link_id),
    )
    db.commit()


def _mark_link_send_failed(link_id, msg):
    db = get_db()
    db.execute(
        "UPDATE campaign_links SET reply_status='drafted', scheduled_at=NULL, send_error=? WHERE id=?",
        (str(msg)[:500], link_id),
    )
    db.commit()


def _lead_reply_ai_adapter(settings):
    """Per-feature routed adapter for lead replies (and SMS text-only generation)."""
    return make_text_adapter(settings, "lead_reply")


def _lead_reply_context(listing, settings, host_url, link):
    """Shared generation context for single + batch reply generation."""
    vehicle = _campaign_vehicle_dict(listing)
    tracking_url = f"{host_url}r/{link['token']}"
    carfax_on = bool(int(settings_value(settings, "carfax_button_enabled", 1) or 0)) and bool(vehicle["carfax_enabled"])
    report_url = ""
    try:
        report = inspection_reports.get_report_for_listing(get_db(), listing["id"])
        if report and report["status"] == "approved":
            report_url = f"{host_url}inspection-report/{report['report_token']}"
    except Exception:
        report_url = ""
    input_data = lead_dispatch.build_input_data(
        {"name": link["lead_name"], "email": link["lead_email"],
         "phone": link["lead_phone"], "message": link["lead_message"]},
        vehicle, tracking_url,
        availability_status=lead_dispatch.availability_from_status(listing["status"] if "status" in listing.keys() else ""),
        carfax_on_listing=carfax_on,
        carfax_url=vehicle["carfax_url"] if carfax_on else "",
        inspection_report_url=report_url,
        free_delivery_active=bool(int(settings_value(settings, "free_delivery_promo_enabled", 0) or 0)),
    )
    return {"tracking_url": tracking_url, "vehicle": vehicle, "input_data": input_data}


# Columns for the famulor.io AI voice-campaign CSV (order matches the lead-add form).
VOICE_CSV_HEADERS = [
    "Phone number", "Canada", "Name",
    "customer_first_name", "customer_full_name", "customer_phone",
    "phone_already_available", "customer_language", "vehicle_name",
    "advertised_price", "vehicle_url", "vehicle_location",
    "distance_from_office", "vehicle_source", "carfax_available",
    "carfax_on_listing", "carfax_url", "accident_history", "title_status",
    "service_records_available", "third_party_inspection_completed",
    "inspection_report_on_listing", "outside_ppi_allowed",
    "ontario_safety_included", "free_delivery_active",
    "inspection_period_active", "buyer_city", "buyer_province",
    "alberta_tax_estimate_allowed", "offer_amount", "offer_status",
    "estimated_total", "additional_verified_facts",
    "link_opened",
]


def _normalize_phone_ca(phone):
    """Normalize a Canadian phone to +1XXXXXXXXXX for the voice dialer."""
    digits = re.sub(r"\D", "", phone or "")
    if not digits:
        return ""
    if len(digits) == 11 and digits.startswith("1"):
        return "+" + digits
    if len(digits) == 10:
        return "+1" + digits
    return (phone or "").strip()


def _voice_csv_row(link, listing, settings, host_url):
    """One famulor.io voice-campaign CSV row for a lead. Reuses lead_dispatch.
    build_input_data so the values match the email/AI facts. Business defaults
    (free delivery this month, safety certified, 7-day inspection, third-party
    inspection) are filled because they're known business facts. Per-lead unknowns
    (accident history, title status, buyer city/province, offer) are left blank
    rather than invented."""
    vehicle = _campaign_vehicle_dict(listing)
    carfax_on = bool(int(settings_value(settings, "carfax_button_enabled", 1) or 0)) and bool(vehicle["carfax_enabled"])
    report_url = ""
    try:
        report = inspection_reports.get_report_for_listing(get_db(), listing["id"])
        if report and report["status"] == "approved":
            report_url = f"{host_url}inspection-report/{report['report_token']}"
    except Exception:
        report_url = ""
    free_delivery = bool(int(settings_value(settings, "free_delivery_promo_enabled", 0) or 0))
    slug = (listing["slug"] if "slug" in listing.keys() else "") or ""
    vehicle_url = f"{host_url}inventory/{slug}" if slug else ""
    lead = {"name": link["lead_name"], "email": link["lead_email"],
            "phone": link["lead_phone"], "message": link["lead_message"]}
    data = lead_dispatch.build_input_data(
        lead, vehicle, vehicle_url,
        availability_status="available",  # the voice campaign always treats the car as available
        carfax_on_listing=carfax_on,
        carfax_url=vehicle["carfax_url"] if carfax_on else "",
        inspection_report_url=report_url,
        free_delivery_active=free_delivery,
    )
    phone = _normalize_phone_ca(link["lead_phone"])
    tf = lambda b: "true" if b else "false"
    return {
        "Phone number": phone,
        "Canada": "Canada",
        "Name": (link["lead_name"] or "").strip(),
        "customer_first_name": data["customer_first_name"],
        "customer_full_name": data["customer_full_name"],
        "customer_phone": phone,
        "phone_already_available": tf(phone),
        "customer_language": "English",  # default; override per-lead if you know it's French
        "vehicle_name": data["vehicle_name"],
        "advertised_price": data["advertised_price"],
        "vehicle_url": vehicle_url,
        "vehicle_location": "Secure warehouse, North York, Ontario (address shared after appointment)",
        "distance_from_office": "30 to 40 minutes",
        "vehicle_source": "",
        "carfax_available": tf(data["carfax_available"]),
        "carfax_on_listing": tf(data["carfax_on_listing"]),
        "carfax_url": data["carfax_url"],
        "accident_history": "",
        "title_status": "",
        "service_records_available": "false",
        "third_party_inspection_completed": "true",  # business default for all vehicles
        "inspection_report_on_listing": tf(data["inspection_report_on_listing"]),
        "outside_ppi_allowed": "false",
        "ontario_safety_included": "true",
        "free_delivery_active": tf(free_delivery),
        "inspection_period_active": "true",
        "buyer_city": "",
        "buyer_province": "",
        "alberta_tax_estimate_allowed": "false",
        "offer_amount": "",
        "offer_status": "not applicable",
        "estimated_total": "",
        "additional_verified_facts": "",
        "link_opened": tf(link["visited"]) if "visited" in link.keys() else "false",
    }


REPLY_GEN_JOBS = {}
_reply_gen_lock = threading.Lock()


def _reply_gen_candidates(campaign_id=None):
    """Two kinds of work: (a) fresh/missing leads needing a full reply draft
    (assigned, not sent/scheduled/sending, not duplicate, no reply yet, has lead
    info), and (b) leads that already have a reply (drafted or sent) but no SMS
    text yet — text-only fill, requires a phone number.

    Leads whose vehicle no longer exists (deleted/archived listings) are
    excluded: they can never be drafted and only used to be re-failed — and
    re-slept through — on every run, making each run slower than the last."""
    sql = (
        "SELECT cl.id FROM campaign_links cl JOIN lead_campaigns lc ON lc.id = cl.campaign_id "
        "LEFT JOIN listings l ON l.id = lc.listing_id "
        "WHERE lc.listing_id IS NOT NULL AND (l.id IS NULL OR COALESCE(l.is_deleted,0) = 0) "
        "AND COALESCE(cl.duplicate,0) = 0 "
        "AND (COALESCE(cl.lead_name,'') != '' OR COALESCE(cl.lead_email,'') != '' OR COALESCE(cl.lead_message,'') != '') "
        "AND ("
        "  (COALESCE(cl.reply_status,'') NOT IN ('sent','scheduled','sending') AND COALESCE(cl.generated_reply,'') = '')"
        "  OR (COALESCE(cl.lead_phone,'') != '' AND COALESCE(cl.text_message,'') = '' AND COALESCE(cl.generated_reply,'') != '')"
        ")"
    )
    params = []
    if campaign_id is not None:
        sql += " AND cl.campaign_id = ?"
        params.append(campaign_id)
    return get_db().execute(sql + " ORDER BY cl.id", params).fetchall()


def _run_reply_gen_batch(job_id, link_ids, host_url, ai_call=None):
    job = REPLY_GEN_JOBS[job_id]
    job.setdefault("texts", 0)
    # Per-AI-call timing (tests construct bare job dicts — hence setdefault).
    job.setdefault("calls", 0)
    job.setdefault("call_seconds", 0.0)
    job.setdefault("last_call_s", 0.0)
    db = get_db()
    settings = get_settings()
    master_prompt = settings_value(settings, "lead_reply_master_prompt", "") or ""
    dealer_name = settings_value(settings, "company_name", "") or ""
    if ai_call is None:
        _status, ai_call = _lead_reply_ai_adapter(settings)

    def _heartbeat():
        """Prove the worker is alive so the stall-watchdog doesn't restart the batch."""
        try:
            db.execute("UPDATE site_settings SET reply_gen_progress_at=? WHERE id=1", (now_utc(),))
            db.commit()
        except Exception:
            pass

    for lid in link_ids:
        _heartbeat()
        link = db.execute(
            "SELECT cl.*, lc.listing_id AS camp_listing_id FROM campaign_links cl "
            "JOIN lead_campaigns lc ON lc.id = cl.campaign_id WHERE cl.id = ?",
            (lid,),
        ).fetchone()
        if not link or not link["camp_listing_id"]:
            job["skipped"] += 1
            job["done"] += 1
            continue
        job["current"] = lid
        db.execute("UPDATE campaign_links SET gen_status='generating' WHERE id=?", (lid,))
        db.commit()
        listing = db.execute("SELECT * FROM listings WHERE id=?", (link["camp_listing_id"],)).fetchone()
        if listing is None:
            db.execute("UPDATE campaign_links SET gen_status='failed', send_error='Listing not found (deleted?)' WHERE id=?", (lid,))
            db.commit()
            job["failed"] += 1
            job["errors"][str(lid)] = "Listing not found (deleted?)"
            job["done"] += 1
            continue
        ctx = _lead_reply_context(listing, settings, host_url, link)
        lead = {"name": link["lead_name"], "email": link["lead_email"],
                "phone": link["lead_phone"], "message": link["lead_message"]}
        phone = (link["lead_phone"] or "").strip()
        sms_url = f"{host_url}r/{link['sms_token']}" if (phone and link["sms_token"]) else ""
        sms_hint = lead_dispatch.sms_style_hint(link["id"]) if sms_url else ""
        if (link["generated_reply"] or "").strip():
            # Text-only: the email reply already exists (drafted or even sent) —
            # never touch it, just fill the missing SMS text.
            if not sms_url:
                job["skipped"] += 1
                job["done"] += 1
                continue
            result = None
            _t0 = time.time()
            for attempt in (1, 2):
                result = lead_replies.generate_sms_text(ctx["input_data"], sms_url, sms_hint, ai_call)
                if result.get("status") == "ok":
                    break
                if attempt == 1:
                    time.sleep(5)
            _call_s = time.time() - _t0
            job["calls"] += 1
            job["call_seconds"] += _call_s
            job["last_call_s"] = round(_call_s, 1)
            if result.get("status") == "ok":
                db.execute(
                    "UPDATE campaign_links SET text_message=?, gen_status='drafted', send_error=NULL WHERE id=?",
                    (result["text_message"], lid),
                )
                job["texts"] += 1
            else:
                err = str(result.get("error", "unknown"))[:500]
                db.execute("UPDATE campaign_links SET gen_status='failed', send_error=? WHERE id=?", (err, lid))
                job["failed"] += 1
                job["errors"][str(lid)] = err
            db.commit()
            job["done"] += 1
            continue
        result = None
        _t0 = time.time()
        for attempt in (1, 2):  # one automatic retry on transient AI errors
            result = lead_replies.generate_lead_reply(
                lead, ctx["vehicle"], master_prompt, dealer_name, ai_call,
                tracking_url=ctx["tracking_url"], input_data=ctx["input_data"],
                sms_url=sms_url, sms_style_hint=sms_hint,
            )
            if result.get("status") == "ok":
                break
            if attempt == 1:
                time.sleep(5)
        _call_s = time.time() - _t0
        job["calls"] += 1
        job["call_seconds"] += _call_s
        job["last_call_s"] = round(_call_s, 1)
        if result.get("status") == "ok":
            db.execute(
                "UPDATE campaign_links SET generated_reply=?, reply_subject=?, text_message=?, reply_status='drafted', gen_status='drafted', send_error=NULL WHERE id=?",
                (result["body"], result.get("subject", ""), result.get("text_message", ""), lid),
            )
            job["drafted"] += 1
        else:
            err = str(result.get("error", "unknown"))[:500]
            db.execute("UPDATE campaign_links SET gen_status='failed', send_error=? WHERE id=?", (err, lid))
            job["failed"] += 1
            job["errors"][str(lid)] = err
        db.commit()
        job["done"] += 1
    _heartbeat()
    job["current"] = None
    job["finished"] = True


def _prune_reply_gen_jobs():
    finished = [k for k, j in REPLY_GEN_JOBS.items() if j.get("finished")]
    for k in finished[:-10]:
        REPLY_GEN_JOBS.pop(k, None)


REPLY_GEN_STALE_MINUTES = 15


def trigger_reply_generation_batch(campaign_id=None, host_url=""):
    """Start a background batch. Returns job_id, or None if one is already running.

    Two-level lock: fast in-process lock + a site_settings flag so other
    processes (e.g. Passenger workers) also see the batch as running. The flag
    is stamped with a start time and considered stale after 15 min (dead worker).
    Candidate links are stamped with gen_job up front so the status endpoint can
    rebuild progress from the DB if this process's memory is lost."""
    if not _reply_gen_lock.acquire(blocking=False):
        return None
    try:
        db = get_db()
        job_id = uuid.uuid4().hex[:12]
        stale_before = (datetime.utcnow() - timedelta(minutes=REPLY_GEN_STALE_MINUTES)).isoformat(timespec="seconds")
        cur = db.execute(
            "UPDATE site_settings SET reply_gen_active_job=?, reply_gen_started_at=?, reply_gen_progress_at=? WHERE id=1 "
            "AND (COALESCE(reply_gen_active_job,'') = '' OR COALESCE(reply_gen_started_at,'') < ?)",
            (job_id, now_utc(), now_utc(), stale_before),
        )
        db.commit()
        if cur.rowcount != 1:
            _reply_gen_lock.release()
            return None
        ids = [r["id"] for r in _reply_gen_candidates(campaign_id)]
        if ids:
            placeholders = ",".join("?" for _ in ids)
            db.execute(
                f"UPDATE campaign_links SET gen_job=?, "
                f"gen_status=CASE WHEN gen_status='generating' THEN NULL ELSE gen_status END "
                f"WHERE id IN ({placeholders})",
                [job_id, *ids],
            )
            db.commit()
        REPLY_GEN_JOBS[job_id] = {"total": len(ids), "done": 0, "drafted": 0, "failed": 0,
                                  "skipped": 0, "finished": False, "current": None, "errors": {},
                                  "calls": 0, "call_seconds": 0.0, "last_call_s": 0.0}

        def _wrapped():
            with app.app_context():
                try:
                    _run_reply_gen_batch(job_id, ids, host_url)
                except Exception as e:
                    job = REPLY_GEN_JOBS.get(job_id)
                    if job:
                        job["finished"] = True
                        job["errors"]["batch"] = f"Batch worker crashed: {e}"
                        job["current"] = None
                finally:
                    try:
                        get_db().execute(
                            "UPDATE site_settings SET reply_gen_active_job=NULL, reply_gen_started_at=NULL "
                            "WHERE reply_gen_active_job=?",
                            (job_id,),
                        )
                        get_db().commit()
                    except Exception:
                        pass
                    _prune_reply_gen_jobs()
                    _reply_gen_lock.release()

        threading.Thread(target=_wrapped, daemon=True).start()
        return job_id
    except Exception:
        if _reply_gen_lock.locked():
            _reply_gen_lock.release()
        raise


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns", methods=("GET", "POST"))
@login_required
def admin_lead_campaigns():
    db = get_db()
    if request.method == "POST":
        action = (request.form.get("campaign_action") or "").strip()
        if action == "save_master_prompt":
            db.execute(
                "UPDATE site_settings SET lead_reply_master_prompt = ?, updated_at = ? WHERE id = 1",
                ((request.form.get("lead_reply_master_prompt") or "").strip(), now_utc()),
            )
            db.commit()
            flash("Master reply prompt saved.", "success")
            return redirect(url_for("admin_lead_campaigns"))
        if action == "create":
            try:
                count = max(1, min(50, int(request.form.get("count") or 1)))
            except ValueError:
                count = 1
            label = (request.form.get("label") or "").strip()
            listing_id = _resolve_vehicle_search(request.form.get("vehicle_search", ""))
            if not listing_id:
                flash("Pick a valid vehicle — type the VIN, make, or model and select from the list.", "warning")
                return redirect(url_for("admin_lead_campaigns"))
            cur = db.execute(
                "INSERT INTO lead_campaigns (listing_id, label, created_at) VALUES (?, ?, ?)",
                (listing_id, label, now_utc()),
            )
            cid = cur.lastrowid
            for _ in range(count):
                db.execute(
                    "INSERT INTO campaign_links (campaign_id, token, sms_token, created_at) VALUES (?, ?, ?, ?)",
                    (cid, _campaign_link_token(), _campaign_link_token(), now_utc()),
                )
            db.commit()
            flash(f"Campaign created with {count} link(s).", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid))
        if action == "delete" and (request.form.get("campaign_id") or "").isdigit():
            cid = int(request.form.get("campaign_id"))
            db.execute("DELETE FROM campaign_links WHERE campaign_id = ?", (cid,))
            db.execute("DELETE FROM lead_campaigns WHERE id = ?", (cid,))
            db.commit()
            flash("Campaign deleted.", "success")
            return redirect(url_for("admin_lead_campaigns"))
    campaigns = db.execute(
        """SELECT lc.*, l.title_en, l.stock_number, l.public_stock_number,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id) AS link_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND cl.visited = 1) AS visited_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND cl.reply_status = 'sent') AS sent_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND cl.reply_status = 'drafted') AS drafted_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND COALESCE(cl.send_error,'') != '') AS error_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND cl.texted_at IS NOT NULL) AS texted_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id AND COALESCE(cl.duplicate,0)=0 AND COALESCE(cl.lead_phone,'') != '') AS phone_count,
                  (SELECT COUNT(*) FROM campaign_links cl WHERE cl.campaign_id = lc.id
                    AND COALESCE(cl.duplicate,0) = 0 AND COALESCE(cl.generated_reply,'') = ''
                    AND COALESCE(cl.reply_status,'') NOT IN ('sent','scheduled','sending')
                    AND (COALESCE(cl.lead_name,'') != '' OR COALESCE(cl.lead_email,'') != '' OR COALESCE(cl.lead_message,'') != '')) AS missing_count
           FROM lead_campaigns lc LEFT JOIN listings l ON l.id = lc.listing_id
           ORDER BY lc.created_at DESC, lc.id DESC"""
    ).fetchall()
    listings = db.execute(
        "SELECT id, vin, stock_number, public_stock_number, title_en, year, make, model "
        "FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY year DESC, make, model"
    ).fetchall()
    counters = db.execute(
        """SELECT
             SUM(CASE WHEN COALESCE(duplicate,0)=0 AND (COALESCE(lead_name,'') != '' OR COALESCE(lead_email,'') != '' OR COALESCE(lead_message,'') != '') THEN 1 ELSE 0 END) AS leads_loaded,
             SUM(CASE WHEN COALESCE(duplicate,0)=0 AND reply_status='drafted' THEN 1 ELSE 0 END) AS drafts_ready,
             SUM(CASE WHEN COALESCE(duplicate,0)=0 AND COALESCE(generated_reply,'')=''
                  AND COALESCE(reply_status,'') NOT IN ('sent','scheduled','sending')
                  AND (COALESCE(lead_name,'') != '' OR COALESCE(lead_email,'') != '' OR COALESCE(lead_message,'') != '') THEN 1 ELSE 0 END) AS drafts_missing,
             SUM(CASE WHEN COALESCE(duplicate,0)=0 AND COALESCE(lead_phone,'') != ''
                  AND COALESCE(generated_reply,'') != '' AND COALESCE(text_message,'') = '' THEN 1 ELSE 0 END) AS texts_missing,
             SUM(CASE WHEN COALESCE(send_error,'') != '' AND COALESCE(gen_status,'') = 'failed' THEN 1 ELSE 0 END) AS gen_errors,
             SUM(CASE WHEN COALESCE(send_error,'') != '' AND COALESCE(gen_status,'') != 'failed' THEN 1 ELSE 0 END) AS send_errors
           FROM campaign_links"""
    ).fetchone()
    settings = get_settings()
    master_prompt = settings_value(settings, "lead_reply_master_prompt", "") or ""
    active_job = settings_value(settings, "reply_gen_active_job", "") or ""
    title_map_rows = db.execute("SELECT title, stock_number FROM lead_title_map ORDER BY id ASC").fetchall()
    title_map_text = "\n".join(f"{r['title']}|{r['stock_number']}" for r in title_map_rows)
    return render_template(
        "admin/lead_campaigns.html", campaigns=campaigns, listings=listings, master_prompt=master_prompt,
        counters=counters, active_job=active_job, title_map_text=title_map_text,
    )


LEAD_STATUS_BUCKETS = {
    "drafts_ready": (
        "Drafts ready to send",
        "Replies that are drafted but were never sent. Drafts are NOT sent automatically — send one here, "
        "schedule it from the campaign page, or delete it.",
    ),
    "drafts_missing": (
        "Drafts missing",
        "Leads without a generated reply. Generate a reply here, or see why a lead can't be drafted "
        "(leads on deleted cars or in the Unassigned campaign cannot be drafted until reassigned).",
    ),
    "texts_missing": (
        "Texts missing",
        "Leads that have a phone number and a reply, but no SMS text yet.",
    ),
    "gen_errors": (
        "Generation errors",
        "Leads whose reply drafting failed — the error is shown per lead. Retry, clear, or delete. "
        "Leads on deleted cars always fail until the lead is reassigned to a live car.",
    ),
    "send_errors": (
        "Send errors",
        "Leads whose email send failed — the error is shown per lead. Retry the send, clear the error, or delete.",
    ),
}


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/status/<bucket>", methods=("GET", "POST"))
@login_required
def admin_lead_campaigns_status(bucket):
    """Drill-down for the counters on the Lead Campaigns page: click a counter,
    see the exact leads behind it, and act on them (send / retry / generate /
    clear error / delete) without digging through each campaign."""
    if bucket not in LEAD_STATUS_BUCKETS:
        abort(404)
    db = get_db()
    back = url_for("admin_lead_campaigns_status", bucket=bucket)
    if request.method == "POST":
        action = (request.form.get("link_action") or "").strip()
        lid = request.form.get("link_id") or ""
        if action in ("send_now", "retry_send") and lid.isdigit():
            link = db.execute("SELECT * FROM campaign_links WHERE id=?", (int(lid),)).fetchone()
            if not link:
                flash("Lead not found.", "warning")
                return redirect(back)
            body = (link["generated_reply"] or "").strip()
            if not (link["lead_email"] or "").strip():
                flash("Cannot send: this lead has no email address.", "warning")
                return redirect(back)
            if not body:
                flash("Cannot send: no reply text yet — use Generate first.", "warning")
                return redirect(back)
            tracking_url = f"{request.host_url}r/{link['token']}"
            if tracking_url not in body:
                body = body.rstrip() + f"\n\nView the vehicle here: {tracking_url}"
                db.execute("UPDATE campaign_links SET generated_reply=? WHERE id=?", (body, link["id"]))
                db.commit()
            link = db.execute("SELECT * FROM campaign_links WHERE id=?", (link["id"],)).fetchone()
            ok, msg = send_campaign_link_reply(link, get_settings())
            if ok:
                _mark_link_sent(link["id"], body)
                flash("Reply sent.", "success")
            else:
                _mark_link_send_failed(link["id"], msg)
                flash(f"Send failed: {msg}", "warning")
            return redirect(back)
        if action == "generate" and lid.isdigit():
            link = db.execute("SELECT * FROM campaign_links WHERE id=?", (int(lid),)).fetchone()
            if not link:
                flash("Lead not found.", "warning")
                return redirect(back)
            ok, payload = _generate_reply_for_link(link, request.host_url)
            if ok:
                flash("Reply drafted — ready to send.", "success")
            else:
                db.execute(
                    "UPDATE campaign_links SET gen_status='failed', send_error=? WHERE id=?",
                    (str(payload)[:500], link["id"]),
                )
                db.commit()
                flash(f"Generation failed: {payload}", "warning")
            return redirect(back)
        if action == "clear_error" and lid.isdigit():
            db.execute(
                "UPDATE campaign_links SET send_error=NULL, "
                "gen_status=CASE WHEN gen_status='failed' THEN NULL ELSE gen_status END WHERE id=?",
                (int(lid),),
            )
            db.commit()
            flash("Error cleared.", "success")
            return redirect(back)
        if action == "delete" and lid.isdigit():
            db.execute("DELETE FROM campaign_links WHERE id=?", (int(lid),))
            db.commit()
            flash("Lead deleted.", "success")
            return redirect(back)
        flash("Unknown action.", "warning")
        return redirect(back)
    where = {
        "drafts_ready": "COALESCE(cl.reply_status,'') = 'drafted' AND COALESCE(cl.duplicate,0) = 0",
        "drafts_missing": (
            "COALESCE(cl.duplicate,0) = 0 AND COALESCE(cl.generated_reply,'') = '' "
            "AND COALESCE(cl.reply_status,'') NOT IN ('sent','scheduled','sending') "
            "AND (COALESCE(cl.lead_name,'') != '' OR COALESCE(cl.lead_email,'') != '' OR COALESCE(cl.lead_message,'') != '')"
        ),
        "texts_missing": (
            "COALESCE(cl.duplicate,0) = 0 AND COALESCE(cl.lead_phone,'') != '' "
            "AND COALESCE(cl.generated_reply,'') != '' AND COALESCE(cl.text_message,'') = ''"
        ),
        "gen_errors": "COALESCE(cl.send_error,'') != '' AND COALESCE(cl.gen_status,'') = 'failed'",
        "send_errors": "COALESCE(cl.send_error,'') != '' AND COALESCE(cl.gen_status,'') != 'failed'",
    }
    rows = db.execute(
        f"""SELECT cl.*, lc.label AS campaign_label, lc.listing_id AS camp_listing_id,
                   l.year, l.make, l.model, l.trim, COALESCE(l.is_deleted,0) AS listing_deleted
            FROM campaign_links cl
            JOIN lead_campaigns lc ON lc.id = cl.campaign_id
            LEFT JOIN listings l ON l.id = lc.listing_id
            WHERE {where[bucket]}
            ORDER BY cl.id DESC LIMIT 500"""
    ).fetchall()
    title, subtitle = LEAD_STATUS_BUCKETS[bucket]
    return render_template(
        "admin/lead_campaign_status.html", bucket=bucket, title=title, subtitle=subtitle, rows=rows,
    )


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/import", methods=("POST",))
@login_required
def admin_lead_campaigns_import():
    db = get_db()
    raw = request.form.get("paste_leads", "") or ""
    create_new = bool(request.form.get("create_new_campaign"))
    blocks = [b.strip() for b in re.split(r"\n\s*\n", raw) if b.strip()]
    assigned, unassigned, duplicates = 0, 0, 0
    for block in blocks:
        parsed = _parse_lead_block(block)
        if not parsed:
            continue
        lid = _match_listing(parsed.get("cas_ref"), parsed.get("vin"))
        if not lid:
            # Fallback: no CAS/VIN → look the title up in the admin's reference map
            # (lead_title_map: "Title $price|CAS-####") to auto-assign the car.
            lid = _match_listing_by_title(parsed.get("car_title"))
        cid = _find_or_create_campaign_for_listing(db, lid, create_new) if lid else _unassigned_campaign_id(db)
        dup = 0
        if parsed["email"]:
            # A lead is a duplicate ONLY if a copy was already SENT or is in-flight
            # (scheduled/sending) — so re-importing an UNSNET lead still lets exactly one
            # copy send. Mark any prior unsent copies as dup so only this new one sends
            # (was: any prior row at all → dup, which marked ALL re-imported copies as
            # dup and sent none).
            prior_active = db.execute(
                "SELECT 1 FROM campaign_links WHERE lead_email=? AND reply_status IN ('sent','scheduled','sending') LIMIT 1",
                (parsed["email"],)).fetchone()
            if prior_active:
                dup = 1
            else:
                db.execute(
                    "UPDATE campaign_links SET duplicate=1 WHERE lead_email=? AND COALESCE(duplicate,0)=0 "
                    "AND reply_status NOT IN ('sent','scheduled','sending')",
                    (parsed["email"],))
        db.execute(
            "INSERT INTO campaign_links (campaign_id, token, sms_token, lead_name, lead_email, lead_phone, lead_message, notes, duplicate, created_at) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (cid, _campaign_link_token(), _campaign_link_token(), parsed["name"], parsed["email"], parsed["phone"],
             parsed["message"], parsed["car_title"], dup, now_utc()),
        )
        if dup:
            duplicates += 1
        elif lid:
            assigned += 1
        else:
            unassigned += 1
    db.commit()
    flash(f"Imported {assigned} assigned + {unassigned} unassigned + {duplicates} duplicate lead(s). Duplicates are marked so you can skip them.", "success")
    return redirect(url_for("admin_lead_campaigns"))


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/title-map", methods=("POST",))
@login_required
def admin_lead_title_map():
    """Append new Title→stock# mappings from the reference editor on the lead-campaigns
    page. One per line: 'Title $price|CAS-####'. Duplicates (same normalized title +
    stock) are ignored (INSERT OR IGNORE). Most-recent mapping wins at lookup time."""
    db = get_db()
    raw = request.form.get("title_map", "") or ""
    added, skipped = 0, 0
    for line in raw.splitlines():
        line = line.strip()
        if not line or "|" not in line:
            continue
        title, _, stock = line.partition("|")
        title = (title or "").strip()
        stock = (stock or "").strip()
        if not title or not stock:
            continue
        key = _normalize_title(title)
        cur = db.execute(
            "INSERT OR IGNORE INTO lead_title_map (title, title_key, stock_number, created_at) VALUES (?,?,?,?)",
            (title, key, stock, now_utc()))
        if cur.rowcount:
            added += 1
        else:
            skipped += 1
    db.commit()
    flash(f"Title map updated: {added} new mapping(s) added, {skipped} already existed (ignored).", "success")
    return redirect(url_for("admin_lead_campaigns"))


def _start_reply_gen_response(campaign_id=None):
    settings = get_settings()
    status, _adapter = _lead_reply_ai_adapter(settings)
    if not status["active_configured"]:
        return jsonify({"ok": False, "error": f"{status['provider'].title()} API key is missing."}), 400
    job_id = trigger_reply_generation_batch(campaign_id, host_url=request.host_url)
    if job_id is None:
        return jsonify({"ok": False, "error": "A generation batch is already running."}), 409
    return jsonify({"ok": True, "job_id": job_id})


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/generate-all", methods=("POST",))
@login_required
def admin_lead_campaigns_generate_all():
    return _start_reply_gen_response(None)


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/<int:cid>/generate-all", methods=("POST",))
@login_required
def admin_lead_campaign_generate_all(cid):
    return _start_reply_gen_response(cid)


REPLY_GEN_STALL_SECONDS = 400  # > worst-case single link (2x150s timeout + retry gap)


def _reply_gen_db_counts(job_id):
    rows = get_db().execute(
        "SELECT COALESCE(gen_status,'') AS gs, COUNT(*) AS n FROM campaign_links WHERE gen_job=? "
        "GROUP BY COALESCE(gen_status,'')",
        (job_id,),
    ).fetchall()
    counts = {r["gs"]: r["n"] for r in rows}
    total = sum(counts.values())
    drafted = counts.get("drafted", 0)
    failed = counts.get("failed", 0)
    return {"total": total, "done": drafted + failed, "drafted": drafted, "failed": failed}


def _maybe_continue_stalled_batch(job_id):
    """If the batch's worker process died mid-run (restart / host recycling), pick up
    the remaining links from this process. Claimed via a conditional settings UPDATE
    so only one process can continue; the in-process lock guards this one."""
    db = get_db()
    settings = get_settings()
    if (settings_value(settings, "reply_gen_active_job", "") or "") != job_id:
        return False
    progress_at = settings_value(settings, "reply_gen_progress_at", "") or settings_value(settings, "reply_gen_started_at", "") or ""
    try:
        last = datetime.fromisoformat(progress_at)
    except ValueError:
        return False
    if (datetime.utcnow() - last).total_seconds() < REPLY_GEN_STALL_SECONDS:
        return False  # healthy worker, just slow
    remaining = [r["id"] for r in db.execute(
        "SELECT id FROM campaign_links WHERE gen_job=? AND COALESCE(gen_status,'') IN ('','generating')",
        (job_id,)).fetchall()]
    if not remaining:
        return False
    cur = db.execute(
        "UPDATE site_settings SET reply_gen_progress_at=?, reply_gen_started_at=? "
        "WHERE id=1 AND reply_gen_active_job=?",
        (now_utc(), now_utc(), job_id),
    )
    db.commit()
    if cur.rowcount != 1:
        return False  # another process claimed the continuation first
    if not _reply_gen_lock.acquire(blocking=False):
        return False
    try:
        db.execute("UPDATE campaign_links SET gen_status=NULL WHERE gen_job=? AND gen_status='generating'", (job_id,))
        db.commit()
        counts = _reply_gen_db_counts(job_id)
        REPLY_GEN_JOBS[job_id] = {"total": counts["total"], "done": counts["done"],
                                  "drafted": counts["drafted"], "failed": counts["failed"],
                                  "skipped": 0, "texts": 0, "finished": False,
                                  "current": None, "errors": {}, "continued": True}
        host_url = request.host_url

        def _wrapped():
            with app.app_context():
                try:
                    _run_reply_gen_batch(job_id, remaining, host_url)
                except Exception as e:
                    job = REPLY_GEN_JOBS.get(job_id)
                    if job:
                        job["finished"] = True
                        job["errors"]["batch"] = f"Continuation worker crashed: {e}"
                        job["current"] = None
                finally:
                    try:
                        get_db().execute(
                            "UPDATE site_settings SET reply_gen_active_job=NULL, reply_gen_started_at=NULL "
                            "WHERE reply_gen_active_job=?",
                            (job_id,),
                        )
                        get_db().commit()
                    except Exception:
                        pass
                    _reply_gen_lock.release()

        threading.Thread(target=_wrapped, daemon=True).start()
        return True
    except Exception:
        if _reply_gen_lock.locked():
            _reply_gen_lock.release()
        raise


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/generate-status/<job_id>")
@login_required
def admin_lead_campaign_generate_status(job_id):
    job = REPLY_GEN_JOBS.get(job_id)
    if job:
        payload = {"ok": True, **{k: job.get(k, 0) for k in ("total", "done", "drafted", "failed", "skipped", "texts", "finished")}}
        calls = job.get("calls") or 0
        if calls:
            payload["last_call_s"] = job.get("last_call_s") or 0
            payload["avg_call_s"] = round((job.get("call_seconds") or 0) / calls, 1)
    else:
        # DB fallback (multi-process / restart-safe): progress derived from gen_job stamps.
        counts = _reply_gen_db_counts(job_id)
        if counts["total"] == 0:
            return jsonify({"ok": False, "error": "Unknown job."}), 404
        payload = {"ok": True, "skipped": 0, "texts": 0,
                   "finished": counts["done"] == counts["total"], **counts}
    if not payload["finished"]:
        _maybe_continue_stalled_batch(job_id)
    return jsonify(payload)


# ---------------- AI speed diagnostics ----------------
# One-click answer to "why is every AI call slow": timed probes against the real
# provider config on THIS server, comparing thinking-off vs default behavior,
# tiny vs realistic prompt sizes — plus whether the endpoint honors the
# thinking:disabled param at all.

_ai_diag_jobs = {"current": None}


def _run_ai_diagnostics(job):
    settings = get_settings()
    provider = provider_for(settings, "lead_reply")
    job["provider"] = provider
    job["model"] = (settings["zai_model"] if provider == "zai"
                    else settings["openai_model"] if provider == "openai"
                    else DEFAULT_FIREWORKS_MODEL)
    fallback_before = _zai_thinking_fallbacks["count"]

    def probe(label, prompt, thinking, max_tokens, endpoint="anthropic"):
        t0 = time.time()
        entry = {"label": label, "thinking": "disabled" if thinking else "default",
                 "endpoint": endpoint, "max_tokens": max_tokens, "prompt_chars": len(prompt),
                 "ok": False, "seconds": None, "stop_reason": "", "input_tokens": 0,
                 "output_tokens": 0, "text_chars": 0, "error": ""}
        try:
            if endpoint == "v4":
                payload, _model = _zai_v4_request(
                    settings, [{"role": "user", "content": prompt}],
                    system="Return one complete valid JSON object only. Never use markdown fences.",
                    max_tokens=max_tokens, timeout=150, thinking=thinking)
                usage = payload.get("usage") or {}
                try:
                    finish = str((payload.get("choices") or [{}])[0].get("finish_reason") or "")
                except (IndexError, AttributeError):
                    finish = ""
                entry.update({"ok": True, "seconds": round(time.time() - t0, 1),
                              "stop_reason": finish,
                              "input_tokens": usage.get("prompt_tokens") or 0,
                              "output_tokens": usage.get("completion_tokens") or 0,
                              "text_chars": len(_zai_v4_first_text(payload))})
            else:
                payload, _model = _zai_anthropic_request(
                    settings, [{"role": "user", "content": prompt}],
                    system="Return one complete valid JSON object only. Never use markdown fences.",
                    max_tokens=max_tokens, timeout=150, thinking=thinking)
                usage = payload.get("usage") or {}
                entry.update({"ok": True, "seconds": round(time.time() - t0, 1),
                              "stop_reason": payload.get("stop_reason") or "",
                              "input_tokens": usage.get("input_tokens") or 0,
                              "output_tokens": usage.get("output_tokens") or 0,
                              "text_chars": len(_zai_first_text(payload))})
        except Exception as e:
            entry.update({"seconds": round(time.time() - t0, 1), "error": str(e)[:250]})
        job["probes"].append(entry)
        job["log"].append(f"{label}: {entry['seconds']}s {'OK' if entry['ok'] else 'FAILED — ' + entry['error'][:80]}")
        return entry

    # 1-2. Anthropic endpoint: thinking-off floor vs default (reasoning) — is the param honored there?
    p1 = probe("Anthropic · tiny · thinking OFF", 'Return {"ok": true}', {"type": "disabled"}, 300)
    p2 = probe("Anthropic · tiny · thinking DEFAULT", 'Return {"ok": true}', None, 300)
    # 3. Native v4 endpoint with thinking off — the path the adapter now prefers.
    p3 = probe("v4 · tiny · thinking OFF", 'Return {"ok": true}', {"type": "disabled"}, 300, endpoint="v4")
    # 4. A realistic lead-reply prompt (your real master prompt + vehicle + lead) via v4.
    master_prompt = settings_value(settings, "lead_reply_master_prompt", "") or ""
    listing = get_db().execute(
        "SELECT * FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY id DESC LIMIT 1").fetchone()
    input_data = {
        "lead_name": "Jane Doe", "lead_email": "jane@example.com", "lead_phone": "4165550100",
        "lead_message": "Hi, is this car still available? Can we come see it this weekend?",
        "vehicle_year": (listing["year"] if listing else ""), "vehicle_make": (listing["make"] if listing else ""),
        "vehicle_model": (listing["model"] if listing else ""), "vehicle_price": (listing["price"] if listing else ""),
        "vehicle_odometer": (listing["odometer"] if listing else ""), "availability": "Available",
        "carfax_on_listing": True, "link_opened": "false",
    }
    real_prompt = lead_replies.build_lead_reply_prompt(
        {"name": "Jane Doe", "email": "jane@example.com", "phone": "4165550100",
         "message": "Hi, is this car still available? Can we come see it this weekend?"},
        {"year": input_data["vehicle_year"], "make": input_data["vehicle_make"],
         "model": input_data["vehicle_model"]},
        master_prompt, settings["company_name"] or "",
        tracking_url="https://46cars.ca/r/sample", input_data=input_data,
        sms_url="https://46cars.ca/r/sample", sms_style_hint="natural and direct")
    p4 = probe("Real reply prompt · v4 · thinking OFF", real_prompt, {"type": "disabled"}, 1000, endpoint="v4")
    # 5. The turbo payload description prompt (what every payload generation pays) via v4.
    p5 = probe("Turbo description · v4 · thinking OFF",
               _turbo_description_prompt(listing) if listing else "Return {\"ok\": true}",
               {"type": "disabled"}, 600, endpoint="v4")

    fallback_delta = _zai_thinking_fallbacks["count"] - fallback_before
    job["thinking_fallbacks"] = {**_zai_thinking_fallbacks, "this_run": fallback_delta}
    job["v4_fallbacks"] = {**_zai_v4_fallbacks}
    notes = []
    if not p1["ok"] and not p3["ok"]:
        notes.append(f"Z.ai rejected even the smallest calls: {(p1['error'] or p3['error'])[:120]} — fix the API key first.")
    elif p3["ok"] and p3["seconds"] < 15 and (
            not p1["ok"] or p1["seconds"] > 15 or
            (p2["ok"] and (p2["seconds"] > p3["seconds"] * 2 + 8 or p2["stop_reason"] == "max_tokens"))):
        notes.append(f"FOUND IT: the Anthropic-compat endpoint does not really apply thinking:disabled "
                     f"({p1['seconds']}s / {p2['seconds']}s with reasoning vs {p3['seconds']}s on v4). "
                     "The adapter now prefers the v4 endpoint — AI calls should be fast again.")
    elif not p3["ok"] and _zai_v4_worth_fallback(p3["error"]):
        notes.append(f"⚠ The native v4 endpoint is unavailable ({p3['error'][:100]}) — the adapter falls back to the "
                     "Anthropic endpoint, where thinking may stay on. Send me this message and I'll switch approach "
                     "(e.g. pin a non-reasoning model for utility calls).")
    elif p1["ok"] and p1["seconds"] > 15 and p3["ok"] and p3["seconds"] > 15:
        notes.append(f"Even minimal calls are slow on BOTH endpoints ({p1['seconds']}s / {p3['seconds']}s) — "
                     "the server→Z.ai network path itself is slow (hosting egress / routing), not the model.")
    if p4["ok"] and p3["ok"] and p4["seconds"] > p3["seconds"] + 10:
        notes.append(f"Your real reply prompt adds {round(p4['seconds'] - p3['seconds'], 1)}s over the tiny one — "
                     f"prompt is {len(real_prompt)} chars (master prompt length drives this). Trimming it speeds every lead.")
    if fallback_delta > 0:
        notes.append("⚠ The Anthropic endpoint REJECTED the thinking param on some calls this run — those calls ran "
                     "with reasoning on.")
    ok_probes = [p for p in (p1, p2, p3, p4, p5) if p["ok"]]
    if ok_probes and all(p["seconds"] < 10 for p in ok_probes):
        notes.append("All probes are fast on this server right now. If batches still crawl, re-run diagnostics "
                     "during a slow call and compare.")
    job["notes"] = notes or ["Everything measured within normal range on this run."]


@app.route(f"{ADMIN_URL_PREFIX}/ai/diagnostics", methods=("POST",))
@login_required
def admin_ai_diagnostics():
    """Run timed AI probes in the background to pinpoint where slow AI calls spend
    their time (network vs reasoning vs prompt size)."""
    if _ai_diag_jobs.get("current") and not _ai_diag_jobs["current"].get("finished"):
        return jsonify({"ok": False, "error": "A diagnostics run is already in progress — watch the results panel."}), 409
    job = {"finished": False, "started_at": now_utc(), "probes": [], "log": [], "notes": [],
           "provider": "", "model": "", "thinking_fallbacks": {}}
    _ai_diag_jobs["current"] = job

    def _run():
        with app.app_context():
            try:
                _run_ai_diagnostics(job)
            except Exception as e:
                job["notes"].append(f"Diagnostics crashed: {type(e).__name__}: {e}")
            finally:
                job["finished"] = True

    threading.Thread(target=_run, daemon=True).start()
    return jsonify({"ok": True})


@app.route(f"{ADMIN_URL_PREFIX}/ai/diagnostics/status")
@login_required
def admin_ai_diagnostics_status():
    return jsonify(_ai_diag_jobs.get("current") or {"finished": True})


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/<int:cid>/link/<int:link_id>/generate-text", methods=("POST",))
@login_required
def admin_lead_campaign_generate_text(cid, link_id):
    """Generate ONLY the SMS text for a lead — never touches the email reply
    (safe for drafted and already-sent leads)."""
    try:
        db = get_db()
        link = db.execute(
            "SELECT * FROM campaign_links WHERE id=? AND campaign_id=?", (link_id, cid)
        ).fetchone()
        if not link:
            return jsonify({"ok": False, "error": "Link not found."}), 404
        if not (link["lead_phone"] or "").strip():
            return jsonify({"ok": False, "error": "Lead has no phone number."}), 400
        campaign = db.execute("SELECT * FROM lead_campaigns WHERE id=?", (cid,)).fetchone()
        listing = db.execute("SELECT * FROM listings WHERE id=?", (campaign["listing_id"],)).fetchone() if campaign else None
        if not listing:
            return jsonify({"ok": False, "error": "Vehicle not found."}), 404
        settings = get_settings()
        status, ai_adapter = _lead_reply_ai_adapter(settings)
        if not status["active_configured"]:
            return jsonify({"ok": False, "error": f"{status['provider'].title()} API key is missing."}), 400
        ctx = _lead_reply_context(listing, settings, request.host_url, link)
        sms_url = f"{request.host_url}r/{link['sms_token']}"
        result = lead_replies.generate_sms_text(
            ctx["input_data"], sms_url, lead_dispatch.sms_style_hint(link["id"]), ai_adapter,
        )
        if result.get("status") == "error":
            return jsonify({"ok": False, "error": result.get("error", "Unknown error")}), 400
        db.execute("UPDATE campaign_links SET text_message=? WHERE id=?", (result["text_message"], link["id"]))
        db.commit()
        return jsonify({"ok": True, "text_message": result["text_message"]})
    except Exception as error:
        return jsonify({"ok": False, "error": str(error)}), 500


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/schedule", methods=("POST",))
@login_required
def admin_lead_campaigns_schedule():
    """Queue selected links for sending. mode=now sends ASAP; mode=schedule uses
    hours/minutes from now. Sends are staggered 60-180s apart. The tracking link is
    appended here (request context) because the send daemon has none."""
    db = get_db()
    cid = (request.form.get("cid") or "").strip()
    scope = (request.form.get("scope") or "").strip()
    ids = [int(x) for x in request.form.getlist("link_ids") if x.isdigit()]
    back = url_for("admin_lead_campaign_detail", cid=int(cid)) if cid.isdigit() else url_for("admin_lead_campaigns")
    if scope != "all_drafted" and not ids:
        flash("Select at least one lead first.", "warning")
        return redirect(back)
    mode = (request.form.get("mode") or "schedule").strip()
    try:
        hours = max(0, min(720, int(request.form.get("hours") or 0)))
    except ValueError:
        hours = 0
    try:
        minutes = max(0, min(59, int(request.form.get("minutes") or 0)))
    except ValueError:
        minutes = 0
    if mode == "now":
        hours = minutes = 0
    base = datetime.utcnow() + timedelta(hours=hours, minutes=minutes)
    if scope == "all_drafted":
        links = db.execute(
            "SELECT cl.* FROM campaign_links cl JOIN lead_campaigns lc ON lc.id = cl.campaign_id "
            "WHERE cl.reply_status = 'drafted' AND COALESCE(cl.duplicate,0) = 0 ORDER BY cl.id"
        ).fetchall()
    else:
        placeholders = ",".join("?" for _ in ids)
        links = db.execute(f"SELECT * FROM campaign_links WHERE id IN ({placeholders})", ids).fetchall()
    valid = [
        l for l in links
        if (l["lead_email"] or "").strip()
        and (l["generated_reply"] or "").strip()
        and (l["reply_status"] or "") not in ("sent", "sending")
    ]
    times = lead_dispatch.staggered_schedule_times(len(valid), base)
    for link, when in zip(valid, times):
        body = link["generated_reply"]
        tracking_url = f"{request.host_url}r/{link['token']}"
        if tracking_url not in body:
            body = body.rstrip() + f"\n\nView the vehicle here: {tracking_url}"
        db.execute(
            "UPDATE campaign_links SET generated_reply=?, reply_status='scheduled', scheduled_at=? WHERE id=?",
            (body, when.isoformat(timespec="seconds"), link["id"]),
        )
    db.commit()
    skipped = (len(links) if scope == "all_drafted" else len(ids)) - len(valid)
    when_label = "now (staggered 1-3 min apart)" if mode == "now" else f"in {hours}h {minutes}m (staggered)"
    flash(f"Queued {len(valid)} repl{'y' if len(valid) == 1 else 'ies'} to send {when_label}."
          + (f" Skipped {skipped} (need an email + a reply, not already sent)." if skipped else ""), "success")
    return redirect(back)


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/follow-up", methods=("GET", "POST"))
@login_required
def admin_lead_followups():
    """Leads whose email was sent but never clicked — manual SMS follow-up list."""
    db = get_db()
    try:
        hours = int(request.values.get("hours", 24) or 24)
    except ValueError:
        hours = 24
    if hours not in (1, 2, 4, 8, 12, 24, 48, 72):
        hours = 24
    if request.method == "POST":
        if (request.form.get("fu_action") or "") == "toggle_texted" and (request.form.get("link_id") or "").isdigit():
            lid = int(request.form.get("link_id"))
            row = db.execute("SELECT texted_at FROM campaign_links WHERE id=?", (lid,)).fetchone()
            if row:
                db.execute("UPDATE campaign_links SET texted_at=? WHERE id=?",
                           (None if row["texted_at"] else now_utc(), lid))
                db.commit()
                flash("Texted mark removed." if row["texted_at"] else "Marked as texted — hidden from this list.", "success")
        return redirect(url_for("admin_lead_followups", hours=hours))
    cutoff = (datetime.utcnow() - timedelta(hours=hours)).isoformat(timespec="seconds")
    rows = db.execute(
        """SELECT cl.*, l.year, l.make, l.model, l.trim, l.price
           FROM campaign_links cl
           JOIN lead_campaigns lc ON lc.id = cl.campaign_id
           JOIN listings l ON l.id = lc.listing_id
           WHERE cl.reply_status = 'sent' AND COALESCE(cl.visited,0) = 0 AND COALESCE(cl.sms_visited,0) = 0
             AND cl.sent_at IS NOT NULL AND cl.sent_at <= ?
             AND COALESCE(cl.lead_phone,'') != '' AND COALESCE(cl.duplicate,0) = 0
             AND cl.texted_at IS NULL
           ORDER BY cl.sent_at""",
        (cutoff,),
    ).fetchall()
    enriched = []
    for r in rows:
        try:
            price_str = "${:,}".format(int(float(str(r["price"]).replace(",", ""))))
        except (ValueError, TypeError):
            price_str = ""
        car_line = f"{r['year']} {r['make']} {r['model']} {r['trim'] or ''} {price_str}".strip()
        enriched.append({"row": r, "car_line": car_line})
    return render_template("admin/lead_followups.html", rows=enriched, hours=hours)


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/all-leads")
@login_required
def admin_lead_all():
    """All loaded leads, newest first — quick lookup for when a customer calls.
    Car / price / name / email / phone at a glance, plus a popup with their
    message and our reply. Row limit selectable (50/100/200, default 100)."""
    db = get_db()
    try:
        limit = int(request.args.get("limit", 100) or 100)
    except ValueError:
        limit = 100
    if limit not in (50, 100, 200):
        limit = 100
    rows = db.execute(
        """SELECT cl.*, l.year, l.make, l.model, l.trim, l.price
           FROM campaign_links cl
           JOIN lead_campaigns lc ON lc.id = cl.campaign_id
           LEFT JOIN listings l ON l.id = lc.listing_id
           ORDER BY cl.created_at DESC, cl.id DESC
           LIMIT ?""",
        (limit,),
    ).fetchall()
    enriched = []
    for r in rows:
        if r["make"]:
            try:
                price_str = "${:,}".format(int(float(str(r["price"]).replace(",", ""))))
            except (ValueError, TypeError):
                price_str = ""
            car = f"{r['year']} {r['make']} {r['model']} {r['trim'] or ''}".strip()
        else:
            car = (r["notes"] or "Unassigned").strip()
            price_str = ""
        enriched.append({"row": r, "car": car, "price": price_str})
    return render_template("admin/lead_all.html", rows=enriched, limit=limit)


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/export-voice-csv", methods=("POST",))
@login_required
def admin_lead_export_voice_csv():
    """Export selected leads (or all phone-bearing leads) to a CSV whose columns match
    the famulor.io AI voice-campaign lead-add form. Only leads with a phone + a
    vehicle are included (the dialer requires a phone)."""
    db = get_db()
    mode = (request.form.get("mode") or "selected").strip()
    ids = [int(x) for x in request.form.getlist("link_ids") if x.isdigit()]
    if mode != "all" and not ids:
        flash("Select at least one lead to export (or use Export all).", "warning")
        return redirect(url_for("admin_lead_all"))
    if mode == "all":
        links = db.execute(
            "SELECT cl.* FROM campaign_links cl JOIN lead_campaigns lc ON lc.id = cl.campaign_id "
            "WHERE lc.listing_id IS NOT NULL AND COALESCE(cl.lead_phone, '') != '' "
            "ORDER BY cl.id DESC"
        ).fetchall()
    else:
        placeholders = ",".join("?" for _ in ids)
        links = db.execute(f"SELECT * FROM campaign_links WHERE id IN ({placeholders})", ids).fetchall()
    settings = get_settings()
    host = request.host_url
    import csv as _csv
    import io as _io
    buf = _io.StringIO()
    writer = _csv.writer(buf)
    writer.writerow(VOICE_CSV_HEADERS)
    count = 0
    for link in links:
        camp = db.execute("SELECT listing_id FROM lead_campaigns WHERE id=?", (link["campaign_id"],)).fetchone()
        if not camp or not camp["listing_id"]:
            continue  # unassigned lead — no vehicle to call about
        listing = db.execute("SELECT * FROM listings WHERE id=?", (camp["listing_id"],)).fetchone()
        if not listing:
            continue
        row = _voice_csv_row(link, listing, settings, host)
        writer.writerow([row[h] for h in VOICE_CSV_HEADERS])
        count += 1
    if count == 0:
        flash("No exportable leads found — each needs a phone number and an assigned vehicle.", "warning")
        return redirect(url_for("admin_lead_all"))
    fname = f"voice_campaign_leads_{now_utc()[:10]}.csv"
    return buf.getvalue(), 200, {
        "Content-Type": "text/csv; charset=utf-8",
        "Content-Disposition": f"attachment; filename={fname}",
    }


# ---------------- Inbox (auto-replier: IMAP pull, match leads, generate follow-up replies) ----------------

_inbox_daemon_started = False


def start_inbox_daemon():
    """Start the autonomous inbox daemon (polls IMAP every N minutes)."""
    global _inbox_daemon_started
    if _inbox_daemon_started:
        return
    _inbox_daemon_started = True

    def _loop():
        import time as _time
        while True:
            try:
                with app.app_context():
                    settings = get_settings()
                    if settings_value(settings, "inbox_autonomous_enabled", 0):
                        _inbox_process_all(settings)
            except Exception as e:
                logging.getLogger(__name__).warning(f"inbox daemon tick failed: {e}")
            try:
                with app.app_context():
                    interval = int(settings_value(get_settings(), "inbox_autonomous_interval", 30) or 30)
            except Exception:
                interval = 30
            _time.sleep(max(60, interval * 60))

    threading.Thread(target=_loop, daemon=True).start()


def _inbox_process_all(settings):
    """Pull inbox, match, generate replies. Auto-send if enabled + 10 min passed.
    Uses INSERT OR IGNORE for safe dedup (handles empty/duplicate Message-IDs)."""
    db = get_db()
    emails = inbox_reader.fetch_inbox(settings, limit=50)
    _inbox_pull_jobs["current"]["total"] = len(emails)
    master_prompt = settings_value(settings, "inbox_master_prompt", "") or ""
    auto_send = bool(settings_value(settings, "inbox_auto_send", 0))
    fstatus, ai_call = make_text_adapter(settings, "lead_reply")
    for em in emails:
        if not em["message_id"]:
            continue
        existing = db.execute("SELECT id FROM inbox_messages WHERE message_id=?", (em["message_id"],)).fetchone()
        if existing:
            _inbox_pull_jobs["current"]["skipped"] = (_inbox_pull_jobs["current"].get("skipped", 0)) + 1
            _inbox_pull_jobs["current"]["done"] += 1
            continue
        lead = inbox_reader.match_lead(db, em["from_email"])
        # Generate reply
        generated = ""
        if fstatus["active_configured"] and lead:
            try:
                prompt = inbox_reader.build_followup_prompt(master_prompt, lead, em["body_text"], em["subject"])
                raw = (ai_call(prompt) or "").strip()
                try:
                    cleaned = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.I).strip()
                    cleaned = re.sub(r"\s*```$", "", cleaned).strip()
                    start = cleaned.find("{")
                    if start >= 0:
                        obj = json.loads(cleaned[start:])
                        generated = (obj.get("body") or "").strip()
                except Exception:
                    generated = raw
            except Exception:
                generated = ""
        db.execute(
            "INSERT OR IGNORE INTO inbox_messages (message_id, from_email, from_name, subject, body_text, received_at, received_at_iso, pulled_at, "
            "lead_link_id, lead_email, car_line, lead_message, our_reply, generated_reply, reply_status, processed) "
            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1)",
            (em["message_id"], em["from_email"], em["from_name"], em["subject"], em["body_text"],
             em["received_at"], em.get("received_at_iso", ""), now_utc(),
             lead["link_id"] if lead else None, em["from_email"],
             lead["car_line"] if lead else "", lead["lead_message"] if lead else "",
             lead["our_reply"] if lead else "", generated,
             "drafted" if generated else "no_reply",),
        )
        db.commit()
        _inbox_pull_jobs["current"]["new"] = (_inbox_pull_jobs["current"].get("new", 0)) + 1
        _inbox_pull_jobs["current"]["done"] += 1
        # Auto-send if enabled + 10 min passed
        if auto_send and generated and lead:
            _inbox_try_auto_send(db, em, generated, settings)


def _inbox_try_auto_send(db, em, generated, settings):
    """Send the reply if 10+ min have passed since the email was received."""
    try:
        from email.utils import parsedate_to_datetime
        received = parsedate_to_datetime(em["received_at"])
        if received and (datetime.utcnow() - received.replace(tzinfo=None)).total_seconds() >= 600:
            subject = em["subject"]
            if not subject.lower().startswith("re:"):
                subject = "Re: " + subject
            ok, msg = smtp_send_message(settings, subject, generated, None, recipient=em["from_email"])
            if ok:
                db.execute("UPDATE inbox_messages SET reply_status='sent', reply_sent_at=? WHERE message_id=?",
                           (now_utc(), em["message_id"]))
                db.commit()
    except Exception as e:
        logging.getLogger(__name__).warning(f"inbox auto-send failed: {e}")


@app.route(f"{ADMIN_URL_PREFIX}/inbox")
@login_required
def admin_inbox():
    db = get_db()
    messages = db.execute(
        "SELECT * FROM inbox_messages WHERE COALESCE(archived,0)=0 ORDER BY received_at_iso DESC NULLS LAST, id DESC LIMIT 200"
    ).fetchall()
    settings = get_settings()
    master_prompt = settings_value(settings, "inbox_master_prompt", "") or ""
    auto_send = bool(settings_value(settings, "inbox_auto_send", 0))
    autonomous = bool(settings_value(settings, "inbox_autonomous_enabled", 0))
    interval = int(settings_value(settings, "inbox_autonomous_interval", 30) or 30)
    pull_job = _inbox_pull_jobs.get("current", {})
    return render_template("admin/inbox.html", messages=messages, master_prompt=master_prompt,
                           auto_send=auto_send, autonomous=autonomous, interval=interval, pull_job=pull_job)


_inbox_pull_jobs = {}


@app.route(f"{ADMIN_URL_PREFIX}/inbox/pull", methods=("POST",))
@login_required
def admin_inbox_pull():
    job = _inbox_pull_jobs.get("current")
    if job and not job.get("finished"):
        flash("A pull is already running — refresh in a moment to see results.", "warning")
        return redirect(url_for("admin_inbox"))
    _inbox_pull_jobs["current"] = {"finished": False, "total": 0, "done": 0, "new": 0, "skipped": 0, "error": None}
    settings_snapshot = get_settings()

    def _wrapped():
        with app.app_context():
            try:
                _inbox_process_all(settings_snapshot)
                _inbox_pull_jobs["current"]["finished"] = True
            except Exception as e:
                _inbox_pull_jobs["current"]["finished"] = True
                _inbox_pull_jobs["current"]["error"] = str(e)

    threading.Thread(target=_wrapped, daemon=True).start()
    flash("Pulling inbox in background — the page will auto-update with progress.", "success")
    return redirect(url_for("admin_inbox"))


@app.route(f"{ADMIN_URL_PREFIX}/inbox/pull-status")
@login_required
def admin_inbox_pull_status():
    job = _inbox_pull_jobs.get("current", {})
    return jsonify(job)


@app.route(f"{ADMIN_URL_PREFIX}/inbox/test-imap", methods=("POST",))
@login_required
def admin_inbox_test_imap():
    """One-click IMAP diagnostic: connect, login, count UNSEEN + ALL in INBOX. Tells
    you exactly why the pull finds nothing (auth fail, wrong host, or 0 unread)."""
    res = inbox_reader.test_connection(get_settings())
    if res.get("ok"):
        flash(f"IMAP OK — connected to INBOX: {res['unread']} unread, {res['total']} total. (The pull only imports UNREAD messages — mark some unread in webmail if it shows 0 unread.)", "success")
    else:
        flash(f"IMAP failed: {res.get('error')}", "warning")
    return redirect(url_for("admin_inbox"))


def _inbox_composer_thread(db, link_id):
    """Build the full lead-conversation thread text for the Reply Composer input box:
    contact info, the car inquired, the original inquiry, our previous email reply,
    our SMS text, and all incoming emails from the lead. The admin pastes the new
    incoming message at the bottom. This is the 'dump whole thread into the input'."""
    link = db.execute(
        "SELECT cl.*, l.year, l.make, l.model, l.trim, l.price "
        "FROM campaign_links cl JOIN lead_campaigns lc ON lc.id=cl.campaign_id "
        "LEFT JOIN listings l ON l.id=lc.listing_id WHERE cl.id=?", (link_id,)).fetchone()
    if not link:
        return ""
    lines = []
    name = (link["lead_name"] or "").strip()
    email = (link["lead_email"] or "").strip()
    phone = (link["lead_phone"] or "").strip()
    contact = name or email or "(no name)"
    if email:
        contact += f" <{email}>"
    if phone:
        contact += f" · {phone}"
    lines.append(f"LEAD: {contact}")
    if link["make"]:
        try:
            price_str = "${:,}".format(int(float(str(link["price"]).replace(",", ""))))
        except (ValueError, TypeError):
            price_str = ""
        car = f"{link['year']} {link['make']} {link['model']} {link['trim'] or ''}".strip()
        lines.append(f"VEHICLE: {car}" + (f" — {price_str}" if price_str else ""))
    if (link["lead_message"] or "").strip():
        lines.append(f"ORIGINAL INQUIRY:\n{link['lead_message'].strip()}")
    if (link["generated_reply"] or "").strip():
        lines.append(f"OUR PREVIOUS EMAIL REPLY:\n{link['generated_reply'].strip()}")
    if "text_message" in link.keys() and (link["text_message"] or "").strip():
        lines.append(f"SMS TEXT WE SENT:\n{link['text_message'].strip()}")
    incoming = db.execute(
        "SELECT subject, body_text, received_at FROM inbox_messages "
        "WHERE LOWER(COALESCE(from_email,''))=LOWER(?) "
        "ORDER BY received_at_iso ASC NULLS LAST, id ASC", (email,)).fetchall() if email else []
    if incoming:
        lines.append("INCOMING EMAILS FROM LEAD:")
        for m in incoming:
            dt = (m["received_at"] or "")[:16]
            lines.append(f"  [{dt}] {m['subject'] or '(no subject)'}\n  {m['body_text'] or ''}\n  ---")
    lines.append("\n=== PASTE THE NEW INCOMING MESSAGE (email or text) BELOW THIS LINE ===")
    return "\n\n".join(lines)


@app.route(f"{ADMIN_URL_PREFIX}/inbox/composer/search")
@login_required
def admin_inbox_composer_search():
    """Search leads by name / email / phone for the Reply Composer selector."""
    q = (request.args.get("q") or "").strip().lower()
    if len(q) < 2:
        return jsonify([])
    db = get_db()
    like = f"%{q}%"
    q_digits = re.sub(r"\D", "", q)
    params = [like, like]
    phone_clause = ""
    if q_digits:
        phone_clause = " OR REPLACE(REPLACE(REPLACE(REPLACE(COALESCE(cl.lead_phone,''),' ',''),'-',''),'(',''),')','') LIKE ?"
        params.append(f"%{q_digits}%")
    rows = db.execute(
        "SELECT cl.id, cl.lead_name, cl.lead_email, cl.lead_phone, l.year, l.make, l.model "
        "FROM campaign_links cl JOIN lead_campaigns lc ON lc.id=cl.campaign_id "
        "LEFT JOIN listings l ON l.id=lc.listing_id "
        f"WHERE (LOWER(COALESCE(cl.lead_name,'')) LIKE ? OR LOWER(COALESCE(cl.lead_email,'')) LIKE ?{phone_clause}) "
        "ORDER BY cl.id DESC LIMIT 20", params).fetchall()
    return jsonify([{"id": r["id"],
                      "label": f"{r['lead_name'] or '—'} · {r['lead_email'] or '—'} · {r['lead_phone'] or '—'} · {r['year'] or ''} {r['make'] or ''} {r['model'] or ''}".strip()}
                     for r in rows])


@app.route(f"{ADMIN_URL_PREFIX}/inbox/composer/load")
@login_required
def admin_inbox_composer_load():
    """Load a lead's full thread + saved draft (input/comments/output) into the
    composer. First load (no draft): input = the fresh thread. Subsequent loads:
    input = the saved draft (preserves the rep's edits)."""
    lead_id = (request.args.get("lead_id") or "").strip()
    if not lead_id.isdigit():
        return jsonify({"thread": "", "input": "", "comments": "", "output": ""})
    db = get_db()
    thread = _inbox_composer_thread(db, int(lead_id))
    draft = db.execute("SELECT input_text, comments, output_text FROM inbox_composer_drafts WHERE lead_id=?", (int(lead_id),)).fetchone()
    if draft:
        return jsonify({"thread": thread, "input": draft["input_text"] or "", "comments": draft["comments"] or "", "output": draft["output_text"] or ""})
    return jsonify({"thread": thread, "input": thread, "comments": "", "output": ""})


@app.route(f"{ADMIN_URL_PREFIX}/inbox/composer/generate", methods=("POST",))
@login_required
def admin_inbox_composer_generate():
    """Generate an AI reply from the inbox master prompt + the input thread + comments."""
    try:
        db = get_db()
        settings = get_settings()
        master_prompt = settings_value(settings, "inbox_master_prompt", "") or ""
        input_text = (request.form.get("input") or "").strip()
        comments = (request.form.get("comments") or "").strip()
        if not input_text:
            return jsonify({"ok": False, "error": "Paste the incoming message / context into the input box first."}), 400
        fstatus, ai_call = make_text_adapter(settings, "lead_reply")
        if not fstatus["active_configured"]:
            return jsonify({"ok": False, "error": f"{fstatus.get('provider', 'AI')} API key is not configured."}), 400
        prompt = (master_prompt or "You are a helpful sales representative writing a reply to a vehicle lead.") + "\n\nFULL CONVERSATION CONTEXT (lead thread + new incoming message):\n" + input_text
        if comments:
            prompt += "\n\nADDITIONAL INSTRUCTIONS FROM THE REP:\n" + comments
        prompt += '\n\nWrite a natural reply to the lead\'s newest message, using the full context above. Reuse the subject of the lead\'s email (with Re: if not present). Keep it concise and human. Return JSON only: {"subject": "the subject", "body": "the reply body"}'
        raw = (ai_call(prompt) or "").strip()
        body = raw
        subject = ""
        try:
            cleaned = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.I).strip()
            cleaned = re.sub(r"\s*```$", "", cleaned).strip()
            start = cleaned.find("{")
            if start >= 0:
                obj = json.loads(cleaned[start:])
                body = (obj.get("body") or "").strip()
                subject = (obj.get("subject") or "").strip()
        except Exception:
            pass
        return jsonify({"ok": True, "output": body, "subject": subject})
    except Exception as e:
        logging.getLogger(__name__).exception("inbox composer generate crashed")
        return jsonify({"ok": False, "error": f"{type(e).__name__}: {e}"}), 500


@app.route(f"{ADMIN_URL_PREFIX}/inbox/composer/save", methods=("POST",))
@login_required
def admin_inbox_composer_save():
    """Persist the composer draft (input + comments + output) for a lead."""
    db = get_db()
    lead_id = (request.form.get("lead_id") or "").strip()
    if not lead_id.isdigit():
        return jsonify({"ok": False, "error": "Select a lead first."}), 400
    db.execute(
        "INSERT OR REPLACE INTO inbox_composer_drafts (lead_id, input_text, comments, output_text, updated_at) VALUES (?,?,?,?,?)",
        (int(lead_id), request.form.get("input") or "", request.form.get("comments") or "", request.form.get("output") or "", now_utc()))
    db.commit()
    return jsonify({"ok": True})


@app.route(f"{ADMIN_URL_PREFIX}/inbox/save-settings", methods=("POST",))
@login_required
def admin_inbox_save_settings():
    db = get_db()
    master_prompt = (request.form.get("inbox_master_prompt") or "").strip()
    auto_send = 1 if request.form.get("inbox_auto_send") else 0
    autonomous = 1 if request.form.get("inbox_autonomous_enabled") else 0
    try:
        interval = max(5, min(1440, int(request.form.get("inbox_autonomous_interval") or 30)))
    except ValueError:
        interval = 30
    db.execute(
        "UPDATE site_settings SET inbox_master_prompt=?, inbox_auto_send=?, inbox_autonomous_enabled=?, inbox_autonomous_interval=? WHERE id=1",
        (master_prompt, auto_send, autonomous, interval),
    )
    db.commit()
    flash("Inbox settings saved.", "success")
    return redirect(url_for("admin_inbox"))


@app.route(f"{ADMIN_URL_PREFIX}/inbox/archive", methods=("POST",))
@login_required
def admin_inbox_archive():
    """Archive one or many messages."""
    db = get_db()
    msg_id = (request.form.get("msg_id") or "").strip()
    if msg_id.isdigit():
        db.execute("UPDATE inbox_messages SET archived=1 WHERE id=?", (int(msg_id),))
    # Bulk archive via checkboxes
    for mid in request.form.getlist("msg_ids"):
        if mid.isdigit():
            db.execute("UPDATE inbox_messages SET archived=1 WHERE id=?", (int(mid),))
    db.commit()
    return redirect(url_for("admin_inbox"))


@app.route(f"{ADMIN_URL_PREFIX}/inbox/<int:msg_id>/send", methods=("POST",))
@login_required
def admin_inbox_send(msg_id):
    db = get_db()
    msg = db.execute("SELECT * FROM inbox_messages WHERE id=?", (msg_id,)).fetchone()
    if not msg:
        abort(404)
    body = (request.form.get("generated_reply") or msg["generated_reply"] or "").strip()
    if not body:
        flash("No reply text to send.", "warning")
        return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")
    if not msg["from_email"]:
        flash("No recipient email.", "warning")
        return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")
    subject = msg["subject"] or "Re: your enquiry"
    if not subject.lower().startswith("re:"):
        subject = "Re: " + subject
    try:
        settings = get_settings()
        ok, errmsg = smtp_send_message(settings, subject, body, None, recipient=msg["from_email"])
    except Exception as e:
        logging.getLogger(__name__).exception(f"inbox send msg {msg_id} crashed")
        flash(f"Send failed (unexpected): {type(e).__name__}: {e}", "warning")
        return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")
    if ok:
        db.execute("UPDATE inbox_messages SET generated_reply=?, reply_status='sent', reply_sent_at=? WHERE id=?",
                   (body, now_utc(), msg_id))
        db.commit()
        flash("Reply sent.", "success")
    else:
        flash(f"Send failed: {errmsg}", "warning")
    return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")


@app.route(f"{ADMIN_URL_PREFIX}/inbox/<int:msg_id>/regenerate", methods=("POST",))
@login_required
def admin_inbox_regenerate(msg_id):
    db = get_db()
    msg = db.execute("SELECT * FROM inbox_messages WHERE id=?", (msg_id,)).fetchone()
    if not msg:
        abort(404)
    settings = get_settings()
    try:
        fstatus, ai_call = make_text_adapter(settings, "lead_reply")
        if not fstatus["active_configured"]:
            flash(f"{fstatus['provider'].title()} API key is missing.", "warning")
            return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")
        lead = None
        if msg["lead_link_id"]:
            lead = {"car_line": msg["car_line"], "lead_name": msg["from_name"],
                    "lead_message": msg["lead_message"], "our_reply": msg["our_reply"]}
        master_prompt = settings_value(settings, "inbox_master_prompt", "") or ""
        extra_instructions = (request.form.get("extra_instructions") or "").strip()
        prompt = inbox_reader.build_followup_prompt(master_prompt, lead, msg["body_text"], msg["subject"])
        if extra_instructions:
            prompt += f"\n\nADDITIONAL INSTRUCTIONS FROM THE USER:\n{extra_instructions}\n"
        raw = (ai_call(prompt) or "").strip()
        try:
            cleaned = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.I).strip()
            cleaned = re.sub(r"\s*```$", "", cleaned).strip()
            start = cleaned.find("{")
            if start >= 0:
                obj = json.loads(cleaned[start:])
                generated = (obj.get("body") or "").strip()
            else:
                generated = raw
        except Exception:
            generated = raw
    except Exception as e:
        logging.getLogger(__name__).exception(f"inbox regenerate msg {msg_id} crashed")
        flash(f"Generation failed: {type(e).__name__}: {e}", "warning")
        return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")
    db.execute("UPDATE inbox_messages SET generated_reply=?, reply_status='drafted' WHERE id=?",
               (generated, msg_id))
    db.commit()
    flash("Reply regenerated.", "success")
    return redirect(url_for("admin_inbox") + f"#msg-{msg_id}")


# ---------------- Turbo Listing (AutoTrader.ca listing builder + submitter) ----------------

_turbo_send_locks = {}


def _turbo_description_prompt(listing):
    """Build the AI description prompt for a listing. Pass the VERIFIED specs so the AI
    describes THIS car (not a brochure), and forbid invented location/equipment/history
    so it reads like a factual dealer listing — neutral, no city spam, no ad copy."""
    def g(name):
        try:
            v = listing[name] if name in listing.keys() else None
        except (KeyError, AttributeError, TypeError):
            v = None
        return (str(v).strip() if v not in (None, "") else "")
    year, make, model = g("year"), g("make"), g("model")
    trim = g("trim")
    odo = g("odometer")
    body = g("body_style"); fuel = g("fuel_type"); drive = g("drive_line")
    trans = g("transmission"); doors = g("doors"); color = g("exterior_color")
    stock = g("public_stock_number") or g("stock_number")
    facts_lines = [f"Vehicle: {year} {make} {model} {trim}".strip()]
    if odo:
        facts_lines.append(f"Mileage: {odo} km")
    if body:
        facts_lines.append(f"Body style: {body}")
    if fuel:
        facts_lines.append(f"Fuel: {fuel}")
    if drive:
        facts_lines.append(f"Drivetrain: {drive}")
    if trans:
        facts_lines.append(f"Transmission: {trans}")
    if doors:
        facts_lines.append(f"Doors: {doors}")
    if color:
        facts_lines.append(f"Exterior color: {color}")
    if stock:
        facts_lines.append(f"Reference: {stock}")
    facts = "\n".join(facts_lines)
    return (
        "Write a factual used-vehicle listing description like a reputable Canadian dealer would publish "
        "(think Clutch / Carvana vehicle pages — informative, dry, trustworthy — NOT a magazine ad or brochure).\n\n"
        f"{facts}\n\n"
        "STRUCTURE: 2-3 short paragraphs, 4-7 sentences total.\n"
        "Paragraph 1: what the vehicle is — year, make, model, trim, body style, powertrain (engine/fuel/drivetrain/transmission "
        "ONLY if supplied above), and mileage. State these as facts.\n"
        "Paragraph 2 (optional): notable verified features if the facts above list them. Do NOT invent features, packages, "
        "trim names, or equipment that is not in the facts above.\n"
        "Last line: a neutral, low-pressure invitation to view or inquire (no urgency, no 'today', no 'don't miss').\n\n"
        "HARD RULES:\n"
        "- Do NOT mention any city, province, region, or location (no 'Toronto', 'in your area', 'across Canada', etc.). The location is not provided and must not be invented.\n"
        "- Do NOT invent any specification, feature, package, trim detail, or equipment not listed in the facts above.\n"
        "- Do NOT use marketing superlatives ('stunning', 'exceptional', 'iconic', 'rare blend', 'ultimate', 'unleash', 'elevate'). Dry, factual tone.\n"
        "- Do NOT claim accident-free, one-owner, clean title, full service history, warranty, certified, or inspection status unless the facts above state it.\n"
        "- Do NOT mention dealer, dealership, sales, financing, delivery, warranty, private seller/owner, or any commercial terms.\n"
        "- Do NOT use dash characters (no '-' '–' '—'); reword instead.\n"
        "- No bullet points, no exclamation marks, no questions.\n"
        "- Canadian English, CAD, kilometres.\n"
        'Return JSON only: {"description": "the description text"}.'
    )


def _campaign_get_description(listing, settings):
    """AI description for a listing, cached in turbo_descriptions by (listing_id,
    input_hash). Re-posting the same car across batches reuses the description instead
    of a fresh ~20-30s Fireworks call each time. A change to year/make/model/trim/odo
    changes the hash → regenerates. Returns '' if no AI provider is configured."""
    key_fields = "|".join(str(x or "") for x in (
        listing["id"], listing["year"], listing["make"], listing["model"],
        listing["trim"] if "trim" in listing.keys() else "", listing["odometer"]))
    input_hash = hashlib.md5(key_fields.encode("utf-8")).hexdigest()
    db = get_db()
    row = db.execute(
        "SELECT description FROM turbo_descriptions WHERE listing_id=? AND input_hash=?",
        (listing["id"], input_hash)).fetchone()
    if row and row["description"]:
        return row["description"]
    fstatus, ai_call = make_text_adapter(settings, "descriptions")
    if not fstatus["active_configured"]:
        return ""
    raw = (ai_call(_turbo_description_prompt(listing)) or "").strip()
    desc = ""
    try:
        cleaned = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.I).strip()
        start = cleaned.find("{")
        if start >= 0:
            desc = (json.loads(cleaned[start:]).get("description") or "").strip()
    except Exception:
        desc = raw
    try:
        db.execute(
            "INSERT OR REPLACE INTO turbo_descriptions (listing_id, input_hash, description, generated_at) "
            "VALUES (?,?,?,?)",
            (listing["id"], input_hash, desc, now_utc()))
        db.commit()
    except Exception:
        pass
    return desc


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing")
@login_required
def admin_turbo_listing():
    db = get_db()
    listings = db.execute(
        "SELECT id, slug, title_en, year, make, model, trim, body_style, odometer, price, vin, "
        "stock_number, public_stock_number, transmission, drive_line, fuel_type, doors "
        "FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY year DESC, make, model"
    ).fetchall()
    images = {}
    for l in listings:
        img = db.execute(
            "SELECT image_url FROM listing_images WHERE listing_id=? ORDER BY is_primary DESC, id LIMIT 1",
            (l["id"],)).fetchone()
        images[l["id"]] = img["image_url"] if img else ""
    profiles = db.execute("SELECT id, name, created_at FROM turbo_session_profiles ORDER BY name").fetchall()
    history = db.execute("SELECT * FROM turbo_history ORDER BY id DESC LIMIT 50").fetchall()
    # Load stored payloads so they survive page refresh.
    payloads = {}
    for pr in db.execute("SELECT listing_id, payload_json FROM turbo_payloads").fetchall():
        try:
            payloads[pr["listing_id"]] = json.dumps(json.loads(pr["payload_json"]), indent=2)
        except Exception:
            payloads[pr["listing_id"]] = pr["payload_json"] or ""
    # Cities grouped by province for the dropdowns
    cities_by_province = {}
    for c in db.execute("SELECT province, city FROM turbo_cities ORDER BY province, city").fetchall():
        cities_by_province.setdefault(c["province"], []).append(c["city"])
    # Tracking data — unified post tracking (every post: campaign, manual, external)
    posted = db.execute(
        "SELECT tcp.*, l.title_en, l.year, l.make, l.model, l.trim, tsp.email AS account_email "
        "FROM turbo_campaign_posts tcp "
        "LEFT JOIN listings l ON l.id = tcp.listing_id "
        "LEFT JOIN turbo_session_profiles tsp ON tsp.id = tcp.profile_id "
        "ORDER BY tcp.id DESC LIMIT 100"
    ).fetchall()
    return render_template("admin/turbo_listing.html", listings=listings, images=images,
                           profiles=profiles, history=history, payloads=payloads,
                           cities_by_province=cities_by_province, posted=posted)


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/parse-session", methods=("POST",))
@login_required
def admin_turbo_parse_session():
    try:
        parsed = turbo_listing.parse_powershell_request(request.form.get("powershell", ""))
        return jsonify({"ok": True, "method": parsed["method"], "url": parsed["url"],
                        "cookie_count": len(parsed["cookies"]), "header_count": len(parsed["headers"]),
                        "user_agent": parsed["user_agent"], "content_type": parsed["content_type"],
                        "has_body": bool(parsed["body"])})
    except Exception as e:
        return jsonify({"ok": False, "error": str(e)}), 400


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/refresh-taxonomy", methods=("POST",))
@login_required
def admin_turbo_refresh_taxonomy():
    db = get_db()
    makes = [r["make"] for r in db.execute(
        "SELECT DISTINCT make FROM listings WHERE COALESCE(is_deleted,0)=0 AND make IS NOT NULL AND make != ''"
    ).fetchall()]
    fetched, errors, now = 0, [], now_utc()
    for make_name in makes:
        make_id = turbo_listing.resolve_make_id(db, make_name)
        if not make_id:
            errors.append(f"No AutoTrader make match for '{make_name}'.")
            continue
        try:
            url = f"{turbo_listing.AUTOTRADER_MODELS_URL}?culture=en-CA&makeId={make_id}&vehicleType=C&marketplace=ca"
            resp = requests.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
            models = turbo_listing.extract_models(resp.json())
            if not models:
                errors.append(f"{make_name}: models endpoint returned no models (kept existing).")
                continue
            db.execute("DELETE FROM turbo_models WHERE make_id=?", (make_id,))
            for m in models:
                mid = m.get("id") or m.get("modelId")
                mname = m.get("name") or m.get("modelName")
                if mid and mname:
                    db.execute("INSERT OR REPLACE INTO turbo_models (id, make_id, name, fetched_at) VALUES (?,?,?,?)",
                               (int(mid), make_id, str(mname), now))
            fetched += 1
        except Exception as e:
            errors.append(f"{make_name}: {type(e).__name__}")
    db.commit()
    return jsonify({"ok": True, "fetched": fetched, "errors": errors})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/generate", methods=("POST",))
@login_required
def admin_turbo_generate():
    db = get_db()
    try:
        listing_id = int(request.form.get("listing_id"))
    except (TypeError, ValueError):
        return jsonify({"ok": False, "error": "Missing vehicle."}), 400
    listing = db.execute("SELECT * FROM listings WHERE id=?", (listing_id,)).fetchone()
    if not listing:
        return jsonify({"ok": False, "error": "Vehicle not found."}), 404
    include_vin = request.form.get("include_vin") == "1"
    vin_warn = ""
    # VIN-cooldown logic (same as the campaign): even if "Include VIN" is checked,
    # omit it when the VIN is hot (recently posted / INACTIVE / excluded) to avoid
    # AutoTrader deactivating the listing. A VIN replacement bypasses this.
    if include_vin and not (request.form.get("vin_replacement", "") or "").strip():
        vin_norm = (listing["vin"] or "").strip().upper()
        if vin_norm:
            safe, reason = _campaign_vin_cooldown_state(db, vin_norm)
            if not safe:
                include_vin = False
                vin_warn = f"VIN omitted — {reason}"
    overrides = {
        "include_vin": include_vin,
        "vin_replacement": request.form.get("vin_replacement", ""),
        "mileage_override": request.form.get("mileage_override", ""),
        "price_override": request.form.get("price_override", ""),
    }
    settings = get_settings()
    description = ""
    try:
        fstatus, ai_call = make_text_adapter(settings, "descriptions")
        if fstatus["active_configured"]:
            raw = (ai_call(_turbo_description_prompt(listing)) or "").strip()
            try:
                cleaned = re.sub(r"^```(?:json)?\s*", "", raw, flags=re.I).strip()
                cleaned = re.sub(r"\s*```$", "", cleaned).strip()
                start = cleaned.find("{")
                if start >= 0:
                    obj = json.loads(cleaned[start:])
                    description = (obj.get("description") or "").strip() if isinstance(obj, dict) else ""
                if not description:
                    description = raw
            except Exception:
                description = raw
    except Exception:
        description = ""
    try:
        payload = turbo_listing.build_payload(listing, overrides, db, description=description)
    except RuntimeError as e:
        return jsonify({"ok": False, "error": str(e)}), 400
    # Store the payload per-vehicle so Send can publish it with any session/profile later.
    db.execute(
        "INSERT INTO turbo_payloads (listing_id, payload_json, generated_at) VALUES (?,?,?) "
        "ON CONFLICT(listing_id) DO UPDATE SET payload_json=excluded.payload_json, generated_at=excluded.generated_at",
        (listing_id, json.dumps(payload), now_utc()),
    )
    vin_mode = "replacement" if overrides["vin_replacement"] else ("inventory" if overrides["include_vin"] else "null")
    db.execute(
        "INSERT INTO turbo_history (listing_id, generated_at, price_used, mileage_used, vin_mode, make_id, model_id, status, created_at) "
        "VALUES (?,?,?,?,?,?,?,?,?)",
        (listing_id, now_utc(), str(payload["prices"]["public"]["price"]),
         str(payload["condition"]["mileage"]), vin_mode, payload["vehicleData"]["make"],
         payload["vehicleData"]["model"], "generated", now_utc()),
    )
    db.commit()
    return jsonify({"ok": True, "payload": payload, "body": json.dumps(payload, indent=2), "vin_warning": vin_warn})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/send", methods=("POST",))
@login_required
def admin_turbo_send():
    db = get_db()
    try:
        listing_id = int(request.form.get("listing_id"))
    except (TypeError, ValueError):
        return jsonify({"ok": False, "error": "Missing vehicle."}), 400
    if _turbo_send_locks.get(listing_id):
        return jsonify({"ok": False, "error": "A submission is already in progress for this vehicle."}), 409
    _turbo_send_locks[listing_id] = True
    try:
        # Payload: use the edited textarea content if provided, else the stored one.
        payload_text = (request.form.get("payload") or "").strip()
        if payload_text:
            try:
                payload = json.loads(payload_text)
            except Exception:
                return jsonify({"ok": False, "error": "Edited payload is not valid JSON."}), 400
        else:
            row = db.execute("SELECT payload_json FROM turbo_payloads WHERE listing_id=?", (listing_id,)).fetchone()
            if not row or not row["payload_json"]:
                return jsonify({"ok": False, "error": "No generated payload for this vehicle — Generate first."}), 400
            try:
                payload = json.loads(row["payload_json"])
            except Exception:
                return jsonify({"ok": False, "error": "Stored payload is corrupt — regenerate."}), 400
        proxy = request.form.get("proxy", "").strip()
        # Session: a selected saved profile, or the pasted PowerShell textarea.
        ps_text = ""
        profile_id = (request.form.get("profile_id") or "").strip()
        if profile_id.isdigit():
            prow = db.execute("SELECT powershell_encrypted FROM turbo_session_profiles WHERE id=?", (int(profile_id),)).fetchone()
            if prow and prow["powershell_encrypted"]:
                try:
                    ps_text = decrypt_secret(prow["powershell_encrypted"])
                except Exception:
                    ps_text = ""
        if not ps_text:
            ps_text = request.form.get("powershell", "")
        try:
            parsed = turbo_listing.parse_powershell_request(ps_text)
        except ValueError as e:
            return jsonify({"ok": False, "error": f"Session: {e}"}), 400
        ok, errs = turbo_listing.validate_request(parsed, payload, proxy or None)
        if not ok:
            return jsonify({"ok": False, "error": "Validation: " + "; ".join(errs)}), 400
        # Province/city for profile update + duplicate tracking
        province = (request.form.get("province") or "").strip()
        city_name = (request.form.get("city") or "").strip()
        account_email = ""
        if profile_id.isdigit():
            erow = db.execute("SELECT email FROM turbo_session_profiles WHERE id=?", (int(profile_id),)).fetchone()
            if erow:
                account_email = erow["email"] or ""
        if not account_email:
            account_email = (request.form.get("account_email") or "").strip()

        # Duplicate check: same VIN + same province already posted successfully
        # (unified tracking first, legacy table as a fallback for pre-tracking posts)
        vin = (payload.get("vehicleData", {}).get("vin") or "").strip().upper()
        if province and vin:
            dup = db.execute(
                "SELECT id FROM turbo_campaign_posts WHERE vin=? AND province=? AND post_status='success'",
                (vin, province)).fetchone()
            if not dup:
                dup = db.execute(
                    "SELECT id FROM turbo_posted WHERE vin=? AND province=? AND status='success'",
                    (vin, province)).fetchone()
            if dup:
                return jsonify({"ok": False, "error": f"Duplicate: VIN {vin} was already posted in {province}."}), 409

        # Profile update (if province selected) — PATCH before POST
        profile_result = None
        first, last = turbo_listing.random_name()
        if province and city_name:
            city_row = db.execute("SELECT postal_code, latitude, longitude FROM turbo_cities WHERE province=? AND city=?", (province, city_name)).fetchone()
            if city_row:
                profile_payload = turbo_listing.build_profile_payload(
                    province, city_name, city_row["postal_code"],
                    city_row["latitude"], city_row["longitude"], first, last)
                profile_result = turbo_listing.submit_profile_update(ps_text, profile_payload, proxy or None)
                if not profile_result.get("ok"):
                    return jsonify({"ok": False, "error": f"Profile update failed: {profile_result.get('error')}"}), 400

        # Submit the listing
        result = turbo_listing.submit_listing(ps_text, payload, proxy or None)
        status = "success" if result["ok"] else ("uncertain" if result["http_status"] is None else "failed")
        db.execute(
            "INSERT INTO turbo_history (listing_id, submitted_at, http_status, at_request_id, status, error_summary, created_at) "
            "VALUES (?,?,?,?,?,?,?)",
            (listing_id, now_utc(), result["http_status"], result["at_request_id"], status,
             (result["error"] or "")[:500], now_utc()),
        )
        # Unified tracking: EVERY manual submission is recorded with its full
        # response info (listing id, HTTP status, request id) + the values used.
        at_listing_id = ""
        if isinstance(result.get("json"), dict):
            rdata = result["json"].get("data") or {}
            at_listing_id = str(rdata.get("listingId") or result["json"].get("id") or result["json"].get("listingId") or "")
        try:
            price_used = str(((payload.get("prices") or {}).get("public") or {}).get("price") or "")
        except Exception:
            price_used = ""
        try:
            mileage_used = str(((payload.get("condition") or {}).get("mileage")) or "")
        except Exception:
            mileage_used = ""
        vin_mode = "vin" if vin else ("replacement" if (request.form.get("vin_replacement") or "").strip() else "none")
        profile_id_value = int(profile_id) if profile_id.isdigit() else None
        cur = db.execute(
            "INSERT INTO turbo_campaign_posts (campaign_id, profile_id, listing_id, province, city, vin, "
            "first_name, last_name, autotrader_listing_id, posted_at, post_status, error, source, "
            "price_used, mileage_used, vin_mode, http_status, at_request_id) "
            "VALUES (NULL, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
            (profile_id_value, listing_id, province, city_name, vin, first, last, at_listing_id,
             now_utc(), status, (result.get("error") or "")[:500], "manual",
             price_used, mileage_used, vin_mode, result.get("http_status"), result.get("at_request_id")))
        unified_post_id = cur.lastrowid
        # Track successful posts (legacy table kept in sync for the old views)
        if result["ok"] and vin:
            db.execute(
                "INSERT INTO turbo_posted (listing_id, vin, make, model, province, city, account_email, autotrader_listing_id, posted_at, http_status, at_request_id, status) "
                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
                (listing_id, vin, payload.get("vehicleData", {}).get("make"), payload.get("vehicleData", {}).get("model"),
                 province, city_name, account_email, at_listing_id, now_utc(),
                 result["http_status"], result["at_request_id"], status))
        db.commit()
        # Auto live-check a few minutes after posting (retrieve first, page check
        # as fallback) so fresh posts get a verdict without a manual step.
        if result["ok"] and at_listing_id and unified_post_id:
            _schedule_post_live_check(unified_post_id)
        # If the listing was posted successfully, start a background INACTIVE monitor
        # (checks every 5 min for 60 min — deletes if AutoTrader flags it INACTIVE)
        if result["ok"]:
            if at_listing_id and ps_text:
                def _inactive_monitor():
                    with app.app_context():
                        for _ in range(12):  # 12 checks × 5 min = 60 min
                            time.sleep(300)
                            try:
                                listings, ret_err = turbo_listing.retrieve_listings(ps_text, proxy or None)
                                if ret_err:
                                    continue
                                for item in listings:
                                    uid = item["uuid"] if isinstance(item, dict) else str(item)
                                    lst_status = item.get("status", "") if isinstance(item, dict) else ""
                                    if uid == at_listing_id and lst_status == "INACTIVE":
                                        turbo_listing.submit_listing_delete(ps_text, at_listing_id, proxy or None)
                                        mdb = get_db()
                                        mdb.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (at_listing_id,))
                                        mdb.execute("UPDATE turbo_history SET status='deleted' WHERE listing_id=? AND status='success' ORDER BY id DESC LIMIT 1", (listing_id,))
                                        mdb.execute(
                                            "UPDATE turbo_campaign_posts SET live_status='inactive', delete_status='deleted', "
                                            "deleted_at=?, live_checked_at=?, live_check_method='auto-retrieve' WHERE autotrader_listing_id=?",
                                            (now_utc(), now_utc(), at_listing_id))
                                        mdb.commit()
                                        logging.getLogger(__name__).info(f"Manual send: auto-deleted INACTIVE listing {at_listing_id}.")
                                        return
                            except Exception as e:
                                logging.getLogger(__name__).warning(f"INACTIVE monitor error: {e}")
                threading.Thread(target=_inactive_monitor, daemon=True).start()
        if profile_result:
            result["profile_result"] = {"ok": profile_result["ok"], "http_status": profile_result["http_status"]}
        return jsonify(result)
    finally:
        _turbo_send_locks.pop(listing_id, None)


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/load/<int:profile_id>")
@login_required
def admin_turbo_profile_load(profile_id):
    db = get_db()
    row = db.execute("SELECT name, powershell_encrypted, proxy_encrypted, email FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if not row:
        return jsonify({"ok": False, "error": "Profile not found."}), 404
    ps_text = proxy_text = ""
    if row["powershell_encrypted"]:
        try:
            ps_text = decrypt_secret(row["powershell_encrypted"])
        except Exception:
            pass
    if row["proxy_encrypted"]:
        try:
            proxy_text = decrypt_secret(row["proxy_encrypted"])
        except Exception:
            pass
    return jsonify({"ok": True, "name": row["name"], "powershell": ps_text, "proxy": proxy_text, "email": row["email"] or ""})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles", methods=("POST",))
@login_required
def admin_turbo_profiles():
    db = get_db()
    action = (request.form.get("profile_action") or "").strip()
    if action == "delete" and (request.form.get("profile_id") or "").isdigit():
        db.execute("DELETE FROM turbo_session_profiles WHERE id=?", (int(request.form.get("profile_id")),))
        db.commit()
        flash("Session profile deleted.", "success")
    elif action == "save":
        name = (request.form.get("profile_name") or "").strip()
        ps_text = request.form.get("powershell", "")
        proxy_text = (request.form.get("proxy") or "").strip()
        email = (request.form.get("email") or "").strip()
        if not name:
            flash("Profile name is required.", "warning")
        elif not ps_text.strip():
            flash("Paste a PowerShell request before saving the profile.", "warning")
        else:
            try:
                parsed = turbo_listing.parse_powershell_request(ps_text)  # validate before saving
            except ValueError as e:
                flash(f"Could not save profile: {e}", "warning")
                return redirect(url_for("admin_turbo_listing"))
            try:
                enc_ps = encrypt_secret(ps_text)
                enc_parsed = encrypt_secret(json.dumps(parsed))
                enc_proxy = encrypt_secret(proxy_text) if proxy_text else ""
            except RuntimeError as e:
                flash(str(e), "warning")
                return redirect(url_for("admin_turbo_listing"))
            db.execute(
                "INSERT INTO turbo_session_profiles (name, parsed_request_encrypted, powershell_encrypted, proxy_encrypted, email, created_at, updated_at) VALUES (?,?,?,?,?,?,?) "
                "ON CONFLICT(name) DO UPDATE SET parsed_request_encrypted=excluded.parsed_request_encrypted, "
                "powershell_encrypted=excluded.powershell_encrypted, proxy_encrypted=excluded.proxy_encrypted, "
                "email=excluded.email, updated_at=excluded.updated_at",
                (name, enc_parsed, enc_ps, enc_proxy, email, now_utc(), now_utc()),
            )
            db.commit()
            flash(f"Session profile '{name}' saved (encrypted).", "success")
            # Reload the page with this profile auto-loaded (instead of blank
            # fields + a reset dropdown, which forced a manual re-select).
            saved = db.execute("SELECT id FROM turbo_session_profiles WHERE name=?", (name,)).fetchone()
            if saved:
                return redirect(url_for("admin_turbo_listing", profile=saved["id"]))
    return redirect(url_for("admin_turbo_listing"))


# ---------------- Turbo Listing Campaigns (auto publish/delete rotation) ----------------

_turbo_campaign_threads = {}
_turbo_campaign_locks = {}


def _campaign_lock(campaign_id):
    """Per-campaign mutex held only during publish (not during the live-wait),
    so add-user publish serializes with the normal cycle instead of racing it
    on SQLite + duplicate inserts (BUG #7)."""
    lock = _turbo_campaign_locks.get(campaign_id)
    if lock is None:
        lock = threading.Lock()
        _turbo_campaign_locks[campaign_id] = lock
    return lock


def _campaign_log(db, campaign_id, phase, message, level="info", batch_number=None,
                  profile_id=None, listing_id=None):
    """Structured per-post lifecycle log (improvement #6). Best-effort: never raises."""
    try:
        db.execute(
            "INSERT INTO turbo_campaign_log "
            "(campaign_id, batch_number, profile_id, listing_id, phase, level, message, logged_at) "
            "VALUES (?,?,?,?,?,?,?,?)",
            (campaign_id, batch_number, profile_id, listing_id, phase, level, str(message)[:1000], now_utc()),
        )
        db.commit()
    except Exception:
        pass


def _campaign_set_error(db, campaign_id, err, phase="error"):
    """Record a fatal error on the campaign + set phase (BUG #5)."""
    try:
        db.execute(
            "UPDATE turbo_campaigns SET current_phase=?, last_error=?, last_error_at=? WHERE id=?",
            (phase, str(err)[:1000], now_utc(), campaign_id),
        )
        db.commit()
    except Exception:
        pass
    _campaign_log(db, campaign_id, phase, err, level="error")


def _campaign_heartbeat(db, campaign_id, phase=None):
    """Touch last_heartbeat so a watchdog can detect a dead thread (improvement #1)."""
    try:
        if phase:
            db.execute("UPDATE turbo_campaigns SET last_heartbeat=? WHERE id=?", (now_utc(), campaign_id))
        else:
            db.execute("UPDATE turbo_campaigns SET last_heartbeat=? WHERE id=?", (now_utc(), campaign_id))
        db.commit()
    except Exception:
        pass


def _campaign_wait_seconds(timeframe_hours):
    """Float-aware conversion of timeframe_hours to seconds (BUG #6: int() truncated
    0.5h to 0 → instant delete loop). Min 0.25h to avoid a self-DoS."""
    try:
        hours = float(timeframe_hours)
    except (TypeError, ValueError):
        hours = 5.0
    if hours < 0.25:
        hours = 0.25
    return hours * 3600.0


def _campaign_recent_failures(db, profile_id, n=3):
    """True if the profile's last N posts are ALL failed (BUG #3: the old COUNT(*)
    ... ORDER BY id DESC LIMIT n returned the lifetime failure total, not 'last N
    consecutive')."""
    rows = db.execute(
        "SELECT post_status FROM turbo_campaign_posts WHERE profile_id=? ORDER BY id DESC LIMIT ?",
        (profile_id, n),
    ).fetchall()
    return len(rows) >= n and all((r["post_status"] == "failed") for r in rows)


def _campaign_delete_sleep(campaign):
    """Same-account delete stagger with jitter (BUG #8: was a flat 10s, which bursts
    deletes and trips AutoTrader alarms). Uses the campaign's same-account delay
    settings + up to 20% jitter, falling back to 90-150s."""
    import random as _r
    same_min = int(campaign["delay_same_min"]) if campaign["delay_same_min"] else 90
    same_max = int(campaign["delay_same_max"]) if campaign["delay_same_max"] else 150
    if same_max <= same_min:
        same_max = same_min + 30
    base = _r.randint(same_min, same_max)
    jitter = _r.randint(0, max(1, base // 5))
    time.sleep(base + jitter)


def _campaign_weekly_success_count(db, profile_id):
    """Count successful posts in the rolling 7-day window (improvement #2: enforce
    AutoTrader's weekly limit locally so we stop assigning BEFORE tripping 422)."""
    cutoff = (datetime.utcnow() - timedelta(days=7)).isoformat(timespec="seconds")
    return db.execute(
        "SELECT COUNT(*) FROM turbo_campaign_posts WHERE profile_id=? AND post_status='success' "
        "AND posted_at >= ?",
        (profile_id, cutoff),
    ).fetchone()[0]


def _campaign_first_post_at(db, profile_id):
    """Earliest successful post for a profile (used to compute the 7-day reset)."""
    row = db.execute(
        "SELECT MIN(posted_at) AS fp FROM turbo_campaign_posts WHERE profile_id=? AND post_status='success'",
        (profile_id,)).fetchone()
    if row and row["fp"]:
        try:
            return datetime.fromisoformat(row["fp"])
        except (ValueError, TypeError):
            return None
    return None


DUPLICATE_VIN_WINDOW_DAYS = 7


def _campaign_vin_recently_posted(db, vin):
    """Has this VIN been posted (by any account, success OR deleted) in the last 7 days?
    AutoTrader deactivates a re-posted VIN within minutes (it goes INACTIVE) — and it
    does this EVEN IF the prior listing was already deleted (AutoTrader remembers the
    VIN within its dedup window). So we must skip a VIN that's been posted at all in the
    last 7 days, not just one that's still live. This is a BEFORE-posting check.
    Normalizes the VIN (strip+upper) to match how turbo_posted.vin is stored."""
    vin = (vin or "").strip().upper()
    if not vin:
        return False
    cutoff = (datetime.utcnow() - timedelta(days=DUPLICATE_VIN_WINDOW_DAYS)).isoformat(timespec="seconds")
    return db.execute(
        "SELECT 1 FROM turbo_posted WHERE vin=? AND posted_at >= ? LIMIT 1",
        (vin, cutoff)).fetchone() is not None


VIN_COOLDOWN_DAYS = 7


def _campaign_vin_cooldown_state(db, vin):
    """Decide whether to INCLUDE the VIN when posting this car. Returns (include_vin,
    reason). include_vin=False (post WITHOUT the VIN) when the VIN is:
      - permanently excluded (2+ INACTIVE strikes), OR
      - in cooldown (went INACTIVE within VIN_COOLDOWN_DAYS), OR
      - was posted at all in the last VIN_COOLDOWN_DAYS (would likely go INACTIVE).
    The car is still posted — just without the VIN — so the listing stays live while the
    VIN is hot. After the cooldown passes, the VIN is re-included (and a 2nd INACTIVE
    then permanently excludes it)."""
    vin = (vin or "").strip().upper()
    if not vin:
        return True, ""  # no VIN on the inventory row — post as-is (no VIN)
    cd = db.execute(
        "SELECT inactive_count, last_inactive_at, excluded, reason FROM turbo_vin_cooldown WHERE vin=?",
        (vin,)).fetchone()
    if cd and cd["excluded"]:
        return False, f"VIN permanently excluded (INACTIVE {cd['inactive_count']}x): {cd['reason'] or ''}"
    if cd and cd["last_inactive_at"]:
        try:
            if datetime.utcnow() - datetime.fromisoformat(cd["last_inactive_at"]) < timedelta(days=VIN_COOLDOWN_DAYS):
                return False, f"VIN in cooldown since {cd['last_inactive_at'][:16]} (INACTIVE): {cd['reason'] or ''}"
        except (ValueError, TypeError):
            pass
    if _campaign_vin_recently_posted(db, vin):
        return False, "VIN posted in last 7d (would likely go INACTIVE)"
    return True, ""


def _campaign_strike_vin(db, vin, reason):
    """Record an INACTIVE strike on a VIN. 1st strike → cooldown (posted without VIN for
    VIN_COOLDOWN_DAYS). 2nd strike → permanently excluded (always without VIN)."""
    vin = (vin or "").strip().upper()
    if not vin:
        return
    reason = (reason or "")[:300]
    now = now_utc()
    cd = db.execute("SELECT inactive_count FROM turbo_vin_cooldown WHERE vin=?", (vin,)).fetchone()
    if cd:
        new_count = int(cd["inactive_count"] or 0) + 1
        excluded = 1 if new_count >= 2 else 0
        db.execute(
            "UPDATE turbo_vin_cooldown SET inactive_count=?, last_inactive_at=?, reason=?, excluded=? WHERE vin=?",
            (new_count, now, reason, excluded, vin))
    else:
        db.execute(
            "INSERT INTO turbo_vin_cooldown (vin, inactive_count, last_inactive_at, excluded, reason) VALUES (?,?,?,?,?)",
            (vin, 1, now, 0, reason))
    db.commit()


def _summarize_inactive_reason(item):
    """Best-effort: pull a human reason out of AutoTrader's INACTIVE listing item. We
    don't know the exact response schema, so check common reason-looking fields, then
    fall back to a truncated JSON of the whole item (minus images/media) so the admin
    can see exactly what AutoTrader returned."""
    if not isinstance(item, dict):
        return ""
    for k in ("statusReason", "statusReasons", "reason", "rejectionReason", "rejection_reason",
              "message", "detail", "statusDetails", "description", "comment"):
        v = item.get(k)
        if v:
            return str(v)[:300]
    try:
        slim = {k: v for k, v in item.items() if k not in ("images", "media", "mediaIds")}
        return json.dumps(slim, ensure_ascii=False)[:300]
    except Exception:
        return ""


def _campaign_pause_reset_at(db, profile_id):
    """When can a 422-paused profile create listings again? AutoTrader allows 3 creations
    per rolling 7-day window. The 422 fires when 3 successes exist in the last 7 days; the
    profile can create again once the OLDEST of those 3 falls outside the window → that
    post's posted_at + 7 days. (The old code used first_post+7d, which is wrong if the
    profile has posts older than 7d — it could set paused_until in the past, so the pause
    never took effect and the profile kept 422-ing.) Falls back to now+7d."""
    rows = db.execute(
        "SELECT posted_at FROM turbo_campaign_posts WHERE profile_id=? AND post_status='success' "
        "ORDER BY posted_at DESC LIMIT 3",
        (profile_id,)).fetchall()
    if len(rows) < 3:
        return datetime.utcnow() + timedelta(days=7)
    try:
        return datetime.fromisoformat(rows[-1]["posted_at"]) + timedelta(days=7)
    except (ValueError, TypeError):
        return datetime.utcnow() + timedelta(days=7)


def _campaign_active_count(db, profile_id):
    """Count ACTIVE listings for this profile.
    Only counts posts where delete was NOT attempted or succeeded.
    delete_failed posts are excluded (the listing may already be gone from AutoTrader
    but the delete API failed due to expired session — use 'Mark all deleted' to clear them)."""
    return db.execute(
        "SELECT COUNT(*) FROM turbo_campaign_posts WHERE profile_id=? AND post_status='success' "
        "AND COALESCE(delete_status,'') NOT IN ('deleted','delete_failed')",
        (profile_id,)).fetchone()[0]


def _campaign_can_post(db, campaign, profile_id):
    """Can this profile post right now?
    - Not in a 422 cooldown (paused_until, BUG #4).
    - Under 3 ACTIVE (concurrent) listings for this profile.

    NOTE: this is a 3-CONCURRENT cap, NOT 3/week. Deleted posts do NOT count against
    it — so the intended workflow (post 3 → live window → delete → repost 3) works
    within the same week. An earlier revision added a rolling 3/week local cap, but
    that blocked the delete→repost cycle after 3 posts/week even when AutoTrader had
    0 active, producing the "only 1 listing per batch" symptom. AutoTrader's own real
    limit (if any) is enforced by the 422 response → paused_until cooldown, not by a
    local guess. Extended mode: after 7 days from first post, raise the concurrent cap."""
    row = db.execute("SELECT paused_until FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if row and row["paused_until"]:
        try:
            if datetime.utcnow() < datetime.fromisoformat(row["paused_until"]):
                return False, 0
        except (ValueError, TypeError):
            pass  # unparseable paused_until → treat as not paused
    active = _campaign_active_count(db, profile_id)
    active_cap = 3
    if campaign["extended_mode"]:
        first = _campaign_first_post_at(db, profile_id)
        if first and datetime.utcnow() - first >= timedelta(days=7):
            active_cap = 6
    remaining = active_cap - active
    if remaining <= 0:
        return False, 0
    return True, remaining


def _campaign_pick_location(db, vin):
    import random as _r
    posted_cities = [r["city"] for r in db.execute("SELECT DISTINCT city FROM turbo_posted WHERE vin=? AND status='success'", (vin,)).fetchall()] if vin else []
    candidates = [r for r in db.execute("SELECT province, city, postal_code, latitude, longitude FROM turbo_cities").fetchall() if r["city"] not in posted_cities]
    if not candidates:
        candidates = db.execute("SELECT province, city, postal_code, latitude, longitude FROM turbo_cities").fetchall()
    return _r.choice(candidates)


def _campaign_last_post_time(db, profile_id):
    """Get the last successful post time for a profile from the DB (survives restarts)."""
    row = db.execute(
        "SELECT posted_at FROM turbo_campaign_posts WHERE profile_id=? AND post_status='success' ORDER BY id DESC LIMIT 1",
        (profile_id,)).fetchone()
    if not row or not row["posted_at"]:
        return None
    try:
        return datetime.fromisoformat(row["posted_at"])
    except ValueError:
        return None


def _campaign_wait_for_delay(db, campaign, profile_id, last_post_by_user):
    """Enforce same-account delay (2-3 min) using BOTH in-memory + DB last-post time.
    Handles NULL delay settings by falling back to defaults (120/180/10/30).
    Tolerates either datetime or float (epoch) in last_post_by_user so a stale
    time.time()-style value can never crash the max() comparison (BUG #1)."""
    import random as _r
    same_min = int(campaign["delay_same_min"]) if campaign["delay_same_min"] else 120
    same_max = int(campaign["delay_same_max"]) if campaign["delay_same_max"] else 180
    cross_min = int(campaign["delay_cross_min"]) if campaign["delay_cross_min"] else 10
    cross_max = int(campaign["delay_cross_max"]) if campaign["delay_cross_max"] else 30
    mem_time = last_post_by_user.get(profile_id)
    if isinstance(mem_time, (int, float)):
        mem_time = datetime.utcfromtimestamp(float(mem_time))
    db_time = _campaign_last_post_time(db, profile_id)
    last_time = max(filter(None, [mem_time, db_time]), default=None)
    if last_time:
        elapsed = (datetime.utcnow() - last_time).total_seconds()
        min_delay = _r.randint(same_min, same_max)
        if elapsed < min_delay:
            time.sleep(max(0, min_delay - elapsed))
    else:
        time.sleep(_r.randint(cross_min, cross_max))


def _campaign_delete_previous_batch(db, campaign_id):
    """Delete all live listings from previous batches before starting a new cycle.
    Stored-UUID delete only (no per-post retrieve-fallback — it burned minutes per post
    on expired sessions and made cleanup look stuck). Uses the same-account delete
    stagger (BUG #8: was a flat 10s)."""
    campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
    posts = db.execute(
        "SELECT tcp.*, tsp.powershell_encrypted, tsp.proxy_encrypted FROM turbo_campaign_posts tcp "
        "JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id "
        "WHERE tcp.campaign_id=? AND tcp.post_status='success' AND tcp.autotrader_listing_id != '' "
        "AND COALESCE(tcp.delete_status,'') NOT IN ('deleted','delete_failed')",
        (campaign_id,)).fetchall()
    for p in posts:
        try:
            ps_text = decrypt_secret(p["powershell_encrypted"]) if p["powershell_encrypted"] else ""
        except Exception:
            ps_text = ""
        if not ps_text:
            db.execute("UPDATE turbo_campaign_posts SET delete_status='delete_failed', delete_error='no session' WHERE id=?", (p["id"],))
            db.commit()
            continue
        proxy = ""
        if p["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(p["proxy_encrypted"])
            except Exception:
                proxy = ""
        # Stored-UUID delete only (see _campaign_delete_batch for why no retrieve-fallback).
        result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
        del_status = "deleted" if result.get("ok") else "delete_failed"
        del_err = (result.get("error") or "")[:500] if not result.get("ok") else None
        db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status=?, delete_error=? WHERE id=?",
                   (now_utc(), del_status, del_err, p["id"]))
        if result.get("ok"):
            db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?",
                       (p["autotrader_listing_id"],))
        db.commit()
        _campaign_log(db, campaign_id, "delete", f"{del_status}: post {p['id']} ({p['autotrader_listing_id']})", level="info" if result.get("ok") else "warning")
        _campaign_delete_sleep(campaign)


def _campaign_reconcile_profile(db, campaign, profile_id, ps_text, proxy):
    """Best-effort orphan reconciliation (improvement #5): retrieve the account's live
    AutoTrader listings and delete any that are NOT tracked as an active success post
    for this profile (out-of-band posts or a previously failed delete). Called once per
    user per batch. Never raises — a retrieve failure just skips reconciliation."""
    try:
        listings, err = turbo_listing.retrieve_listings(ps_text, proxy or None)
    except Exception:
        return
    if err or not listings:
        return
    tracked = {r["autotrader_listing_id"] for r in db.execute(
        "SELECT autotrader_listing_id FROM turbo_campaign_posts WHERE profile_id=? "
        "AND post_status='success' AND COALESCE(delete_status,'') != 'deleted' "
        "AND autotrader_listing_id != ''",
        (profile_id,)).fetchall()}
    for item in listings:
        uid = item["uuid"] if isinstance(item, dict) else str(item)
        if uid in tracked:
            continue
        try:
            dr = turbo_listing.submit_listing_delete(ps_text, uid, proxy or None)
            if dr.get("ok"):
                db.execute("UPDATE turbo_campaign_posts SET delete_status='deleted', deleted_at=?, delete_error='reconciled orphan' WHERE profile_id=? AND autotrader_listing_id=?",
                           (now_utc(), profile_id, uid))
                db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (uid,))
                db.commit()
                _campaign_log(db, campaign["id"], "reconcile", f"deleted orphan {uid}", profile_id=profile_id)
        except Exception:
            pass
        _campaign_delete_sleep(campaign)


def _campaign_publish_one(db, campaign, user, vid, batch_number, last_post_by_user,
                          patched_profiles, reconciled_profiles):
    """Publish ONE vehicle for ONE user. Shared by _campaign_publish_batch (initial fill
    to 3 active) and _campaign_topup (live-window replacements). Returns:
      'success' | 'failed' | 'skipped' (already posted / can't post / no listing / no session)
      'paused'  (422 weekly-limit OR 3 consecutive failures → profile paused, auto-resumes)
    A paused profile is skipped by _campaign_can_post on subsequent vehicles."""
    campaign_id = campaign["id"]
    pid = user["profile_id"]
    c = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
    if not c or c["status"] != "running":
        return "skipped"
    already = db.execute("SELECT id FROM turbo_campaign_posts WHERE campaign_id=? AND batch_number=? AND listing_id=? AND profile_id=?", (campaign_id, batch_number, vid, pid)).fetchone()
    if already:
        return "skipped"  # this user already posted this vehicle this batch (incl. a since-deleted one)
    can, remaining = _campaign_can_post(db, campaign, pid)
    if not can or remaining <= 0:
        return "skipped"
    listing = get_db().execute("SELECT * FROM listings WHERE id=?", (vid,)).fetchone()
    if not listing:
        return "skipped"
    # VIN-inclusion decision: "Always no-VIN" campaign setting OVERRULES everything
    # (cooldown, recently-posted, etc.) — if checked, never include the VIN. Otherwise
    # use the cooldown + escalation logic (omit when hot, include when safe).
    vin_norm = (listing["vin"] or "").strip().upper()
    always_no_vin = campaign["always_no_vin"] if "always_no_vin" in campaign.keys() else 0
    if always_no_vin:
        include_vin = False
        vin_reason = "always-no-VIN campaign setting"
    else:
        include_vin, vin_reason = _campaign_vin_cooldown_state(db, vin_norm)
    if not include_vin:
        _campaign_log(db, campaign_id, "publish",
                      f"posting {listing['make']} {listing['model']} WITHOUT VIN ({vin_reason})",
                      batch_number=batch_number, profile_id=pid, listing_id=vid, level="warning")
    # Delay: check BOTH in-memory + DB for last post time (survives restarts)
    _campaign_wait_for_delay(db, campaign, pid, last_post_by_user)
    try:
        ps_text = decrypt_secret(user["powershell_encrypted"]) if user["powershell_encrypted"] else ""
    except Exception:
        ps_text = ""
    if not ps_text:
        return "skipped"
    proxy = ""
    if user["proxy_encrypted"]:
        try:
            proxy = decrypt_secret(user["proxy_encrypted"])
        except Exception:
            proxy = ""
    # Reconcile orphans once per user per batch (improvement #5).
    if pid not in reconciled_profiles:
        _campaign_reconcile_profile(db, campaign, pid, ps_text, proxy)
        reconciled_profiles.add(pid)
    loc = _campaign_pick_location(db, (listing["vin"] or "").strip().upper())
    first, last = turbo_listing.random_name()
    # Profile PATCH once per user per batch (BUG #9: was per-post → name/city
    # morphing every 2-3 min, a flagging signal).
    if pid not in patched_profiles:
        profile_payload = turbo_listing.build_profile_payload(loc["province"], loc["city"], loc["postal_code"], loc["latitude"], loc["longitude"], first, last)
        turbo_listing.submit_profile_update(ps_text, profile_payload, proxy or None)
        patched_profiles.add(pid)
    description = _campaign_get_description(listing, get_settings())
    try:
        payload = turbo_listing.build_payload(listing, {"include_vin": include_vin, "vin_replacement": "", "mileage_override": "", "price_override": ""}, db, description=description)
    except Exception as e:
        db.execute("INSERT INTO turbo_campaign_posts (campaign_id, batch_number, listing_id, profile_id, posted_at, post_status, error) VALUES (?,?,?,?,?,?,?)",
                   (campaign_id, batch_number, vid, pid, now_utc(), "failed", f"build_payload: {e}"[:500]))
        db.commit()
        return "failed"
    result = turbo_listing.submit_listing(ps_text, payload, proxy or None)
    vin = (payload.get("vehicleData", {}).get("vin") or "").strip().upper()
    at_id = ""
    if result.get("ok") and isinstance(result.get("json"), dict):
        rdata = result["json"].get("data") or {}
        at_id = str(rdata.get("listingId") or "")
    status = "success" if result.get("ok") else "failed"
    try:
        price_used = str(((payload.get("prices") or {}).get("public") or {}).get("price") or "")
    except Exception:
        price_used = ""
    try:
        mileage_used = str(((payload.get("condition") or {}).get("mileage")) or "")
    except Exception:
        mileage_used = ""
    vin_mode = "vin" if vin else "none"
    # INSERT OR IGNORE: the partial unique index (success-only) makes a duplicate
    # successful post a no-op instead of an IntegrityError crash (BUG #7).
    cur = db.execute("INSERT OR IGNORE INTO turbo_campaign_posts (campaign_id, batch_number, listing_id, profile_id, province, city, vin, first_name, last_name, autotrader_listing_id, posted_at, post_status, error, source, price_used, mileage_used, vin_mode, http_status, at_request_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                     (campaign_id, batch_number, vid, pid, loc["province"], loc["city"], vin, first, last, at_id, now_utc(), status, (result.get("error") or "")[:500], "campaign", price_used, mileage_used, vin_mode, result.get("http_status"), result.get("at_request_id")))
    if cur.rowcount and result.get("ok") and vin:
        db.execute("INSERT INTO turbo_posted (listing_id, vin, make, model, province, city, account_email, autotrader_listing_id, posted_at, http_status, at_request_id, status) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
                   (vid, vin, payload.get("vehicleData", {}).get("make"), payload.get("vehicleData", {}).get("model"), loc["province"], loc["city"], user["email"] or "", at_id, now_utc(), result.get("http_status"), result.get("at_request_id"), status))
    db.commit()
    # Auto live-check the fresh listing a few minutes later (retrieve first, page
    # check as fallback) so every campaign post gets a live verdict on its own.
    if result.get("ok") and at_id and cur.rowcount:
        _schedule_post_live_check(cur.lastrowid)
    last_post_by_user[pid] = datetime.utcnow()
    _campaign_log(db, campaign_id, "publish",
                  f"{status}: {listing['make']} {listing['model']} -> profile {pid}" + (f" ({at_id})" if at_id else ""),
                  batch_number=batch_number, profile_id=pid, listing_id=vid,
                  level="info" if status == "success" else "warning")
    if not result.get("ok"):
        err_msg = (result.get("error") or "").lower()
        if ("limit" in err_msg and "exceeded" in err_msg) or result.get("http_status") == 422:
            # 422 limit hit: PAUSE creation until the rolling 7-day window drops a
            # creation (oldest of the last 3 successes + 7d). Deletion is NOT blocked —
            # only posting/creation (AutoTrader's "3 creations per week" rule).
            reset_at = _campaign_pause_reset_at(db, pid)
            db.execute("UPDATE turbo_session_profiles SET paused_until=?, paused_reason=? WHERE id=?",
                       (reset_at.isoformat(timespec="seconds"), (result.get("error") or "422 limit exceeded")[:300], pid))
            db.commit()
            _campaign_log(db, campaign_id, "publish", f"profile {pid} paused until {reset_at.isoformat(timespec='seconds')} (422)", profile_id=pid, level="error")
            logging.getLogger(__name__).warning(f"Profile {pid} paused (422 limit) until {reset_at.isoformat()}.")
            return "paused"
        if _campaign_recent_failures(db, pid, 3):
            # 3 consecutive failures (usually an expired session / bad proxy). PAUSE the
            # profile for 2h with auto-resume (_campaign_can_post checks paused_until)
            # instead of permanently flagging + ripping it out of all campaigns — that
            # was too harsh on transient failures. Admin sees the PAUSED badge + reason;
            # refresh cookies and use Unpause to resume early.
            reset_at = datetime.utcnow() + timedelta(hours=2)
            db.execute("UPDATE turbo_session_profiles SET paused_until=?, paused_reason=? WHERE id=?",
                       (reset_at.isoformat(timespec="seconds"), (result.get("error") or "3 consecutive failures")[:300], pid))
            db.commit()
            _campaign_log(db, campaign_id, "publish", f"profile {pid} paused until {reset_at.isoformat(timespec='seconds')} (3 consecutive failures)", profile_id=pid, level="error")
            logging.getLogger(__name__).warning(f"Profile {pid} paused (3 consecutive failures) until {reset_at.isoformat()}.")
            return "paused"
    return status


def _campaign_publish_batch(campaign_id, batch_number):
    """Initial fill: publish one vehicle per available user until the pool or 3-active
    caps are exhausted. Per-post details + failure/422 handling live in
    _campaign_publish_one. _campaign_topup handles live-window replacements when
    listings go INACTIVE."""
    with _campaign_lock(campaign_id):
        db = get_db()
        campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
        if not campaign or campaign["status"] != "running":
            return
        users = db.execute("SELECT tcu.*, tsp.powershell_encrypted, tsp.proxy_encrypted, tsp.email, tsp.flagged FROM turbo_campaign_users tcu JOIN turbo_session_profiles tsp ON tsp.id=tcu.profile_id WHERE tcu.campaign_id=? AND COALESCE(tsp.flagged,0)=0", (campaign_id,)).fetchall()
        vehicle_ids = [r["listing_id"] for r in db.execute("SELECT listing_id FROM turbo_campaign_vehicles WHERE campaign_id=?", (campaign_id,)).fetchall()]
        import random as _r
        _r.shuffle(vehicle_ids)
        last_post_by_user = {}
        patched_profiles = set()
        reconciled_profiles = set()
        for vid in vehicle_ids:
            c = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
            if not c or c["status"] != "running":
                return
            assigned = None
            for u in users:
                can, remaining = _campaign_can_post(db, campaign, u["profile_id"])
                if can and remaining > 0:
                    assigned = u
                    break
            if not assigned:
                continue
            status = _campaign_publish_one(db, campaign, assigned, vid, batch_number,
                                           last_post_by_user, patched_profiles, reconciled_profiles)
            if status == "flagged":
                users = [u for u in users if u["profile_id"] != assigned["profile_id"]]


def _campaign_topup(campaign_id, batch_number):
    """Live-window top-up: post replacements for any user whose ACTIVE count dropped
    below 3 (e.g. a listing went INACTIVE and was auto-deleted). Draws fresh vehicles
    from the pool, skipping vehicles this user already posted this batch — so it will
    NOT re-post a listing that just got deleted; it picks a different vehicle. Shares
    _campaign_publish_one with the initial fill. Returns the number of replacements
    posted. No-op (fast) when every user is already at 3."""
    posted = 0
    with _campaign_lock(campaign_id):
        db = get_db()
        campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
        if not campaign or campaign["status"] != "running":
            return posted
        users = db.execute("SELECT tcu.*, tsp.powershell_encrypted, tsp.proxy_encrypted, tsp.email, tsp.flagged FROM turbo_campaign_users tcu JOIN turbo_session_profiles tsp ON tsp.id=tcu.profile_id WHERE tcu.campaign_id=? AND COALESCE(tsp.flagged,0)=0", (campaign_id,)).fetchall()
        vehicle_ids = [r["listing_id"] for r in db.execute("SELECT listing_id FROM turbo_campaign_vehicles WHERE campaign_id=?", (campaign_id,)).fetchall()]
        import random as _r
        _r.shuffle(vehicle_ids)
        last_post_by_user = {}
        patched_profiles = set()
        reconciled_profiles = set()
        for u in users:
            pid = u["profile_id"]
            # Stop once this user is back at 3 (or can't post: paused / pool exhausted).
            while True:
                can, remaining = _campaign_can_post(db, campaign, pid)
                if not can or remaining <= 0:
                    break
                c = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
                if not c or c["status"] != "running":
                    return posted
                # Find the next vehicle this user hasn't posted this batch.
                vid = None
                for v in vehicle_ids:
                    got = db.execute("SELECT id FROM turbo_campaign_posts WHERE campaign_id=? AND batch_number=? AND listing_id=? AND profile_id=?", (campaign_id, batch_number, v, pid)).fetchone()
                    if not got:
                        vid = v
                        break
                if vid is None:
                    break  # pool exhausted for this user
                status = _campaign_publish_one(db, campaign, u, vid, batch_number,
                                               last_post_by_user, patched_profiles, reconciled_profiles)
                if status == "success":
                    posted += 1
                elif status == "flagged":
                    break  # user removed; next user
                elif status == "paused":
                    break  # user paused; next user
                # 'failed'/'skipped' → loop re-checks can_post and tries the next vehicle
        if posted:
            _campaign_log(db, campaign_id, "topup", f"posted {posted} replacement(s) in batch {batch_number}", batch_number=batch_number, level="info")
    return posted


def _campaign_check_inactive_listings(campaign_id, batch_number):
    """Check all live (non-deleted) posts in a batch for INACTIVE status.
    Delete any that are INACTIVE + mark as deleted + log it. Returns count deleted.

    Retrieve-once-per-profile: groups posts by account and makes ONE retrieve_listings
    call per profile, then matches all that profile's posts against the result set
    (was N retrieve calls per profile — a major AutoTrader API/rate-limit cost)."""
    db = get_db()
    posts = db.execute(
        "SELECT tcp.*, tsp.powershell_encrypted, tsp.proxy_encrypted FROM turbo_campaign_posts tcp "
        "JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id "
        "WHERE tcp.campaign_id=? AND tcp.batch_number=? AND tcp.post_status='success' "
        "AND tcp.autotrader_listing_id != '' AND COALESCE(tcp.delete_status,'') != 'deleted'",
        (campaign_id, batch_number)).fetchall()
    # Group posts by profile so we retrieve each account's listings once.
    by_profile = {}
    for p in posts:
        by_profile.setdefault(p["profile_id"], []).append(p)
    deleted_count = 0
    for pid, prof_posts in by_profile.items():
        first = prof_posts[0]
        try:
            ps_text = decrypt_secret(first["powershell_encrypted"]) if first["powershell_encrypted"] else ""
        except Exception:
            continue
        if not ps_text:
            continue
        proxy = ""
        if first["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(first["proxy_encrypted"])
            except Exception:
                proxy = ""
        listings, ret_err = turbo_listing.retrieve_listings(ps_text, proxy or None)
        if ret_err or not listings:
            continue
        # Index this account's live listings by uuid → status + raw item (one retrieve,
        # many lookups). The raw item lets us capture WHY AutoTrader marked it INACTIVE.
        status_by_uid = {}
        raw_by_uid = {}
        for item in listings:
            uid = item["uuid"] if isinstance(item, dict) else str(item)
            st = item.get("status", "") if isinstance(item, dict) else ""
            status_by_uid[uid] = st
            if isinstance(item, dict):
                raw_by_uid[uid] = item.get("raw") or {}
        for p in prof_posts:
            if status_by_uid.get(p["autotrader_listing_id"]) != "INACTIVE":
                continue
            reason = _summarize_inactive_reason(raw_by_uid.get(p["autotrader_listing_id"]))
            del_result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
            del_status = "deleted" if del_result.get("ok") else "delete_failed"
            err_text = "auto-deleted: listing was INACTIVE" + (f" — {reason}" if reason else "") + (f" (delete failed: {del_result.get('error','')[:200]}" if not del_result.get("ok") else "")
            db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status=?, delete_error=? WHERE id=?",
                       (now_utc(), del_status, err_text, p["id"]))
            if del_result.get("ok"):
                db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (p["autotrader_listing_id"],))
            db.commit()
            # Strike the VIN (cooldown + escalation) — only if this post was made WITH
            # a VIN. Without-VIN posts have no VIN to strike.
            post_vin = (p["vin"] or "").strip().upper() if "vin" in p.keys() else ""
            if post_vin:
                _campaign_strike_vin(db, post_vin, reason)
            deleted_count += 1
            _campaign_log(db, campaign_id, "inactive", f"INACTIVE deleted {p['autotrader_listing_id']} (VIN {post_vin or '—'}): {reason or 'no reason returned'}", batch_number=batch_number, profile_id=pid, level="warning")
            logging.getLogger(__name__).info(f"Campaign {campaign_id}: auto-deleted INACTIVE listing {p['autotrader_listing_id']} (post {p['id']}, VIN {post_vin or '—'}): {reason}")
    return deleted_count


def _campaign_delete_batch(campaign_id, batch_number):
    db = get_db()
    campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
    # Skip posts already marked deleted (e.g. force-advanced) so the loop can race to
    # completion instead of re-trying already-deleted listings.
    posts = db.execute(
        "SELECT tcp.*, tsp.powershell_encrypted, tsp.proxy_encrypted FROM turbo_campaign_posts tcp "
        "JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id "
        "WHERE tcp.campaign_id=? AND tcp.batch_number=? AND tcp.post_status='success' "
        "AND tcp.autotrader_listing_id != '' AND COALESCE(tcp.delete_status,'') != 'deleted'",
        (campaign_id, batch_number)).fetchall()
    for p in posts:
        c = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
        if not c or c["status"] != "running":
            return
        try:
            ps_text = decrypt_secret(p["powershell_encrypted"]) if p["powershell_encrypted"] else ""
        except Exception:
            ps_text = ""
        if not ps_text:
            db.execute("UPDATE turbo_campaign_posts SET delete_status='delete_failed', delete_error='no session' WHERE id=?", (p["id"],))
            db.commit()
            continue
        proxy = ""
        if p["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(p["proxy_encrypted"])
            except Exception:
                proxy = ""
        # Stored-UUID delete only. A failure is almost always an expired session (403),
        # in which case retrieve_listings would 403 too — so the old per-post
        # retrieve-fallback just burned minutes per post and made "deleting" look stuck.
        # Mark delete_failed (with the error) and move on; the admin can Retrieve+Delete
        # or Mark all deleted to clean up.
        result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
        del_status = "deleted" if result.get("ok") else "delete_failed"
        del_err = (result.get("error") or "")[:500] if not result.get("ok") else None
        db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status=?, delete_error=? WHERE id=?",
                   (now_utc(), del_status, del_err, p["id"]))
        if result.get("ok"):
            db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (p["autotrader_listing_id"],))
        db.commit()
        _campaign_log(db, campaign_id, "delete", f"{del_status}: post {p['id']} ({p['autotrader_listing_id']}) batch {batch_number}", batch_number=batch_number, level="info" if result.get("ok") else "warning")
        _campaign_delete_sleep(campaign)


def _campaign_wait_live(campaign_id, batch, first_dt):
    """Live-window wait loop. Re-reads timeframe each minute so mid-wait updates take
    effect immediately. Interruptible: returns False if the campaign was stopped
    (caller returns), True if the live window elapsed normally. Runs the
    INACTIVE-listing check on a fixed every-5th-iteration cadence (BUG #12: was an
    unreliable int(elapsed)%300<60 modulo gate that could fire every loop or never)."""
    db = get_db()
    inactive_counter = 0
    while True:
        c = db.execute("SELECT status, timeframe_hours FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
        if not c or c["status"] != "running":
            return False
        _campaign_heartbeat(db, campaign_id)
        wait_seconds = _campaign_wait_seconds(c["timeframe_hours"])
        elapsed = (datetime.utcnow() - first_dt).total_seconds()
        inactive_counter += 1
        if inactive_counter % 5 == 0:
            # INACTIVE check only in the first hour (it makes per-user API calls).
            if elapsed < 3600:
                try:
                    inactive_deleted = _campaign_check_inactive_listings(campaign_id, batch)
                    if inactive_deleted:
                        logging.getLogger(__name__).info(f"Campaign {campaign_id} batch {batch}: auto-deleted {inactive_deleted} INACTIVE listing(s).")
                except Exception as e:
                    logging.getLogger(__name__).warning(f"INACTIVE check failed: {e}")
            # Top up throughout the WHOLE live window so each user holds 3 active even
            # after an INACTIVE auto-delete (the user's "should have 3, not 1" requirement).
            # Fast no-op when everyone is already at 3.
            try:
                _campaign_topup(campaign_id, batch)
            except Exception as e:
                logging.getLogger(__name__).warning(f"topup failed: {e}")
        if elapsed >= wait_seconds:
            return True
        time.sleep(60)


def _run_campaign(campaign_id, resume=False):
    with app.app_context():
        db = get_db()
        try:
            campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
            if not campaign:
                return
            _campaign_heartbeat(db, campaign_id)

            # --- Resume / startup reconciliation (BUG #10/#11) ---
            # _run_campaign is always invoked with resume=True (Start button + app
            # startup). The old code had a duplicated, unreachable non-resume branch.
            # If the thread died mid-publish (current_phase=='publishing') or we're
            # resuming on app start, look at the current batch's successful posts:
            #   - within the live window → resume the wait, do NOT re-publish
            #   - expired                → delete the batch, then loop
            #   - no live posts          → fall through to a fresh publish
            batch = campaign["current_batch"] or 0
            if batch > 0:
                first_post = db.execute(
                    "SELECT MIN(posted_at) AS fp FROM turbo_campaign_posts WHERE campaign_id=? "
                    "AND batch_number=? AND post_status='success' AND COALESCE(delete_status,'') != 'deleted'",
                    (campaign_id, batch)).fetchone()
                first_dt = None
                if first_post and first_post["fp"]:
                    try:
                        first_dt = datetime.fromisoformat(first_post["fp"])
                    except (ValueError, TypeError):
                        first_dt = None
                if first_dt:
                    elapsed = (datetime.utcnow() - first_dt).total_seconds()
                    wait_seconds = _campaign_wait_seconds(campaign["timeframe_hours"])
                    if elapsed < wait_seconds:
                        db.execute("UPDATE turbo_campaigns SET current_phase='live' WHERE id=?", (campaign_id,))
                        db.commit()
                        _campaign_log(db, campaign_id, "live", f"resume batch {batch}: {(wait_seconds - elapsed)/60:.0f} min remaining", batch_number=batch)
                        if not _campaign_wait_live(campaign_id, batch, first_dt):
                            return
                    db.execute("UPDATE turbo_campaigns SET current_phase='deleting' WHERE id=?", (campaign_id,))
                    db.commit()
                    try:
                        _campaign_delete_batch(campaign_id, batch)
                    except Exception as e:
                        _campaign_set_error(db, campaign_id, f"delete batch {batch}: {e}", phase="deleting")
                        logging.getLogger(__name__).exception(f"Campaign {campaign_id} delete crashed: {e}")
                else:
                    # No live posts in current batch — clean up any stale ones.
                    db.execute("UPDATE turbo_campaigns SET current_phase='cleanup' WHERE id=?", (campaign_id,))
                    db.commit()
                    try:
                        _campaign_delete_previous_batch(db, campaign_id)
                    except Exception as e:
                        _campaign_set_error(db, campaign_id, f"cleanup: {e}", phase="cleanup")
                        logging.getLogger(__name__).exception(f"Campaign {campaign_id} cleanup crashed: {e}")

            # --- Normal cycle: publish → wait → delete → repeat ---
            while True:
                db = get_db()
                campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
                if not campaign or campaign["status"] != "running":
                    break
                _campaign_heartbeat(db, campaign_id)
                batch = campaign["current_batch"] + 1
                db.execute("UPDATE turbo_campaigns SET current_batch=?, current_phase='publishing' WHERE id=?", (batch, campaign_id))
                db.commit()
                try:
                    _campaign_publish_batch(campaign_id, batch)
                except Exception as e:
                    # BUG #5: a publish crash no longer kills the thread silently.
                    _campaign_set_error(db, campaign_id, f"publish batch {batch}: {e}", phase="publishing")
                    logging.getLogger(__name__).exception(f"Campaign {campaign_id} publish crashed: {e}")
                    # Fall through: whatever did get posted still cycles; if nothing
                    # posted, the no-first_dt path skips the wait and retries.
                db.execute("UPDATE turbo_campaigns SET current_phase='live' WHERE id=?", (campaign_id,))
                db.commit()
                first_post = db.execute(
                    "SELECT MIN(posted_at) AS fp FROM turbo_campaign_posts WHERE campaign_id=? AND batch_number=? AND post_status='success'",
                    (campaign_id, batch)).fetchone()
                first_dt = None
                if first_post and first_post["fp"]:
                    try:
                        first_dt = datetime.fromisoformat(first_post["fp"])
                    except (ValueError, TypeError):
                        pass
                if not first_dt:
                    # Nothing posted this batch (every vehicle failed build_payload or no
                    # user could post). BACK OFF before retrying — without this the engine
                    # tight-loops failed batches as fast as it can call build_payload/submit
                    # (hundreds of thousands of empty batches). 120s is short enough to
                    # resume quickly once a slot frees / a failing vehicle is removed.
                    _campaign_log(db, campaign_id, "publish", f"batch {batch}: 0 successes — backing off 120s", batch_number=batch, level="warning")
                    db.execute("UPDATE turbo_campaigns SET current_phase='backoff' WHERE id=?", (campaign_id,))
                    db.commit()
                    try:
                        _campaign_delete_batch(campaign_id, batch)
                    except Exception as e:
                        _campaign_set_error(db, campaign_id, f"delete batch {batch}: {e}", phase="deleting")
                        logging.getLogger(__name__).exception(f"Campaign {campaign_id} delete crashed: {e}")
                    stopped = False
                    for _ in range(12):  # 12 × 10s = 120s, interruptible by Stop
                        sc = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
                        if not sc or sc["status"] != "running":
                            stopped = True
                            break
                        time.sleep(10)
                    if stopped:
                        return
                    continue
                if not _campaign_wait_live(campaign_id, batch, first_dt):
                    return
                db.execute("UPDATE turbo_campaigns SET current_phase='deleting' WHERE id=?", (campaign_id,))
                db.commit()
                try:
                    _campaign_delete_batch(campaign_id, batch)
                except Exception as e:
                    _campaign_set_error(db, campaign_id, f"delete batch {batch}: {e}", phase="deleting")
                    logging.getLogger(__name__).exception(f"Campaign {campaign_id} delete crashed: {e}")
        except Exception as e:
            # Top-level guard: the thread must never die silently (BUG #5).
            try:
                _campaign_set_error(get_db(), campaign_id, f"run_campaign crashed: {e}")
            except Exception:
                pass
            logging.getLogger(__name__).exception(f"Campaign {campaign_id} thread crashed: {e}")


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns")
@login_required
def admin_turbo_campaigns():
    db = get_db()
    campaigns = db.execute("SELECT * FROM turbo_campaigns ORDER BY id DESC").fetchall()
    profiles = db.execute("SELECT id, name, email, flagged, flagged_reason FROM turbo_session_profiles ORDER BY name").fetchall()
    listings = db.execute("SELECT id, year, make, model, trim, vin FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY year DESC, make, model").fetchall()
    return render_template("admin/turbo_campaigns.html", campaigns=campaigns, profiles=profiles, listings=listings)


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/create", methods=("POST",))
@login_required
def admin_turbo_campaign_create():
    db = get_db()
    name = (request.form.get("name") or "").strip() or "Campaign"
    timeframe = max(0.5, min(48.0, float(request.form.get("timeframe_hours") or 5)))
    location_mode = "ai" if request.form.get("location_mode") == "ai" else "random"
    extended = 1 if request.form.get("extended_mode") else 0
    always_no_vin = 1 if request.form.get("always_no_vin") else 0
    cur = db.execute("INSERT INTO turbo_campaigns (name, status, timeframe_hours, location_mode, extended_mode, always_no_vin, created_at) VALUES (?,?,?,?,?,?,?)", (name, "stopped", timeframe, location_mode, extended, always_no_vin, now_utc()))
    cid = cur.lastrowid
    for pid in request.form.getlist("profile_ids"):
        if pid.isdigit():
            db.execute("INSERT INTO turbo_campaign_users (campaign_id, profile_id) VALUES (?,?)", (cid, int(pid)))
    for vid in request.form.getlist("vehicle_ids"):
        if vid.isdigit():
            db.execute("INSERT INTO turbo_campaign_vehicles (campaign_id, listing_id) VALUES (?,?)", (cid, int(vid)))
    db.commit()
    flash(f"Campaign '{name}' created.", "success")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>")
@login_required
def admin_turbo_campaign_detail(cid):
    db = get_db()
    campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (cid,)).fetchone()
    if not campaign:
        abort(404)
    users = db.execute("SELECT tcu.*, tsp.name AS profile_name, tsp.email, tsp.flagged, tsp.flagged_reason, tsp.paused_until, tsp.paused_reason FROM turbo_campaign_users tcu JOIN turbo_session_profiles tsp ON tsp.id=tcu.profile_id WHERE tcu.campaign_id=?", (cid,)).fetchall()
    available_profiles = db.execute("SELECT id, name, email, flagged, flagged_reason FROM turbo_session_profiles WHERE id NOT IN (SELECT profile_id FROM turbo_campaign_users WHERE campaign_id=?) ORDER BY name", (cid,)).fetchall()
    vehicles = db.execute("SELECT tcv.*, l.year, l.make, l.model, l.trim, l.vin FROM turbo_campaign_vehicles tcv JOIN listings l ON l.id=tcv.listing_id WHERE tcv.campaign_id=?", (cid,)).fetchall()
    posts = db.execute("SELECT tcp.*, l.year, l.make, l.model, tsp.name AS profile_name FROM turbo_campaign_posts tcp LEFT JOIN listings l ON l.id=tcp.listing_id LEFT JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id WHERE tcp.campaign_id=? ORDER BY tcp.id DESC LIMIT 200", (cid,)).fetchall()
    failed_count = db.execute("SELECT COUNT(*) FROM turbo_campaign_posts WHERE campaign_id=? AND post_status='failed'", (cid,)).fetchone()[0]
    activity_log = db.execute(
        "SELECT * FROM turbo_campaign_log WHERE campaign_id=? ORDER BY id DESC LIMIT 40", (cid,)).fetchall()
    # VIN cooldown / excluded (global) — surfaced so the admin sees which VINs are being
    # posted without a VIN and why (INACTIVE reason), and which are permanently excluded.
    vin_cooldown = db.execute(
        "SELECT vin, inactive_count, last_inactive_at, excluded, reason FROM turbo_vin_cooldown "
        "ORDER BY excluded DESC, last_inactive_at DESC LIMIT 100").fetchall()
    now_iso = now_utc()
    timeframe_seconds = int(float(campaign["timeframe_hours"] or 5) * 3600)
    return render_template("admin/turbo_campaign_detail.html", campaign=campaign, users=users, vehicles=vehicles, posts=posts, available_profiles=available_profiles, failed_count=failed_count, activity_log=activity_log, vin_cooldown=vin_cooldown, now_iso=now_iso, timeframe_seconds=timeframe_seconds)


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/export-posts")
@login_required
def admin_turbo_campaign_export_posts(cid):
    """Generate the campaign live-status snapshot as a tab-separated .txt on demand
    (improvement #7: the file is always derivable from the live DB, never stale)."""
    db = get_db()
    campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (cid,)).fetchone()
    if not campaign:
        abort(404)
    posts = db.execute(
        "SELECT tcp.*, l.year, l.make, l.model, l.vin AS inv_vin, tsp.email "
        "FROM turbo_campaign_posts tcp LEFT JOIN listings l ON l.id=tcp.listing_id "
        "LEFT JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id "
        "WHERE tcp.campaign_id=? ORDER BY tcp.id DESC LIMIT 200", (cid,)).fetchall()
    wait_seconds = _campaign_wait_seconds(campaign["timeframe_hours"])
    now = datetime.utcnow()
    lines = ["Posts (last 200)",
             "Batch\tVehicle\tAccount\tVIN\tProv\tCity\tName\tAT Listing ID\tPosted\tDelete in\tPost\tDeleted\tDel"]
    for p in posts:
        vehicle = " ".join(str(x) for x in (p["year"], p["make"], p["model"]) if x) or "(unknown)"
        name = " ".join(x for x in (p["first_name"], p["last_name"]) if x)
        posted = p["posted_at"] or ""
        delete_in = "—"
        if p["post_status"] == "success" and p["posted_at"] and (p["delete_status"] or "") != "deleted":
            try:
                pdt = datetime.fromisoformat(p["posted_at"])
                remaining = wait_seconds - (now - pdt).total_seconds()
                if remaining > 0:
                    h, rem = divmod(int(remaining), 3600)
                    m, s = divmod(rem, 60)
                    delete_in = f"{h}h {m}m {s}s"
                else:
                    delete_in = "expired"
            except (ValueError, TypeError):
                delete_in = "—"
        lines.append("\t".join(str(x) if x is not None else "" for x in (
            p["batch_number"], vehicle, p["email"] or "", p["vin"] or p["inv_vin"] or "",
            p["province"] or "", p["city"] or "", name, p["autotrader_listing_id"] or "",
            posted, delete_in, p["post_status"] or "", p["deleted_at"] or "", p["delete_status"] or "",
        )))
    body = "\n".join(lines)
    return body, 200, {
        "Content-Type": "text/plain; charset=utf-8",
        "Content-Disposition": f"attachment; filename=turbo_listings_campaign_posts.txt",
    }


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/start", methods=("POST",))
@login_required
def admin_turbo_campaign_start(cid):
    db = get_db()
    db.execute("UPDATE turbo_campaigns SET status='running', started_at=?, current_phase='starting' WHERE id=?", (now_utc(), cid))
    db.commit()
    t = threading.Thread(target=_run_campaign, args=(cid,), kwargs={"resume": True}, daemon=True)
    _turbo_campaign_threads[cid] = t
    t.start()
    flash("Campaign started.", "success")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/stop", methods=("POST",))
@login_required
def admin_turbo_campaign_stop(cid):
    db = get_db()
    db.execute("UPDATE turbo_campaigns SET status='stopped', stopped_at=?, current_phase='stopped' WHERE id=?", (now_utc(), cid))
    db.commit()
    flash("Campaign stopping (will finish current action then halt).", "warning")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/update-hours", methods=("POST",))
@login_required
def admin_turbo_campaign_update_hours(cid):
    db = get_db()
    try:
        hours = max(0.5, min(48.0, float(request.form.get("timeframe_hours") or 3)))
    except ValueError:
        hours = 3.0
    db.execute("UPDATE turbo_campaigns SET timeframe_hours=? WHERE id=?", (hours, cid))
    db.commit()
    flash(f"Timeframe updated to {hours} hours (applies from next wait phase).", "success")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/toggle-novin", methods=("POST",))
@login_required
def admin_turbo_campaign_toggle_novin(cid):
    """Toggle the 'Always no-VIN' setting on a campaign. When ON, the campaign never
    includes the VIN (overrules the cooldown logic). When OFF, the cooldown logic runs."""
    db = get_db()
    row = db.execute("SELECT always_no_vin FROM turbo_campaigns WHERE id=?", (cid,)).fetchone()
    new_val = 0 if (row and row["always_no_vin"]) else 1
    db.execute("UPDATE turbo_campaigns SET always_no_vin=? WHERE id=?", (new_val, cid))
    db.commit()
    flash(f"Always no-VIN: {'ON — VIN never included' if new_val else 'OFF — cooldown logic active'}.", "success" if new_val else "warning")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/add-user", methods=("POST",))
@login_required
def admin_turbo_campaign_add_user(cid):
    db = get_db()
    profile_id = (request.form.get("profile_id") or "").strip()
    if profile_id.isdigit():
        existing = db.execute("SELECT id FROM turbo_campaign_users WHERE campaign_id=? AND profile_id=?", (cid, int(profile_id))).fetchone()
        if not existing:
            db.execute("INSERT INTO turbo_campaign_users (campaign_id, profile_id) VALUES (?,?)", (cid, int(profile_id)))
            db.commit()
            # Auto-retrieve + delete existing listings for this profile
            _auto_delete_profile_listings(db, int(profile_id))
            campaign = db.execute("SELECT status, current_batch FROM turbo_campaigns WHERE id=?", (cid,)).fetchone()
            if campaign and campaign["status"] == "running":
                # Campaign is running — kick off an immediate publish for this user in a separate thread
                batch = campaign["current_batch"] or 1
                def _publish_new_user():
                    with app.app_context():
                        _campaign_publish_for_user(cid, int(profile_id), batch)
                threading.Thread(target=_publish_new_user, daemon=True).start()
                flash("User added + existing listings deleted. Publishing started for this user immediately.", "success")
            else:
                flash("User added. Existing listings auto-retrieved and deletion attempted.", "success")
        else:
            flash("User already in this campaign.", "warning")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


def _campaign_publish_for_user(campaign_id, profile_id, batch_number):
    """Publish listings for a single newly-added user (called in a separate thread).
    Delegates to _campaign_publish_one — the SAME per-post logic as the batch fill +
    topup (duplicate-VIN skip, PATCH-once, reconcile, 422-pause with accurate reset,
    3-fail-flag) — so the add-user path can't drift from the main path. Serialized by
    the per-campaign lock (BUG #7)."""
    try:
        with _campaign_lock(campaign_id):
            db = get_db()
            campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
            if not campaign:
                return
            u = db.execute("SELECT tcu.*, tsp.powershell_encrypted, tsp.proxy_encrypted, tsp.email FROM turbo_campaign_users tcu JOIN turbo_session_profiles tsp ON tsp.id=tcu.profile_id WHERE tcu.campaign_id=? AND tcu.profile_id=? AND COALESCE(tsp.flagged,0)=0", (campaign_id, profile_id)).fetchone()
            if not u:
                return
            vehicle_ids = [r["listing_id"] for r in db.execute("SELECT listing_id FROM turbo_campaign_vehicles WHERE campaign_id=?", (campaign_id,)).fetchall()]
            import random as _r
            _r.shuffle(vehicle_ids)
            last_post_by_user = {}
            patched_profiles = set()
            reconciled_profiles = set()
            for vid in vehicle_ids:
                c = get_db().execute("SELECT status FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
                if not c or c["status"] != "running":
                    return
                status = _campaign_publish_one(db, campaign, u, vid, batch_number,
                                               last_post_by_user, patched_profiles, reconciled_profiles)
                if status in ("paused", "flagged"):
                    break  # profile paused/flagged mid-batch — stop posting for it
    except Exception as e:
        logging.getLogger(__name__).error(f"Campaign {campaign_id} publish-for-user crashed: {e}")


_turbo_watchdog_started = False


def turbo_campaign_watchdog():
    """Lightweight watchdog (improvement #1): if a campaign is status='running' but its
    engine thread is no longer alive (crashed), restart it. Pairs with the heartbeat +
    error-phase columns so a dead campaign is detected and recovered without an app
    restart. Runs forever on a daemon thread."""
    while True:
        try:
            with app.app_context():
                db = get_db()
                running = db.execute("SELECT id, name FROM turbo_campaigns WHERE status='running'").fetchall()
                for c in running:
                    cid = c["id"]
                    t = _turbo_campaign_threads.get(cid)
                    if t is None or not t.is_alive():
                        logging.getLogger(__name__).warning(f"Watchdog: campaign {cid} ({c['name']}) thread dead — restarting.")
                        _campaign_log(db, cid, "watchdog", "thread dead — restarting", level="warning")
                        nt = threading.Thread(target=_run_campaign, args=(cid,), kwargs={"resume": True}, daemon=True)
                        _turbo_campaign_threads[cid] = nt
                        nt.start()
        except Exception as e:
            logging.getLogger(__name__).warning(f"turbo_campaign_watchdog tick failed: {e}")
        time.sleep(300)


def resume_turbo_campaigns():
    """On app startup, resume any campaigns that were 'running' when the app died, and
    start the watchdog (improvement #1). Passes resume=True so the engine checks the
    current batch's timestamps and resumes the wait (without deleting) if time hasn't
    expired."""
    global _turbo_watchdog_started
    try:
        with app.app_context():
            db = get_db()
            running = db.execute("SELECT id FROM turbo_campaigns WHERE status='running'").fetchall()
            for c in running:
                cid = c["id"]
                db.execute("UPDATE turbo_campaigns SET current_phase='resuming' WHERE id=?", (cid,))
                db.commit()
                t = threading.Thread(target=_run_campaign, args=(cid,), kwargs={"resume": True}, daemon=True)
                _turbo_campaign_threads[cid] = t
                t.start()
                logging.getLogger(__name__).info(f"Resumed campaign {cid} on startup (will check remaining live time).")
        if not _turbo_watchdog_started:
            _turbo_watchdog_started = True
            threading.Thread(target=turbo_campaign_watchdog, daemon=True).start()
    except Exception as e:
        logging.getLogger(__name__).warning(f"resume_turbo_campaigns failed: {e}")


def _campaign_delete_user_posts(db, campaign_id, profile_id):
    """Delete ONLY this campaign's live AutoTrader listings for the given (already-
    removed) user, and mark those posts deleted. Campaign-scoped: it does NOT touch the
    profile's listings in other campaigns. (The earlier remove-user reused the
    account-wide _auto_delete_profile_listings, which retrieved + deleted EVERY live
    listing on the account and marked posts across ALL campaigns — nuking other
    campaigns' listings too.) Best-effort: never raises."""
    try:
        campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (campaign_id,)).fetchone()
        row = db.execute("SELECT powershell_encrypted, proxy_encrypted FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
        if not row or not row["powershell_encrypted"]:
            db.execute(
                "UPDATE turbo_campaign_posts SET delete_status='deleted', deleted_at=?, delete_error='marked (no session)' "
                "WHERE campaign_id=? AND profile_id=? AND post_status='success' AND COALESCE(delete_status,'') != 'deleted'",
                (now_utc(), campaign_id, profile_id))
            db.commit()
            return
        try:
            ps_text = decrypt_secret(row["powershell_encrypted"])
        except Exception:
            ps_text = ""
        proxy = ""
        if row["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(row["proxy_encrypted"])
            except Exception:
                proxy = ""
        posts = db.execute(
            "SELECT id, autotrader_listing_id FROM turbo_campaign_posts "
            "WHERE campaign_id=? AND profile_id=? AND post_status='success' "
            "AND autotrader_listing_id != '' AND COALESCE(delete_status,'') NOT IN ('deleted')",
            (campaign_id, profile_id)).fetchall()
        for p in posts:
            result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
            del_status = "deleted" if result.get("ok") else "delete_failed"
            del_err = (result.get("error") or "")[:500] if not result.get("ok") else None
            db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status=?, delete_error=? WHERE id=?",
                       (now_utc(), del_status, del_err, p["id"]))
            if result.get("ok"):
                db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (p["autotrader_listing_id"],))
            db.commit()
            _campaign_log(db, campaign_id, "remove-user",
                          f"{del_status} post {p['id']} ({p['autotrader_listing_id']})",
                          profile_id=profile_id, level="info" if result.get("ok") else "warning")
            if campaign:
                _campaign_delete_sleep(campaign)
    except Exception as e:
        logging.getLogger(__name__).exception(f"_campaign_delete_user_posts c{campaign_id} p{profile_id}: {e}")


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/remove-user/<int:profile_id>", methods=("POST",))
@login_required
def admin_turbo_campaign_remove_user(cid, profile_id):
    db = get_db()
    # Remove from THIS campaign only (other campaigns keep the user).
    db.execute("DELETE FROM turbo_campaign_users WHERE campaign_id=? AND profile_id=?", (cid, profile_id))
    # Mark THIS campaign's live posts for the user as 'deleting' so the active-count
    # unblocks right away (the actual AutoTrader delete runs in the background).
    db.execute(
        "UPDATE turbo_campaign_posts SET delete_status='deleting' "
        "WHERE campaign_id=? AND profile_id=? AND post_status='success' "
        "AND autotrader_listing_id != '' AND COALESCE(delete_status,'') NOT IN ('deleted','deleting')",
        (cid, profile_id))
    db.commit()

    # Background: delete only THIS campaign's tracked listings + mark its posts deleted.
    # Campaign-scoped (BUG #2 fix, revised): must NOT delete the profile's listings in
    # other campaigns. API deletes take minutes, so they run on a daemon thread.
    def _cleanup_removed_user():
        with app.app_context():
            cdb = get_db()
            _campaign_delete_user_posts(cdb, cid, profile_id)
            _campaign_log(cdb, cid, "remove-user", f"removed profile {profile_id} from campaign", profile_id=profile_id)

    threading.Thread(target=_cleanup_removed_user, daemon=True).start()
    flash("User removed from this campaign. This campaign's listings are being deleted; other campaigns are untouched.", "success")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/flag", methods=("POST",))
@login_required
def admin_turbo_profile_flag(profile_id):
    db = get_db()
    action = (request.form.get("flag_action") or "").strip()
    if action == "flag":
        reason = (request.form.get("reason") or "Manually flagged").strip()[:300]
        db.execute("UPDATE turbo_session_profiles SET flagged=1, flagged_reason=? WHERE id=?", (reason, profile_id))
        db.execute("DELETE FROM turbo_campaign_users WHERE profile_id=?", (profile_id,))
        db.commit()
        flash("Profile flagged and removed from all campaigns.", "warning")
    elif action == "unflag":
        db.execute("UPDATE turbo_session_profiles SET flagged=0, flagged_reason=NULL, paused_until=NULL, paused_reason=NULL WHERE id=?", (profile_id,))
        db.commit()
        flash("Profile unflagged.", "success")
    elif action == "unpause":
        # Clear a 422/3-fail cooldown so the profile can post again immediately
        # (admin refreshed cookies / resolved the issue).
        db.execute("UPDATE turbo_session_profiles SET paused_until=NULL, paused_reason=NULL WHERE id=?", (profile_id,))
        db.commit()
        flash("Profile unpaused.", "success")
    elif action == "delete":
        db.execute("DELETE FROM turbo_campaign_users WHERE profile_id=?", (profile_id,))
        db.execute("DELETE FROM turbo_session_profiles WHERE id=?", (profile_id,))
        db.commit()
        flash("Profile deleted permanently (removed from all campaigns).", "success")
    return redirect(request.referrer or url_for("admin_turbo_campaigns"))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/posts/<int:post_id>/retry-delete", methods=("POST",))
@login_required
def admin_turbo_campaign_retry_delete(post_id):
    """Retry deleting a specific post's listing (useful after refreshing cookies)."""
    db = get_db()
    p = db.execute("SELECT tcp.*, tsp.powershell_encrypted, tsp.proxy_encrypted, tsp.refresh_cookies_encrypted FROM turbo_campaign_posts tcp JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id WHERE tcp.id=?", (post_id,)).fetchone()
    if not p:
        flash("Post not found.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    if not p["autotrader_listing_id"]:
        flash("No listing ID to delete.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    # Try refresh cookies first if available, else use original
    ps_text = ""
    if p["refresh_cookies_encrypted"]:
        try:
            ps_text = decrypt_secret(p["refresh_cookies_encrypted"])
        except Exception:
            ps_text = ""
    if not ps_text and p["powershell_encrypted"]:
        try:
            ps_text = decrypt_secret(p["powershell_encrypted"])
        except Exception:
            ps_text = ""
    if not ps_text:
        flash("No session available.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    proxy = ""
    if p["proxy_encrypted"]:
        try:
            proxy = decrypt_secret(p["proxy_encrypted"])
        except Exception:
            proxy = ""
    # Try stored UUID delete
    result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
    if not result.get("ok"):
        # Fallback: retrieve + delete all
        uuids, ret_err = turbo_listing.retrieve_listings(ps_text, proxy or None)
        if uuids:
            all_ok = True
            for uid in uuids:
                dr = turbo_listing.submit_listing_delete(ps_text, uid, proxy or None)
                if not dr.get("ok"):
                    all_ok = False
                import time as _t
                _t.sleep(10)
            result = {"ok": all_ok, "error": "Deleted via retrieve-fallback" if all_ok else "Some deletes failed"}
    del_status = "deleted" if result.get("ok") else "delete_failed"
    del_err = (result.get("error") or "")[:500] if not result.get("ok") else None
    db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status=?, delete_error=? WHERE id=?",
               (now_utc(), del_status, del_err, post_id))
    if result.get("ok"):
        db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (p["autotrader_listing_id"],))
    db.commit()
    flash("Delete retried: " + ("success" if result.get("ok") else f"failed — {result.get('error')}"), "success" if result.get("ok") else "warning")
    return redirect(request.referrer or url_for("admin_turbo_campaigns"))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/paste-cookies", methods=("POST",))
@login_required
def admin_turbo_profile_paste_cookies(profile_id):
    """Paste fresh cookies — accepts BOTH formats:
    1. Raw: name=value; name=value; ...
    2. PowerShell: $session.Cookies.Add((New-Object System.Net.Cookie("name", "value", "/", "domain")))
    Merges into the existing PowerShell session — replaces ONLY cookies, keeps headers/UA/Body intact."""
    db = get_db()
    row = db.execute("SELECT powershell_encrypted, proxy_encrypted FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if not row:
        flash("Profile not found.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    cookie_text = (request.form.get("cookie_text") or "").strip()
    if not cookie_text:
        flash("No cookie text pasted.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    try:
        ps_text = decrypt_secret(row["powershell_encrypted"]) if row["powershell_encrypted"] else ""
    except Exception:
        ps_text = ""
    if not ps_text:
        flash("No existing session to merge into. Save a full PowerShell request first.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))

    # Parse pasted cookies — detect format
    fresh_cookies = {}
    if "System.Net.Cookie" in cookie_text or "$session.Cookies.Add" in cookie_text:
        for m in re.finditer(r'System\.Net\.Cookie\(\s*"((?:[^"`]|`.)*)"\s*,\s*"((?:[^"`]|`.)*)"', cookie_text):
            name = turbo_listing._ps_unescape(m.group(1))
            value = turbo_listing._ps_unescape(m.group(2))
            if name:
                fresh_cookies[name] = value
    else:
        for part in re.split(r'[;\n]', cookie_text):
            part = part.strip()
            if "=" in part:
                name, _, value = part.partition("=")
                name = name.strip()
                value = value.strip()
                if name:
                    fresh_cookies[name] = value

    if not fresh_cookies:
        flash("Could not parse any cookies from the pasted text.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))

    # Merge: original cookies overridden by fresh ones
    merged = {}
    orig_ua = ""
    try:
        parsed = turbo_listing.parse_powershell_request(ps_text)
        for c in parsed.get("cookies") or []:
            merged[c["name"]] = c["value"]
        orig_ua = parsed.get("user_agent") or ""
    except Exception:
        pass
    merged.update(fresh_cookies)

    # Rebuild: keep original Invoke-WebRequest block (headers/Body), replace only cookies
    lines = ps_text.splitlines()
    invoke_start = None
    for i, line in enumerate(lines):
        if "Invoke-WebRequest" in line:
            invoke_start = i
            break
    new_lines = ["$session = New-Object Microsoft.PowerShell.Commands.WebRequestSession"]
    if orig_ua:
        new_lines.append(f'$session.UserAgent = "{orig_ua}"')
    for name, value in merged.items():
        esc_val = value.replace('"', '`"')
        new_lines.append(f'$session.Cookies.Add((New-Object System.Net.Cookie("{name}", "{esc_val}", "/", "www.autotrader.ca")))')
    if invoke_start is not None:
        new_lines.extend(lines[invoke_start:])
    else:
        new_lines.append('Invoke-WebRequest -UseBasicParsing -Uri "https://www.autotrader.ca/manual-listing-creation/api/private/listings" -Method "POST" -WebSession $session -ContentType "application/json" -Body "{}"')
    new_ps = "\n".join(new_lines)
    try:
        enc = encrypt_secret(new_ps)
        db.execute("UPDATE turbo_session_profiles SET powershell_encrypted=?, updated_at=? WHERE id=?", (enc, now_utc(), profile_id))
        db.commit()
        flash(f"Cookies updated: {len(fresh_cookies)} fresh cookies merged ({len(merged)} total). Headers/UA/Body preserved.", "success")
    except RuntimeError as e:
        flash(str(e), "warning")
    return redirect(request.referrer or url_for("admin_turbo_campaigns"))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/refresh-cookies", methods=("POST",))
@login_required
def admin_turbo_profile_refresh_cookies(profile_id):
    """Refresh cookies by GETting /account/listings with the profile's session.
    Stores the refreshed PowerShell in refresh_cookies_encrypted."""
    db = get_db()
    row = db.execute("SELECT powershell_encrypted, proxy_encrypted FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if not row:
        flash("Profile not found.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    try:
        ps_text = decrypt_secret(row["powershell_encrypted"]) if row["powershell_encrypted"] else ""
    except Exception:
        ps_text = ""
    if not ps_text:
        flash("No saved PowerShell session for this profile.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_campaigns"))
    proxy = ""
    if row["proxy_encrypted"]:
        try:
            proxy = decrypt_secret(row["proxy_encrypted"])
        except Exception:
            proxy = ""
    refresh_ps, err = turbo_listing.refresh_cookies(ps_text, proxy or None)
    if err:
        # Fallback: try with a listing UUID from this profile's recent posts
        recent = db.execute("SELECT autotrader_listing_id FROM turbo_campaign_posts WHERE profile_id=? AND autotrader_listing_id != '' ORDER BY id DESC LIMIT 1", (profile_id,)).fetchone()
        listing_uuid = recent["autotrader_listing_id"] if recent else None
        if listing_uuid:
            refresh_ps, err = turbo_listing.refresh_cookies(ps_text, proxy or None, listing_uuid=listing_uuid)
    if err:
        flash(f"Cookie refresh failed: {err}", "warning")
    else:
        try:
            enc = encrypt_secret(refresh_ps)
            db.execute("UPDATE turbo_session_profiles SET refresh_cookies_encrypted=? WHERE id=?", (enc, profile_id))
            db.commit()
            flash("Cookies refreshed and stored. The refreshed session will be used as fallback on 403 errors.", "success")
        except RuntimeError as e:
            flash(str(e), "warning")
    return redirect(request.referrer or url_for("admin_turbo_campaigns"))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/mark-deleted", methods=("POST",))
@login_required
def admin_turbo_campaign_mark_deleted(cid):
    """Try to actually delete all delete_failed/stale posts first (using each profile's
    saved session). If the delete succeeds, mark as deleted. If it fails (403/expired),
    mark as deleted anyway so accounts get unblocked — the listing may already be gone."""
    db = get_db()
    posts = db.execute(
        "SELECT tcp.*, tsp.powershell_encrypted, tsp.proxy_encrypted FROM turbo_campaign_posts tcp "
        "JOIN turbo_session_profiles tsp ON tsp.id=tcp.profile_id "
        "WHERE tcp.campaign_id=? AND tcp.post_status='success' AND tcp.autotrader_listing_id != '' "
        "AND COALESCE(tcp.delete_status,'') NOT IN ('deleted')",
        (cid,)).fetchall()
    actually_deleted = 0
    marked = 0
    for p in posts:
        try:
            ps_text = decrypt_secret(p["powershell_encrypted"]) if p["powershell_encrypted"] else ""
        except Exception:
            ps_text = ""
        if not ps_text or not p["autotrader_listing_id"]:
            # Can't try — just mark as deleted
            db.execute("UPDATE turbo_campaign_posts SET delete_status='deleted', delete_error='marked (no session)' WHERE id=?", (p["id"],))
            marked += 1
            continue
        proxy = ""
        if p["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(p["proxy_encrypted"])
            except Exception:
                proxy = ""
        result = turbo_listing.submit_listing_delete(ps_text, p["autotrader_listing_id"], proxy or None)
        if result.get("ok"):
            db.execute("UPDATE turbo_campaign_posts SET deleted_at=?, delete_status='deleted', delete_error=NULL WHERE id=?", (now_utc(), p["id"]))
            db.execute("UPDATE turbo_posted SET status='deleted' WHERE autotrader_listing_id=?", (p["autotrader_listing_id"],))
            actually_deleted += 1
        else:
            # Delete failed — mark as deleted anyway to unblock the account
            db.execute("UPDATE turbo_campaign_posts SET delete_status='deleted', delete_error=? WHERE id=?",
                       (f"marked after failed retry: {(result.get('error') or '')[:300]}", p["id"]))
            marked += 1
        db.commit()
        time.sleep(3)
    flash(f"Tried {len(posts)} posts: {actually_deleted} actually deleted, {marked} marked (session expired). Accounts unblocked.", "success")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/force-advance", methods=("POST",))
@login_required
def admin_turbo_campaign_force_advance(cid):
    """Unstick a campaign stalled in 'deleting' (hung/expired-session deletes): mark the
    current batch's live posts as deleted so the delete loop skips them and the engine
    advances to the next batch (publish). Restarts the engine thread if it died. The
    listings may still be live on AutoTrader — run Retrieve+Delete / Mark all deleted to
    clean them up."""
    db = get_db()
    campaign = db.execute("SELECT * FROM turbo_campaigns WHERE id=?", (cid,)).fetchone()
    if not campaign:
        abort(404)
    batch = campaign["current_batch"] or 0
    n = db.execute(
        "UPDATE turbo_campaign_posts SET delete_status='deleted', deleted_at=?, delete_error='force-advanced by admin' "
        "WHERE campaign_id=? AND batch_number=? AND post_status='success' "
        "AND COALESCE(delete_status,'') != 'deleted'",
        (now_utc(), cid, batch)).rowcount
    db.execute("UPDATE turbo_campaigns SET current_phase='live', last_error=NULL WHERE id=?", (cid,))
    db.commit()
    _campaign_log(db, cid, "force-advance", f"marked {n} post(s) deleted in batch {batch}; advancing", batch_number=batch, level="warning")
    # Restart the engine thread if it died (so a stuck campaign recovers even without
    # the watchdog running).
    t = _turbo_campaign_threads.get(cid)
    if (t is None or not t.is_alive()) and campaign["status"] == "running":
        nt = threading.Thread(target=_run_campaign, args=(cid,), kwargs={"resume": True}, daemon=True)
        _turbo_campaign_threads[cid] = nt
        nt.start()
    flash(f"Force-advanced batch {batch}: {n} post(s) marked deleted. A new batch will publish shortly. Run Retrieve+Delete on affected accounts to clean up AutoTrader.", "warning")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/campaigns/<int:cid>/refresh-all-cookies", methods=("POST",))
@login_required
def admin_turbo_campaign_refresh_all_cookies(cid):
    """Refresh cookies for all users in a campaign."""
    db = get_db()
    users = db.execute("SELECT tcu.profile_id, tsp.powershell_encrypted, tsp.proxy_encrypted FROM turbo_campaign_users tcu JOIN turbo_session_profiles tsp ON tsp.id=tcu.profile_id WHERE tcu.campaign_id=?", (cid,)).fetchall()
    refreshed, failed = 0, 0
    for u in users:
        try:
            ps_text = decrypt_secret(u["powershell_encrypted"]) if u["powershell_encrypted"] else ""
        except Exception:
            ps_text = ""
        if not ps_text:
            failed += 1
            continue
        proxy = ""
        if u["proxy_encrypted"]:
            try:
                proxy = decrypt_secret(u["proxy_encrypted"])
            except Exception:
                proxy = ""
        refresh_ps, err = turbo_listing.refresh_cookies(ps_text, proxy or None)
        if err:
            failed += 1
        else:
            try:
                enc = encrypt_secret(refresh_ps)
                db.execute("UPDATE turbo_session_profiles SET refresh_cookies_encrypted=? WHERE id=?", (enc, u["profile_id"]))
                refreshed += 1
            except Exception:
                failed += 1
    db.commit()
    flash(f"Refreshed {refreshed} profile(s), {failed} failed.", "success" if refreshed else "warning")
    return redirect(url_for("admin_turbo_campaign_detail", cid=cid))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/retrieve-delete", methods=("POST",))
@login_required
def admin_turbo_profile_retrieve_delete(profile_id):
    """Retrieve current active listings for a profile and delete them all."""
    db = get_db()
    _auto_delete_profile_listings(db, profile_id)
    flash("Listings retrieved and deletion attempted.", "success")
    return redirect(request.referrer or url_for("admin_turbo_campaigns"))


def _deep_get(d, *paths):
    """Try multiple dotted paths into a nested dict; return the first non-empty value."""
    for path in paths:
        cur = d
        ok = True
        for part in path.split("."):
            if isinstance(cur, dict) and part in cur:
                cur = cur[part]
            else:
                ok = False
                break
        if ok and cur not in (None, "", [], {}):
            return cur
    return ""


def _get_profile_session(profile_id):
    """Return (ps_text, proxy, error_or_None) for a saved profile."""
    row = get_db().execute("SELECT powershell_encrypted, proxy_encrypted FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if not row or not row["powershell_encrypted"]:
        return "", "", "Profile not found or no saved session."
    try:
        ps_text = decrypt_secret(row["powershell_encrypted"])
    except Exception:
        return "", "", "Could not decrypt the saved session."
    proxy = ""
    if row["proxy_encrypted"]:
        try:
            proxy = decrypt_secret(row["proxy_encrypted"])
        except Exception:
            proxy = ""
    return ps_text, proxy, None


def _summarize_retrieved_listing(item):
    """Pull car/price/mileage/created_at out of a raw AutoTrader listing item. Schema
    confirmed from a real retrieve response: top-level fields make, model, modelLine
    (trim), price, mileageKm, status, bodyType, createdAt (posted-at ISO-8601 with Z).
    No year or location in the retrieve response."""
    if not isinstance(item, dict):
        return {"car": "", "year": "", "price": "", "mileage": "", "location": "", "created_at": ""}
    make = _deep_get(item, "make", "vehicleData.make", "makeName")
    model = _deep_get(item, "model", "vehicleData.model", "modelName")
    trim = _deep_get(item, "modelLine", "vehicleData.modelVersion", "modelVersion", "trim")
    # Year: firstRegistrationDate may contain a date; take the first 4 chars as the year.
    reg = _deep_get(item, "firstRegistrationDate", "vehicleData.modelYear", "modelYear", "year")
    year = str(reg)[:4] if reg else ""
    car = " ".join(str(p) for p in (year, make, model, trim) if p).strip()
    price = _deep_get(item, "price", "prices.public.price", "advertisedPrice")
    try:
        price = "${:,}".format(int(float(str(price)))) if str(price).strip() else ""
    except (ValueError, TypeError):
        price = str(price) if price else ""
    mileage = _deep_get(item, "mileageKm", "condition.mileage", "mileage", "odometer")
    try:
        mileage = f"{int(float(str(mileage)))} km" if str(mileage).strip() else ""
    except (ValueError, TypeError):
        mileage = str(mileage) if mileage else ""
    created = _deep_get(item, "createdAt", "created_at", "postedAt")
    return {"car": car, "year": year, "price": price, "mileage": mileage, "location": "", "created_at": str(created or "")}


def _retrieve_profile_listings(profile_id):
    """Shared by single-profile + check-all: retrieve + summarize. Returns (list, error)."""
    ps_text, proxy, err = _get_profile_session(profile_id)
    if err:
        return [], err
    listings, rerr = turbo_listing.retrieve_listings(ps_text, proxy or None)
    if rerr:
        return [], rerr
    out = []
    for item in listings:
        uid = item["uuid"] if isinstance(item, dict) else str(item)
        status = (item.get("status", "") if isinstance(item, dict) else "")
        summ = _summarize_retrieved_listing(item.get("raw") if isinstance(item, dict) else {})
        out.append({"uuid": uid, "status": status, **summ})
    return out, None


def _turbo_reconcile_profile_listings(profile_id, listings):
    """Reconcile a successful 'View current listings' retrieve against tracked posts:
    present+ACTIVE -> live, present+other -> inactive, absent -> removed. Any retrieved
    listing we never tracked (e.g. posted manually in the browser) is ADDED as a
    source='external' post — auto-assigned to a car via the Title→stock# map, or left
    unassigned for manual mapping. Returns a small summary dict."""
    db = get_db()
    now = now_utc()
    summary = {"live": 0, "inactive": 0, "removed": 0, "external_added": 0, "external_assigned": 0}
    tracked = db.execute(
        "SELECT id, autotrader_listing_id FROM turbo_campaign_posts "
        "WHERE profile_id=? AND COALESCE(autotrader_listing_id,'')!=''",
        (profile_id,)).fetchall()
    retrieved = {}
    for item in listings or []:
        uid = item.get("uuid") if isinstance(item, dict) else item
        if uid:
            retrieved[str(uid)] = item
    for row in tracked:
        item = retrieved.get(row["autotrader_listing_id"])
        if item is None:
            status = "removed"
            summary["removed"] += 1
        elif str(item.get("status") or "").strip().upper() == "ACTIVE":
            status = "live"
            summary["live"] += 1
        else:
            status = "inactive"
            summary["inactive"] += 1
        db.execute(
            "UPDATE turbo_campaign_posts SET live_status=?, live_checked_at=?, live_check_method='auto-retrieve', live_error=NULL WHERE id=?",
            (status, now, row["id"]))
    known_ids = {r["autotrader_listing_id"] for r in tracked}
    for uid, item in retrieved.items():
        if uid in known_ids:
            continue
        car_title = (item.get("car") or "").strip() if isinstance(item, dict) else ""
        price = (item.get("price") or "").strip() if isinstance(item, dict) else ""
        listing_id = None
        for candidate in (car_title, f"{car_title} {price}".strip()):
            if candidate:
                listing_id = _match_listing_by_title(candidate)
                if listing_id:
                    break
        created = (item.get("created_at") or "").strip() if isinstance(item, dict) else ""
        db.execute(
            "INSERT INTO turbo_campaign_posts (campaign_id, profile_id, listing_id, autotrader_listing_id, "
            "posted_at, post_status, source, live_status, live_checked_at, live_check_method, "
            "external_title, external_price, external_json) "
            "VALUES (NULL, ?, ?, ?, ?, 'success', 'external', ?, ?, 'auto-retrieve', ?, ?, ?)",
            (profile_id, listing_id, uid, created or now, "live", now, car_title, price,
             json.dumps({"status": item.get("status", ""), "car": car_title, "price": price,
                         "mileage": item.get("mileage", "")}, ensure_ascii=False)))
        summary["external_added"] += 1
        if listing_id:
            summary["external_assigned"] += 1
    db.commit()
    return summary


def _turbo_live_check_update(post_id, status, method, http_status=None, error=None):
    db = get_db()
    db.execute(
        "UPDATE turbo_campaign_posts SET live_status=?, live_checked_at=?, live_http=?, "
        "live_check_method=?, live_error=? WHERE id=?",
        (status, now_utc(), http_status, method, (str(error)[:500] if error else None), post_id),
    )
    db.commit()


def _turbo_check_post_live(post_id, prefer_retrieve=True):
    """Check one post's listing live status. Priority chain (user spec):
    1. 'View current listings' retrieve (source of truth — also validates the session)
    2. HTTP fetch of the offer page via the profile's SOCKS + cookies
    Returns the new live_status + detail."""
    db = get_db()
    post = db.execute("SELECT * FROM turbo_campaign_posts WHERE id=?", (post_id,)).fetchone()
    if not post:
        return {"live_status": "error", "error": "Post not found."}
    at_id = (post["autotrader_listing_id"] or "").strip()
    if not at_id:
        _turbo_live_check_update(post_id, "error", "none", None, "No AT listing id recorded for this post.")
        return {"live_status": "error", "error": "No AT listing id recorded."}
    ps_text, proxy, err = _get_profile_session(post["profile_id"])
    if err:
        _turbo_live_check_update(post_id, "error", "none", None, err)
        return {"live_status": "error", "error": err}
    retrieve_error = ""
    if prefer_retrieve:
        listings, rerr = turbo_listing.retrieve_listings(ps_text, proxy or None)
        if not rerr:
            found = None
            for item in listings or []:
                if str(item.get("uuid") or "") == at_id:
                    found = item
                    break
            status = "removed" if found is None else (
                "live" if str(found.get("status") or "").strip().upper() == "ACTIVE" else "inactive")
            _turbo_live_check_update(post_id, status, "auto-retrieve")
            return {"live_status": status}
        retrieve_error = f"retrieve: {rerr}"
    result = turbo_listing.check_listing_live(ps_text, at_id, proxy or None)
    if result.get("live") is True:
        _turbo_live_check_update(post_id, "live", "http", result.get("http_status"))
        return {"live_status": "live"}
    if result.get("live") is False:
        _turbo_live_check_update(post_id, "removed", "http", result.get("http_status"))
        return {"live_status": "removed"}
    msg = "; ".join(x for x in (retrieve_error, f"page check: {result.get('error') or 'unclear'}") if x)
    _turbo_live_check_update(post_id, "error", "http", result.get("http_status"), msg)
    return {"live_status": "error", "error": msg}


def _schedule_post_live_check(post_id, delay_seconds=180):
    """Auto-check a freshly posted listing a few minutes later: retrieve first
    (also proves the session still works), offer-page check as fallback."""
    def _run():
        time.sleep(max(10, int(delay_seconds)))
        with app.app_context():
            try:
                _turbo_check_post_live(post_id, prefer_retrieve=True)
            except Exception as e:
                logging.getLogger(__name__).warning(f"Live check for post {post_id} failed: {e}")
    threading.Thread(target=_run, daemon=True).start()


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/listings")
@login_required
def admin_turbo_profile_listings(profile_id):
    """Read-only: retrieve a profile's current AutoTrader listings and return each one's
    uuid, status, year/car, price, mileage, and location as JSON. Does NOT delete.
    Also reconciles the retrieve against tracked posts (live/removed/inactive +
    auto-adds untracked externally-posted listings)."""
    listings, err = _retrieve_profile_listings(profile_id)
    if err:
        return jsonify({"ok": False, "error": err})
    sync = _turbo_reconcile_profile_listings(profile_id, listings)
    return jsonify({"ok": True, "listings": listings, "sync": sync})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/<int:profile_id>/delete-listing", methods=("POST",))
@login_required
def admin_turbo_profile_delete_listing(profile_id):
    """Delete one AutoTrader listing by UUID using the profile's saved session."""
    listing_uuid = (request.form.get("listing_uuid") or "").strip()
    if not listing_uuid:
        return jsonify({"ok": False, "error": "No listing UUID."}), 400
    ps_text, proxy, err = _get_profile_session(profile_id)
    if err:
        return jsonify({"ok": False, "error": err})
    try:
        result = turbo_listing.submit_listing_delete(ps_text, listing_uuid, proxy or None)
        if result.get("ok"):
            return jsonify({"ok": True})
        return jsonify({"ok": False, "error": result.get("error") or "delete failed"})
    except Exception as e:
        return jsonify({"ok": False, "error": f"{type(e).__name__}: {e}"})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/profiles/listings-all")
@login_required
def admin_turbo_profiles_listings_all():
    """Check ALL saved profiles' current AutoTrader listings at once. Returns one entry
    per profile (name, email, error or listings list) for the grouped modal. Each
    successful retrieve is also reconciled against tracked posts (live/removed/
    inactive + auto-adds untracked externally-posted listings)."""
    db = get_db()
    profs = db.execute("SELECT id, name, email FROM turbo_session_profiles ORDER BY name").fetchall()
    accounts = []
    for p in profs:
        listings, err = _retrieve_profile_listings(p["id"])
        sync = _turbo_reconcile_profile_listings(p["id"], listings) if not err else None
        accounts.append({
            "profile_id": p["id"], "name": p["name"], "email": p["email"] or "",
            "error": err, "listings": listings, "sync": sync,
        })
    return jsonify({"ok": True, "accounts": accounts})


# Live-check batch job state (one at a time, like the crawler import job).
_turbo_live_check_jobs = {"current": None}


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/posts/<int:post_id>/check-live", methods=("POST",))
@login_required
def admin_turbo_post_check_live(post_id):
    """Check one post's listing now. Retrieve first (source of truth), offer-page
    check as fallback — the priority chain the admin asked for."""
    try:
        result = _turbo_check_post_live(post_id, prefer_retrieve=True)
        return jsonify({"ok": True, **result})
    except Exception as e:
        return jsonify({"ok": False, "error": f"{type(e).__name__}: {e}"}), 500


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/check-live/batch", methods=("POST",))
@login_required
def admin_turbo_check_live_batch():
    """Background live check for one account or all accounts. Per account: ONE
    retrieve reconciles all its posts (cheap); posts still without a verdict —
    or the whole account, when retrieve fails — fall back to individual
    offer-page checks through the profile's SOCKS, staggered apart."""
    profile_id = (request.form.get("profile_id") or "").strip()
    if profile_id == "all":
        profiles = [r["id"] for r in get_db().execute("SELECT id FROM turbo_session_profiles ORDER BY id").fetchall()]
        label = "all accounts"
    elif profile_id.isdigit():
        profiles = [int(profile_id)]
        label = f"account #{profile_id}"
    else:
        flash("Choose an account or 'all'.", "warning")
        return redirect(url_for("admin_turbo_tracking"))
    if _turbo_live_check_jobs.get("current") and not _turbo_live_check_jobs["current"].get("finished"):
        flash("A live check is already running — wait for it to finish.", "warning")
        return redirect(request.referrer or url_for("admin_turbo_tracking"))
    job = {"finished": False, "label": label, "profiles_total": len(profiles), "profiles_done": 0,
           "checked": 0, "live": 0, "removed": 0, "inactive": 0, "errors": 0, "log": []}
    _turbo_live_check_jobs["current"] = job

    def _run():
        with app.app_context():
            db = get_db()
            try:
                for pid in profiles:
                    posts = db.execute(
                        "SELECT id FROM turbo_campaign_posts WHERE profile_id=? "
                        "AND COALESCE(autotrader_listing_id,'')!='' AND COALESCE(delete_status,'')!='deleted'",
                        (pid,)).fetchall()
                    ps_text, proxy, err = _get_profile_session(pid)
                    if err:
                        job["log"].append(f"account #{pid}: no session ({err[:100]}) — skipped its {len(posts)} post(s)")
                        job["errors"] += len(posts)
                        job["profiles_done"] += 1
                        continue
                    listings, rerr = turbo_listing.retrieve_listings(ps_text, proxy or None)
                    if not rerr:
                        summary = _turbo_reconcile_profile_listings(pid, listings)
                        job["live"] += summary["live"]
                        job["removed"] += summary["removed"]
                        job["inactive"] += summary["inactive"]
                        job["checked"] += summary["live"] + summary["removed"] + summary["inactive"]
                        job["log"].append(
                            f"account #{pid}: retrieve OK — {summary['live']} live, {summary['removed']} removed, "
                            f"{summary['inactive']} inactive, +{summary['external_added']} external added")
                        reconciled_ids = {r["id"] for r in db.execute(
                            "SELECT id FROM turbo_campaign_posts WHERE profile_id=? "
                            "AND live_status IN ('live','removed','inactive')", (pid,)).fetchall()}
                    else:
                        job["log"].append(f"account #{pid}: retrieve failed ({rerr[:100]}) — page-checking its posts")
                        reconciled_ids = set()
                    for row in posts:
                        if row["id"] in reconciled_ids:
                            continue
                        res = _turbo_check_post_live(row["id"], prefer_retrieve=False)
                        status = res.get("live_status")
                        job["checked"] += 1
                        if status == "live":
                            job["live"] += 1
                        elif status == "removed":
                            job["removed"] += 1
                        elif status == "inactive":
                            job["inactive"] += 1
                        else:
                            job["errors"] += 1
                        time.sleep(3)
                    job["profiles_done"] += 1
            finally:
                job["finished"] = True

    threading.Thread(target=_run, daemon=True).start()
    flash(f"Live check started ({label}). Progress shows on the tracking page and dashboard.", "success")
    return redirect(request.referrer or url_for("admin_turbo_tracking"))


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/check-live/status")
@login_required
def admin_turbo_check_live_status():
    return jsonify(_turbo_live_check_jobs.get("current") or {"finished": True})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/posts/<int:post_id>/mark-live", methods=("POST",))
@login_required
def admin_turbo_post_mark_live(post_id):
    """Manual verdict from the open-in-browser fallback: the admin opened the offer
    page (solving any captcha) and reports whether the listing is there."""
    live = (request.form.get("live") or "").strip() == "1"
    db = get_db()
    db.execute(
        "UPDATE turbo_campaign_posts SET live_status=?, live_checked_at=?, live_check_method='manual', live_error=NULL WHERE id=?",
        ("live" if live else "removed", now_utc(), post_id))
    db.commit()
    return jsonify({"ok": True, "live_status": "live" if live else "removed"})


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/posts/<int:post_id>/assign", methods=("POST",))
@login_required
def admin_turbo_post_assign(post_id):
    """Assign an unassigned external post to one of our cars (same vehicle-search
    picker the lead campaigns use)."""
    db = get_db()
    listing_id = _resolve_vehicle_search(request.form.get("vehicle_search", ""))
    if not listing_id:
        flash("Pick a valid vehicle to assign this post to.", "warning")
        return redirect(url_for("admin_turbo_tracking"))
    db.execute("UPDATE turbo_campaign_posts SET listing_id=? WHERE id=?", (listing_id, post_id))
    db.commit()
    flash("Post assigned to that car.", "success")
    return redirect(url_for("admin_turbo_tracking"))


def _turbo_tracking_query(filters=None):
    """Shared query for the tracking page + CSV export. Returns (rows, params)."""
    where = ["1=1"]
    params = []
    filters = filters or {}
    if filters.get("q"):
        like = f"%{filters['q'].strip()}%"
        where.append("(l.title_en LIKE ? OR l.vin LIKE ? OR tcp.vin LIKE ? OR tcp.autotrader_listing_id LIKE ? OR tsp.email LIKE ? OR tcp.external_title LIKE ?)")
        params.extend([like, like, like, like, like, like])
    if filters.get("source"):
        where.append("tcp.source = ?")
        params.append(filters["source"])
    if filters.get("live"):
        where.append("tcp.live_status = ?")
        params.append(filters["live"])
    if filters.get("profile_id"):
        where.append("tcp.profile_id = ?")
        params.append(filters["profile_id"])
    if filters.get("campaign_id"):
        where.append("tcp.campaign_id = ?")
        params.append(filters["campaign_id"])
    sql = (
        "SELECT tcp.*, l.title_en, l.year, l.make, l.model, l.trim, l.public_stock_number, "
        "tsp.email AS account_email, tsp.name AS account_name "
        "FROM turbo_campaign_posts tcp "
        "LEFT JOIN listings l ON l.id = tcp.listing_id "
        "LEFT JOIN turbo_session_profiles tsp ON tsp.id = tcp.profile_id "
        f"WHERE {' AND '.join(where)} ORDER BY tcp.id DESC"
    )
    return sql, params


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/tracking")
@login_required
def admin_turbo_tracking():
    """Unified post tracking: every post (campaign / manual / external) with full
    response info, live status, clickable AutoTrader links, filters + CSV."""
    db = get_db()
    filters = {
        "q": (request.args.get("q") or "").strip(),
        "source": (request.args.get("source") or "").strip(),
        "live": (request.args.get("live") or "").strip(),
        "profile_id": (request.args.get("profile_id") or "").strip(),
        "campaign_id": (request.args.get("campaign_id") or "").strip(),
    }
    sql, params = _turbo_tracking_query(filters)
    rows = db.execute(sql + " LIMIT 300", params).fetchall()
    unassigned = db.execute(
        "SELECT tcp.*, tsp.email AS account_email FROM turbo_campaign_posts tcp "
        "LEFT JOIN turbo_session_profiles tsp ON tsp.id = tcp.profile_id "
        "WHERE tcp.source='external' AND tcp.listing_id IS NULL ORDER BY tcp.id DESC LIMIT 100").fetchall()
    profiles = db.execute("SELECT id, name, email FROM turbo_session_profiles ORDER BY name").fetchall()
    campaigns = db.execute("SELECT id, name FROM turbo_campaigns ORDER BY id DESC").fetchall()
    stats = db.execute(
        "SELECT COALESCE(live_status,'unchecked') AS st, COUNT(*) AS n FROM turbo_campaign_posts "
        "WHERE COALESCE(autotrader_listing_id,'')!='' GROUP BY st").fetchall()
    stats_map = {r["st"]: r["n"] for r in stats}
    listings = db.execute(
        "SELECT id, year, make, model, trim, vin FROM listings WHERE COALESCE(is_deleted,0)=0 "
        "ORDER BY year DESC, make, model").fetchall()
    return render_template(
        "admin/turbo_tracking.html", rows=rows, unassigned=unassigned, profiles=profiles,
        campaigns=campaigns, filters=filters, stats=stats_map, listings=listings,
        live_job=_turbo_live_check_jobs.get("current") or {"finished": True},
    )


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/tracking/csv")
@login_required
def admin_turbo_tracking_csv():
    """Full tracking export as CSV — every post, every column, no row cap."""
    db = get_db()
    filters = {
        "q": (request.args.get("q") or "").strip(),
        "source": (request.args.get("source") or "").strip(),
        "live": (request.args.get("live") or "").strip(),
        "profile_id": (request.args.get("profile_id") or "").strip(),
        "campaign_id": (request.args.get("campaign_id") or "").strip(),
    }
    sql, params = _turbo_tracking_query(filters)
    rows = db.execute(sql, params).fetchall()
    import csv as _csv
    import io as _io
    buf = _io.StringIO()
    writer = _csv.writer(buf)
    writer.writerow(["id", "source", "campaign", "batch", "posted_at", "car", "account", "province",
                     "city", "vin", "price_used", "at_listing_id", "at_url", "post_status",
                     "deleted_at", "live_status", "live_checked_at", "live_check_method",
                     "live_http", "live_error", "error"])
    for p in rows:
        car = " ".join(str(x) for x in (p["year"], p["make"], p["model"], p["trim"] or "") if x) or (p["external_title"] or "")
        writer.writerow([
            p["id"], p["source"] or "campaign", p["campaign_id"] or "", p["batch_number"] or "",
            p["posted_at"] or "", car, p["account_email"] or p["account_name"] or "",
            p["province"] or "", p["city"] or "", p["vin"] or "",
            p["price_used"] or p["external_price"] or "",
            p["autotrader_listing_id"] or "", turbo_listing.at_listing_url(p["autotrader_listing_id"]) if p["autotrader_listing_id"] else "",
            p["post_status"] or "", p["deleted_at"] or "", p["live_status"] or "unchecked",
            p["live_checked_at"] or "", p["live_check_method"] or "", p["live_http"] or "",
            p["live_error"] or "", p["error"] or "",
        ])
    body = buf.getvalue()
    return body, 200, {
        "Content-Type": "text/csv; charset=utf-8",
        "Content-Disposition": f"attachment; filename=turbo_post_tracking_{datetime.utcnow().strftime('%Y%m%d_%H%M')}.csv",
    }


@app.route(f"{ADMIN_URL_PREFIX}/turbo-listing/dashboard")
@login_required
def admin_turbo_dashboard():
    """Live listings dashboard: per car, its LIVE AutoTrader postings (location +
    account + clickable link) side by side with that car's leads. Only live posts
    count as current — removed/deleted ones drop out of the current view."""
    db = get_db()
    posts = db.execute(
        "SELECT tcp.*, l.year, l.make, l.model, l.trim, l.price AS car_price, l.public_stock_number, "
        "tsp.email AS account_email "
        "FROM turbo_campaign_posts tcp "
        "LEFT JOIN listings l ON l.id = tcp.listing_id "
        "LEFT JOIN turbo_session_profiles tsp ON tsp.id = tcp.profile_id "
        "WHERE COALESCE(tcp.autotrader_listing_id,'')!='' "
        "ORDER BY tcp.listing_id IS NULL, tcp.listing_id, tcp.id DESC LIMIT 1500").fetchall()
    lead_rows = db.execute(
        "SELECT lc.listing_id AS listing_id, COUNT(*) AS leads, SUM(COALESCE(cl.visited,0)) AS visits, "
        "SUM(CASE WHEN cl.reply_status='sent' THEN 1 ELSE 0 END) AS sent "
        "FROM campaign_links cl JOIN lead_campaigns lc ON lc.id = cl.campaign_id "
        "WHERE COALESCE(cl.duplicate,0)=0 AND lc.listing_id IS NOT NULL "
        "GROUP BY lc.listing_id").fetchall()
    leads_by_car = {r["listing_id"]: dict(r) for r in lead_rows}
    cars = []
    cars_by_id = {}
    for p in posts:
        key = p["listing_id"] if p["listing_id"] else f"unassigned-{p['id']}"
        if key not in cars_by_id:
            entry = {
                "listing_id": p["listing_id"],
                "car": (" ".join(str(x) for x in (p["year"], p["make"], p["model"], p["trim"] or "") if x)
                        or (p["external_title"] or "Unassigned external listing")),
                "public_ref": p["public_stock_number"] or "",
                "car_price": p["car_price"] or "",
                "leads": leads_by_car.get(p["listing_id"], {}) if p["listing_id"] else {},
                "live": [], "removed": 0, "inactive": 0, "unchecked": 0,
            }
            cars_by_id[key] = entry
            cars.append(entry)
        entry = cars_by_id[key]
        status = p["live_status"] or ""
        if p["delete_status"] == "deleted" or status == "removed":
            entry["removed"] += 1
        elif status == "live":
            entry["live"].append(p)
        elif status == "inactive":
            entry["inactive"] += 1
        else:
            entry["unchecked"] += 1
    live_total = sum(len(c["live"]) for c in cars)
    removed_total = sum(c["removed"] for c in cars)
    inactive_total = sum(c["inactive"] for c in cars)
    unchecked_total = sum(c["unchecked"] for c in cars)
    newest_check = db.execute(
        "SELECT MAX(live_checked_at) AS m FROM turbo_campaign_posts WHERE live_checked_at IS NOT NULL").fetchone()["m"]
    return render_template(
        "admin/turbo_dashboard.html", cars=cars, live_total=live_total, removed_total=removed_total,
        inactive_total=inactive_total, unchecked_total=unchecked_total, newest_check=newest_check,
        profiles=db.execute("SELECT id, name, email FROM turbo_session_profiles ORDER BY name").fetchall(),
        live_job=_turbo_live_check_jobs.get("current") or {"finished": True},
    )


def _auto_delete_profile_listings(db, profile_id):
    """Retrieve current active listings for a profile using its saved session, then DELETE each."""
    row = db.execute("SELECT powershell_encrypted, proxy_encrypted FROM turbo_session_profiles WHERE id=?", (profile_id,)).fetchone()
    if not row or not row["powershell_encrypted"]:
        return
    try:
        ps_text = decrypt_secret(row["powershell_encrypted"])
    except Exception:
        return
    proxy = ""
    if row["proxy_encrypted"]:
        try:
            proxy = decrypt_secret(row["proxy_encrypted"])
        except Exception:
            pass
    listings, err = turbo_listing.retrieve_listings(ps_text, proxy or None)
    if err:
        logging.getLogger(__name__).warning(f"Retrieve listings for profile {profile_id} failed: {err}")
        return
    for item in listings:
        uid = item["uuid"] if isinstance(item, dict) else str(item)
        result = turbo_listing.submit_listing_delete(ps_text, uid, proxy or None)
        del_status = "deleted" if result.get("ok") else "delete_failed"
        db.execute("INSERT INTO turbo_posted (listing_id, vin, province, city, account_email, autotrader_listing_id, posted_at, http_status, status) VALUES (?,?,?,?,?,?,?,?,?)",
                   (None, "", "", "", "", uid, now_utc(), result.get("http_status"), del_status))
        # Also mark any matching turbo_campaign_posts as deleted
        db.execute("UPDATE turbo_campaign_posts SET delete_status='deleted', deleted_at=?, delete_error='auto-deleted via retrieve' WHERE autotrader_listing_id=? AND COALESCE(delete_status,'') != 'deleted'", (now_utc(), uid))
        db.commit()
        time.sleep(3)


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/<int:cid>", methods=("GET", "POST"))
@login_required
def admin_lead_campaign_detail(cid):
    db = get_db()
    campaign = db.execute("SELECT * FROM lead_campaigns WHERE id = ?", (cid,)).fetchone()
    if campaign is None:
        abort(404)
    listing = db.execute("SELECT * FROM listings WHERE id = ?", (campaign["listing_id"],)).fetchone()
    if request.method == "POST":
        action = (request.form.get("link_action") or "").strip()
        link_id = (request.form.get("link_id") or "")
        if action == "add_links":
            try:
                count = max(1, min(50, int(request.form.get("count") or 1)))
            except ValueError:
                count = 1
            for _ in range(count):
                db.execute(
                    "INSERT INTO campaign_links (campaign_id, token, sms_token, created_at) VALUES (?, ?, ?, ?)",
                    (cid, _campaign_link_token(), _campaign_link_token(), now_utc()),
                )
            db.commit()
            flash(f"{count} more link(s) added.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid))
        if action == "remove_blank":
            cur = db.execute(
                "DELETE FROM campaign_links WHERE campaign_id = ? "
                "AND COALESCE(lead_name,'') = '' AND COALESCE(lead_email,'') = '' "
                "AND COALESCE(lead_phone,'') = '' AND COALESCE(lead_message,'') = ''",
                (cid,),
            )
            db.commit()
            flash(f"Removed {cur.rowcount} blank link(s).", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid))
        if action == "delete_link" and link_id.isdigit():
            db.execute("DELETE FROM campaign_links WHERE id=? AND campaign_id=?", (int(link_id), cid))
            db.commit()
            flash("Link deleted.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid))
        if action == "assign" and link_id.isdigit():
            lid = int(link_id)
            new_lid = _resolve_vehicle_search(request.form.get("listing_search", ""))
            if not new_lid:
                flash("Pick a valid vehicle to assign this lead to.", "warning")
                return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{lid}")
            new_cid = _find_or_create_campaign_for_listing(db, new_lid, create_new=False)
            db.execute(
                "UPDATE campaign_links SET campaign_id=? WHERE id=? AND campaign_id=?",
                (new_cid, lid, cid),
            )
            db.commit()
            flash("Lead assigned to that car's campaign.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=new_cid) + f"#link-{lid}")
        if action == "cancel_schedule" and link_id.isdigit():
            db.execute(
                "UPDATE campaign_links SET reply_status='drafted', scheduled_at=NULL WHERE id=? AND campaign_id=? AND reply_status='scheduled'",
                (int(link_id), cid),
            )
            db.commit()
            flash("Schedule cancelled — reply is back to drafted.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{link_id}")
        if action == "undup" and link_id.isdigit():
            # Clear the duplicate flag (+ stale send_error) so this lead can be sent.
            db.execute(
                "UPDATE campaign_links SET duplicate=0, send_error=NULL WHERE id=? AND campaign_id=?",
                (int(link_id), cid),
            )
            db.commit()
            flash("Marked as not-duplicate. You can now Schedule / Send this lead.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{link_id}")
        if action == "toggle_texted" and link_id.isdigit():
            row = db.execute("SELECT texted_at FROM campaign_links WHERE id=? AND campaign_id=?",
                             (int(link_id), cid)).fetchone()
            if row:
                db.execute("UPDATE campaign_links SET texted_at=? WHERE id=?",
                           (None if row["texted_at"] else now_utc(), int(link_id)))
                db.commit()
                flash("Texted mark removed." if row["texted_at"] else "Marked as texted.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{link_id}")
        if action == "save_link" and link_id.isdigit():
            db.execute(
                "UPDATE campaign_links SET lead_name=?, lead_email=?, lead_phone=?, lead_message=?, notes=?, generated_reply=? WHERE id=? AND campaign_id=?",
                (
                    (request.form.get("lead_name") or "").strip(),
                    (request.form.get("lead_email") or "").strip(),
                    (request.form.get("lead_phone") or "").strip(),
                    (request.form.get("lead_message") or "").strip(),
                    (request.form.get("notes") or "").strip(),
                    (request.form.get("generated_reply") or "").strip(),
                    int(link_id), cid,
                ),
            )
            db.commit()
            flash("Link saved.", "success")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{link_id}")
        if action == "send" and link_id.isdigit():
            lid = int(link_id)
            link = db.execute(
                "SELECT * FROM campaign_links WHERE id=? AND campaign_id=?", (lid, cid)
            ).fetchone()
            if not link:
                flash("Link not found.", "warning")
                return redirect(url_for("admin_lead_campaign_detail", cid=cid))
            # Send auto-saves the lead info + reply first, then sends.
            lead_name = (request.form.get("lead_name") or "").strip()
            lead_email = (request.form.get("lead_email") or "").strip()
            lead_phone = (request.form.get("lead_phone") or "").strip()
            lead_message = (request.form.get("lead_message") or "").strip()
            notes = (request.form.get("notes") or "").strip()
            body = (request.form.get("generated_reply") or "").strip()
            db.execute(
                "UPDATE campaign_links SET lead_name=?, lead_email=?, lead_phone=?, lead_message=?, notes=?, generated_reply=? WHERE id=?",
                (lead_name, lead_email, lead_phone, lead_message, notes, body, lid),
            )
            db.commit()
            if not lead_email:
                flash("Cannot send: missing recipient email.", "warning")
                return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{lid}")
            if not body:
                flash("Cannot send: no reply text. Generate or write a reply first.", "warning")
                return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{lid}")
            # Auto-append the tracking link so the lead can view the car/photos.
            tracking_url = f"{request.host_url}r/{link['token']}"
            if tracking_url not in body:
                body = body.rstrip() + f"\n\nView the vehicle here: {tracking_url}"
            db.execute(
                "UPDATE campaign_links SET generated_reply=? WHERE id=?",
                (body, lid),
            )
            db.commit()
            settings = get_settings()
            link = db.execute("SELECT * FROM campaign_links WHERE id=?", (lid,)).fetchone()
            ok, msg = send_campaign_link_reply(link, settings)
            if ok:
                _mark_link_sent(lid, body)
                flash("Reply sent (lead info saved + tracking link included).", "success")
            else:
                _mark_link_send_failed(lid, msg)
                flash(f"Send failed: {msg}", "warning")
            return redirect(url_for("admin_lead_campaign_detail", cid=cid) + f"#link-{lid}")
    links = db.execute(
        "SELECT * FROM campaign_links WHERE campaign_id = ? "
        "ORDER BY duplicate ASC, CASE reply_status WHEN 'sent' THEN 1 ELSE 0 END, id",
        (cid,),
    ).fetchall()
    listings = db.execute(
        "SELECT id, vin, stock_number, public_stock_number, title_en, year, make, model, trim "
        "FROM listings WHERE COALESCE(is_deleted,0)=0 ORDER BY year DESC, make, model"
    ).fetchall()
    total = len(links)
    visited = sum(1 for l in links if l["visited"])
    sent = sum(1 for l in links if l["reply_status"] == "sent")
    unreplied = sum(1 for l in links if l["reply_status"] != "sent" and not l["duplicate"])
    dup_count = sum(1 for l in links if l["duplicate"])
    car_line = ""
    if listing:
        try:
            price_str = "${:,}".format(int(float(str(listing["price"]).replace(",", ""))))
        except (ValueError, TypeError):
            price_str = ""
        car_line = f"{listing['year']} {listing['make']} {listing['model']} {listing['trim'] or ''} {price_str}".strip()
    active_job = settings_value(get_settings(), "reply_gen_active_job", "") or ""
    send_errors = db.execute(
        "SELECT id, lead_name, lead_email, send_error, reply_status, sent_at "
        "FROM campaign_links WHERE campaign_id=? AND COALESCE(send_error,'') != '' "
        "ORDER BY id DESC", (cid,)).fetchall()
    return render_template(
        "admin/lead_campaign_detail.html", campaign=campaign, listing=listing, links=links,
        listings=listings, car_line=car_line, total=total, visited=visited, sent=sent,
        unreplied=unreplied, dup_count=dup_count, active_job=active_job, send_errors=send_errors,
    )


def _generate_reply_for_link(link, host_url):
    """Draft (or re-draft) one link's reply synchronously. Returns
    (ok, payload) — payload is the result dict on success or an error string."""
    db = get_db()
    campaign = db.execute("SELECT * FROM lead_campaigns WHERE id=?", (link["campaign_id"],)).fetchone()
    listing = db.execute("SELECT * FROM listings WHERE id=?", (campaign["listing_id"],)).fetchone() if campaign else None
    if not listing or listing["is_deleted"]:
        return False, "Vehicle not found (deleted?) — reassign this lead to a live car first."
    settings = get_settings()
    status, ai_adapter = _lead_reply_ai_adapter(settings)
    if not status["active_configured"]:
        return False, f"{status['provider'].title()} API key is missing."
    ctx = _lead_reply_context(listing, settings, host_url, link)
    lead = {
        "name": link["lead_name"], "email": link["lead_email"],
        "phone": link["lead_phone"], "message": link["lead_message"],
    }
    master_prompt = settings_value(settings, "lead_reply_master_prompt", "") or ""
    dealer_name = settings_value(settings, "company_name", "") or ""
    phone = (link["lead_phone"] or "").strip()
    sms_url = f"{host_url}r/{link['sms_token']}" if (phone and link["sms_token"]) else ""
    sms_hint = lead_dispatch.sms_style_hint(link["id"]) if sms_url else ""
    result = lead_replies.generate_lead_reply(
        lead, ctx["vehicle"], master_prompt, dealer_name, ai_adapter,
        tracking_url=ctx["tracking_url"], input_data=ctx["input_data"],
        sms_url=sms_url, sms_style_hint=sms_hint,
    )
    if result.get("status") == "error":
        return False, str(result.get("error", "Unknown error"))
    db.execute(
        "UPDATE campaign_links SET generated_reply=?, reply_subject=?, text_message=?, reply_status='drafted', gen_status='drafted', send_error=NULL WHERE id=?",
        (result["body"], result.get("subject", ""), result.get("text_message", ""), link["id"]),
    )
    db.commit()
    return True, result


@app.route(f"{ADMIN_URL_PREFIX}/lead-campaigns/<int:cid>/link/<int:link_id>/generate-reply", methods=("POST",))
@login_required
def admin_lead_campaign_generate_reply(cid, link_id):
    try:
        link = get_db().execute(
            "SELECT * FROM campaign_links WHERE id=? AND campaign_id=?", (link_id, cid)
        ).fetchone()
        if not link:
            return jsonify({"ok": False, "error": "Link not found."}), 404
        ok, payload = _generate_reply_for_link(link, request.host_url)
        if not ok:
            return jsonify({"ok": False, "error": payload}), 400
        return jsonify({"ok": True, "subject": payload.get("subject", ""), "body": payload["body"]})
    except Exception as error:
        return jsonify({"ok": False, "error": str(error)}), 500


@app.route("/r/<token>")
def campaign_redirect(token):
    """Log a visit and 301 redirect to the listing. Token is opaque (no VIN in URL).
    Resolves the email token first, then the SMS follow-up token, attributing the
    visit to the matching channel's counters."""
    db = get_db()
    base_sql = (
        "SELECT cl.id, cl.{vcol} AS visited, l.slug "
        "FROM campaign_links cl JOIN lead_campaigns lc ON lc.id = cl.campaign_id "
        "JOIN listings l ON l.id = lc.listing_id WHERE cl.{tcol} = ?"
    )
    link = db.execute(base_sql.format(vcol="visited", tcol="token"), (token,)).fetchone()
    channel = "email"
    if link is None:
        link = db.execute(base_sql.format(vcol="sms_visited", tcol="sms_token"), (token,)).fetchone()
        channel = "sms"
    if link is None:
        abort(404)
    now = now_utc()
    if channel == "email":
        if link["visited"]:
            db.execute(
                "UPDATE campaign_links SET visit_count = visit_count + 1, last_visit_at = ? WHERE id = ?",
                (now, link["id"]),
            )
        else:
            db.execute(
                "UPDATE campaign_links SET visited = 1, visit_count = 1, first_visit_at = ?, last_visit_at = ? WHERE id = ?",
                (now, now, link["id"]),
            )
    else:
        if link["visited"]:
            db.execute(
                "UPDATE campaign_links SET sms_visit_count = sms_visit_count + 1, sms_last_visit_at = ? WHERE id = ?",
                (now, link["id"]),
            )
        else:
            db.execute(
                "UPDATE campaign_links SET sms_visited = 1, sms_visit_count = 1, sms_first_visit_at = ?, sms_last_visit_at = ? WHERE id = ?",
                (now, now, link["id"]),
            )
    db.commit()
    return redirect(url_for("listing_detail", lang="en", slug=link["slug"]), code=301)


@app.route(f"{ADMIN_URL_PREFIX}/leads/<int:id>/read", methods=("POST",))
@login_required
def admin_lead_read(id):
    get_db().execute("UPDATE leads SET is_read = 1 WHERE id = ?", (id,))
    get_db().commit()
    flash("Lead marked as read.", "success")
    return redirect(url_for("admin_leads"))


def parse_faq_paste(text):
    entries = []
    current_q = ""
    current_a = []
    for raw_line in (text or "").splitlines():
        line = raw_line.strip()
        if not line:
            continue
        q_match = re.match(r"^(Q:|Question:)\s*(.+)$", line, re.I)
        a_match = re.match(r"^(A:|Answer:)\s*(.+)$", line, re.I)
        if q_match:
            if current_q and current_a:
                entries.append((current_q, "\n".join(current_a).strip()))
            current_q = q_match.group(2).strip()
            current_a = []
        elif a_match:
            current_a.append(a_match.group(2).strip())
        elif current_q:
            current_a.append(line)
    if current_q and current_a:
        entries.append((current_q, "\n".join(current_a).strip()))
    return entries


@app.route(f"{ADMIN_URL_PREFIX}/chatbot", methods=("GET", "POST"))
@login_required
def admin_chatbot():
    db = get_db()
    settings = get_settings()
    if request.method == "POST":
        model = request.form.get("chatbot_model", "").strip() or DEFAULT_FIREWORKS_MODEL
        db.execute(
            """
            UPDATE site_settings
            SET chatbot_enabled = ?, chatbot_provider = 'fireworks', chatbot_model = ?, chatbot_custom_model = ?,
                chatbot_greeting = ?, chatbot_fallback_message = ?, chatbot_lead_capture_enabled = ?,
                chatbot_show_desktop = ?, chatbot_show_mobile = ?, chatbot_system_prompt = ?, updated_at = ?
            WHERE id = 1
            """,
            (
                1 if request.form.get("chatbot_enabled") else 0,
                model,
                request.form.get("chatbot_custom_model", "").strip(),
                request.form.get("chatbot_greeting", "").strip() or default_chatbot_greeting(),
                request.form.get("chatbot_fallback_message", "").strip() or default_chatbot_fallback_message(),
                1 if request.form.get("chatbot_lead_capture_enabled") else 0,
                1 if request.form.get("chatbot_show_desktop") else 0,
                1 if request.form.get("chatbot_show_mobile") else 0,
                request.form.get("chatbot_system_prompt", "").strip(),
                now_utc(),
            ),
        )
        db.commit()
        flash("Chatbot settings updated.", "success")
        return redirect(url_for("admin_chatbot"))
    sessions = db.execute(
        """
        SELECT chat_sessions.*, listings.title_en, listings.public_stock_number
        FROM chat_sessions
        LEFT JOIN listings ON listings.id = chat_sessions.listing_id
        ORDER BY chat_sessions.updated_at DESC, chat_sessions.id DESC
        LIMIT 20
        """
    ).fetchall()
    knowledge_count = db.execute("SELECT COUNT(*) FROM chatbot_knowledge").fetchone()[0]
    return render_template(
        "admin/chatbot.html",
        settings=settings,
        sessions=sessions,
        knowledge_count=knowledge_count,
        chatbot_models=chatbot_model_choices(),
        ai_status=ai_provider_status(settings),
    )


@app.route(f"{ADMIN_URL_PREFIX}/chatbot/sessions/<int:id>")
@login_required
def admin_chatbot_session(id):
    chat_session = get_db().execute(
        """
        SELECT chat_sessions.*, listings.title_en, listings.public_stock_number
        FROM chat_sessions
        LEFT JOIN listings ON listings.id = chat_sessions.listing_id
        WHERE chat_sessions.id = ?
        """,
        (id,),
    ).fetchone()
    if chat_session is None:
        abort(404)
    messages = get_db().execute(
        "SELECT * FROM chat_messages WHERE session_id = ? ORDER BY id",
        (id,),
    ).fetchall()
    return render_template("admin/chatbot_session.html", chat_session=chat_session, messages=messages)


@app.route(f"{ADMIN_URL_PREFIX}/chatbot/knowledge", methods=("GET", "POST"))
@login_required
def admin_chatbot_knowledge():
    db = get_db()
    if request.method == "POST":
        action = request.form.get("knowledge_action", "save")
        timestamp = now_utc()
        if action == "delete":
            knowledge_id = request.form.get("knowledge_id", "")
            if knowledge_id.isdigit():
                db.execute("DELETE FROM chatbot_knowledge WHERE id = ?", (knowledge_id,))
                db.commit()
                flash("Knowledge entry deleted.", "success")
            return redirect(url_for("admin_chatbot_knowledge"))
        if action == "import_faqs":
            imported = 0
            rows = db.execute("SELECT * FROM faqs WHERE is_active = 1 ORDER BY sort_order, id").fetchall()
            for faq in rows:
                title = faq["question_en"] or ""
                content = faq["answer_en"] or ""
                if not title or not content:
                    continue
                exists = db.execute("SELECT id FROM chatbot_knowledge WHERE title = ? AND category = 'FAQ'", (title,)).fetchone()
                if exists:
                    continue
                db.execute(
                    """
                    INSERT INTO chatbot_knowledge (title, category, content, language_code, is_active, sort_order, created_at, updated_at)
                    VALUES (?, 'FAQ', ?, 'en', 1, ?, ?, ?)
                    """,
                    (title, content, faq["sort_order"] or 100, timestamp, timestamp),
                )
                imported += 1
            db.commit()
            flash(f"Imported {imported} FAQ entr{'y' if imported == 1 else 'ies'} into chatbot knowledge.", "success")
            return redirect(url_for("admin_chatbot_knowledge"))
        if action == "paste_faq":
            entries = parse_faq_paste(request.form.get("faq_paste", ""))
            for index, (question, answer) in enumerate(entries, start=1):
                db.execute(
                    """
                    INSERT INTO chatbot_knowledge (title, category, content, language_code, is_active, sort_order, created_at, updated_at)
                    VALUES (?, 'Pasted FAQ', ?, ?, 1, ?, ?, ?)
                    """,
                    (question, answer, request.form.get("paste_language", "en"), 100 + index, timestamp, timestamp),
                )
            db.commit()
            flash(f"Added {len(entries)} pasted FAQ entr{'y' if len(entries) == 1 else 'ies'}.", "success")
            return redirect(url_for("admin_chatbot_knowledge"))
        knowledge_id = request.form.get("knowledge_id", "")
        values = (
            request.form.get("title", "").strip(),
            request.form.get("category", "").strip(),
            request.form.get("content", "").strip(),
            request.form.get("language_code", "en").strip() if request.form.get("language_code", "en").strip() in LANGUAGES else "en",
            1 if request.form.get("is_active") else 0,
            request.form.get("sort_order", "100") or 100,
            timestamp,
        )
        if not values[0] or not values[2]:
            flash("Title and content are required.", "warning")
            return redirect(url_for("admin_chatbot_knowledge"))
        if knowledge_id.isdigit():
            db.execute(
                """
                UPDATE chatbot_knowledge
                SET title = ?, category = ?, content = ?, language_code = ?, is_active = ?, sort_order = ?, updated_at = ?
                WHERE id = ?
                """,
                (*values, knowledge_id),
            )
            flash("Knowledge entry updated.", "success")
        else:
            db.execute(
                """
                INSERT INTO chatbot_knowledge (title, category, content, language_code, is_active, sort_order, created_at, updated_at)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (*values[:-1], timestamp, timestamp),
            )
            flash("Knowledge entry added.", "success")
        db.commit()
        return redirect(url_for("admin_chatbot_knowledge"))
    entries = db.execute("SELECT * FROM chatbot_knowledge ORDER BY is_active DESC, sort_order, title").fetchall()
    return render_template("admin/chatbot_knowledge.html", entries=entries, languages=get_active_languages())


@app.route(f"{ADMIN_URL_PREFIX}/pages")
@login_required
def admin_pages():
    pages = get_db().execute("SELECT * FROM site_pages ORDER BY slug").fetchall()
    return render_template("admin/pages.html", pages=pages)


@app.route(f"{ADMIN_URL_PREFIX}/pages/bulk", methods=("POST",))
@login_required
def admin_pages_bulk():
    page_ids = [int(value) for value in request.form.getlist("page_ids") if value.isdigit()]
    action = request.form.get("bulk_action", "").strip()
    lang = request.form.get("bulk_lang", "").strip()
    if not page_ids:
        flash("Select at least one page.", "warning")
        return redirect(url_for("admin_pages"))
    placeholders = ", ".join("?" for _ in page_ids)
    timestamp = now_utc()
    if action == "approve":
        get_db().execute(
            f"UPDATE site_pages SET status = 'approved', updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash("Selected pages approved.", "success")
    elif action == "publish":
        get_db().execute(
            f"UPDATE site_pages SET status = 'published', is_published = 1, updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash("Selected pages published.", "success")
    elif action == "unpublish":
        get_db().execute(
            f"UPDATE site_pages SET is_published = 0, updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash("Selected pages unpublished.", "success")
    elif action == "draft":
        get_db().execute(
            f"UPDATE site_pages SET status = 'draft', is_published = 0, updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash("Selected pages moved to draft.", "success")
    elif action == "translation_approve" and lang in LANGUAGES and lang != "en":
        get_db().execute(
            f"UPDATE site_pages SET translation_status_{lang} = 'approved', updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash(f"Selected {LANGUAGE_NAMES.get(lang, lang)} translations approved.", "success")
    elif action == "translation_publish" and lang in LANGUAGES and lang != "en":
        get_db().execute(
            f"UPDATE site_pages SET translation_status_{lang} = 'published', translation_published_{lang} = 1, updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash(f"Selected {LANGUAGE_NAMES.get(lang, lang)} translations published.", "success")
    elif action == "translation_unpublish" and lang in LANGUAGES and lang != "en":
        get_db().execute(
            f"UPDATE site_pages SET translation_published_{lang} = 0, updated_at = ? WHERE id IN ({placeholders})",
            (timestamp, *page_ids),
        )
        flash(f"Selected {LANGUAGE_NAMES.get(lang, lang)} translations unpublished.", "success")
    else:
        flash("Choose a valid bulk page action.", "warning")
        return redirect(url_for("admin_pages"))
    get_db().commit()
    return redirect(url_for("admin_pages"))


@app.route(f"{ADMIN_URL_PREFIX}/pages/translate", methods=("POST",))
@login_required
def admin_pages_translate():
    page_ids = [int(value) for value in request.form.getlist("page_ids") if value.isdigit()]
    langs = [value.strip() for value in request.form.getlist("langs") if value.strip() in LANGUAGES and value.strip() != "en"]
    mode = request.form.get("translation_mode", "missing").strip()
    force = mode == "regenerate"

    if not page_ids:
        flash("Select at least one page.", "warning")
        return redirect(url_for("admin_pages"))
    if not langs:
        flash("Select at least one target language.", "warning")
        return redirect(url_for("admin_pages"))

    translated = 0
    skipped = 0
    errors = []
    for page_id in page_ids:
        for lang in langs:
            try:
                result = translate_page_with_ai(page_id, lang, force=force, publish=True)
                if result == "translated":
                    translated += 1
                elif result == "skipped":
                    skipped += 1
            except Exception as error:
                errors.append(f"{LANGUAGE_NAMES.get(lang, lang)} page #{page_id}: {error}")

    get_db().commit()

    language_names = ", ".join(LANGUAGE_NAMES.get(code, code) for code in langs)
    if translated:
        flash(
            f"Generated and published {translated} translation{'s' if translated != 1 else ''} from English to {language_names}.",
            "success",
        )
    if skipped and not force:
        flash(
            f"Skipped {skipped} existing published/review translations. Use Regenerate selected translations to overwrite them from English.",
            "info",
        )
    if errors:
        flash("Translation issues: " + " | ".join(errors[:3]), "warning")
    if not translated and not skipped and not errors:
        flash("No translation changes were made.", "info")
    return redirect(url_for("admin_pages"))


@app.route(f"{ADMIN_URL_PREFIX}/pages/save-generation-settings", methods=("POST",))
@login_required
def admin_pages_save_gen_settings():
    master = request.form.get("pages_master_prompt", "").strip()
    enabled = 1 if request.form.get("pages_generation_enabled") else 0
    try:
        interval = max(1, int(request.form.get("pages_generation_interval_hours", "24") or 24))
    except (TypeError, ValueError):
        interval = 24
    get_db().execute(
        "UPDATE site_settings SET pages_master_prompt = ?, pages_generation_enabled = ?, pages_generation_interval_hours = ?, updated_at = ? WHERE id = 1",
        (master, enabled, interval, now_utc()),
    )
    get_db().commit()
    flash("Page generation settings saved.", "success")
    return redirect(url_for("admin_pages"))


@app.route(f"{ADMIN_URL_PREFIX}/pages/run-generation", methods=("POST",))
@login_required
def admin_pages_run_now():
    page_ids = [int(v) for v in request.form.getlist("page_ids") if v.isdigit()] or None
    job_id = trigger_page_generation_batch(page_ids=page_ids)
    if job_id is None:
        flash("A generation batch is already running, or the AI provider is not configured. Wait for the current batch or set an API key.", "warning")
    else:
        flash("Page content generation started. Refresh to see progress.", "success")
    return redirect(url_for("admin_pages"))


@app.route(f"{ADMIN_URL_PREFIX}/pages/generation-status")
@login_required
def admin_pages_gen_status():
    latest = None
    for j in PAGE_GEN_JOBS.values():
        latest = j
    settings = get_settings()
    return jsonify({
        "ok": True,
        "job": latest,
        "enabled": bool(settings["pages_generation_enabled"]) if "pages_generation_enabled" in settings.keys() else False,
        "interval_hours": settings["pages_generation_interval_hours"] if "pages_generation_interval_hours" in settings.keys() else 24,
        "last_run": settings["pages_generation_last_run"] if "pages_generation_last_run" in settings.keys() else None,
    })


@app.route(f"{ADMIN_URL_PREFIX}/pages/edit/<int:id>", methods=("GET", "POST"))
@login_required
def admin_page_edit(id):
    page = get_db().execute("SELECT * FROM site_pages WHERE id = ?", (id,)).fetchone()
    if page is None:
        abort(404)
    if request.method == "POST":
        if request.form.get("translation_action"):
            lang = request.form.get("translation_lang", "").strip()
            action = request.form.get("translation_action")
            if lang in LANGUAGES and lang != "en":
                if action == "translate":
                    try:
                        translate_page_with_ai(id, lang, force=True, publish=True)
                        get_db().commit()
                        flash(f"{LANGUAGE_NAMES.get(lang, lang)} translation regenerated from English and published.", "success")
                    except Exception as error:
                        flash(str(error), "warning")
                elif action in ("approve", "publish", "unpublish"):
                    status = "approved" if action == "approve" else ("published" if action == "publish" else "needs_review")
                    published = 1 if action == "publish" else 0
                    get_db().execute(
                        f"UPDATE site_pages SET translation_status_{lang} = ?, translation_published_{lang} = ?, updated_at = ? WHERE id = ?",
                        (status, published, now_utc(), id),
                    )
                    get_db().commit()
                    flash("Translation status updated.", "success")
            return redirect(url_for("admin_page_edit", id=id))
        old_page = page
        get_db().execute(
            """
            UPDATE site_pages
            SET title_en = ?, excerpt_en = ?, content_html_en = ?, custom_prompt = ?,
                seo_title_en = ?, meta_description_en = ?, hero_image_path = ?,
                page_type = ?, status = ?, menu_visibility = ?, show_in_header = ?, show_in_footer = ?, menu_order = ?,
                cta_title_en = ?, cta_text_en = ?, visual_prompt_en = ?,
                is_published = ?, english_updated_at = ?, updated_at = ?
            WHERE id = ?
            """,
            (
                request.form.get("title_en", "").strip(),
                request.form.get("excerpt_en", "").strip(),
                request.form.get("content_html_en", "").strip(),
                request.form.get("custom_prompt", "").strip(),
                request.form.get("seo_title_en", "").strip(),
                request.form.get("meta_description_en", "").strip(),
                request.form.get("hero_image_path", "").strip(),
                request.form.get("page_type", "standard").strip() or "standard",
                request.form.get("status", "published").strip() or "published",
                1 if request.form.get("show_in_header") else 0,
                1 if request.form.get("show_in_header") else 0,
                1 if request.form.get("show_in_footer") else 0,
                request.form.get("menu_order", "100") or 100,
                request.form.get("cta_title_en", "").strip(),
                request.form.get("cta_text_en", "").strip(),
                request.form.get("visual_prompt_en", "").strip(),
                1 if request.form.get("is_published") else 0,
                now_utc(),
                now_utc(),
                id,
            ),
        )
        if (
            (old_page["title_en"] or "") != request.form.get("title_en", "").strip()
            or (old_page["content_html_en"] or "") != request.form.get("content_html_en", "").strip()
            or (old_page["excerpt_en"] or "") != request.form.get("excerpt_en", "").strip()
        ):
            for lang_code in LANGUAGES:
                if lang_code != "en":
                    get_db().execute(
                        f"UPDATE site_pages SET translation_status_{lang_code} = 'outdated' WHERE id = ? AND translation_status_{lang_code} IN ('needs_review','approved','published')",
                        (id,),
                    )
        get_db().commit()
        flash("Page updated.", "success")
        return redirect(url_for("admin_page_edit", id=id))
    visuals = get_db().execute(
        "SELECT * FROM generated_visuals WHERE page_id = ? ORDER BY is_active DESC, created_at DESC, id DESC",
        (id,),
    ).fetchall()
    return render_template("admin/page_form.html", page=page, languages=get_active_languages(), visuals=visuals)


@app.route(f"{ADMIN_URL_PREFIX}/pages/edit/<int:id>/generate-visual", methods=("POST",))
@login_required
def admin_page_generate_visual(id):
    model = request.form.get("visual_model", "gpt-image-2").strip() or "gpt-image-2"
    try:
        path = generate_page_visual(id, placement_key="hero", model=model)
        flash(f"Generated and assigned hero visual: {path}", "success")
    except Exception as error:
        get_db().rollback()
        flash(str(error), "warning")
    return redirect(url_for("admin_page_edit", id=id))


@app.route(f"{ADMIN_URL_PREFIX}/redirects", methods=("GET", "POST"))
@login_required
def admin_redirects():
    db = get_db()
    if request.method == "POST":
        action = request.form.get("redirect_action", "save")
        redirect_id = request.form.get("redirect_id", "").strip()
        if action == "delete" and redirect_id.isdigit():
            db.execute("DELETE FROM redirects WHERE id = ?", (int(redirect_id),))
            db.commit()
            flash("Redirect deleted.", "success")
            return redirect(url_for("admin_redirects"))

        old_path = normalize_local_redirect_path(request.form.get("old_path", ""))
        new_path = normalize_local_redirect_path(request.form.get("new_path", ""))
        status_code = int(request.form.get("status_code", "301") or 301)
        notes = (request.form.get("notes", "") or "").strip()
        if status_code not in (301, 302, 307, 308):
            status_code = 301
        if not old_path or not new_path:
            flash("Redirect paths must be local paths starting with /.", "warning")
            return redirect(url_for("admin_redirects"))
        if old_path == new_path:
            flash("Old and new paths must be different.", "warning")
            return redirect(url_for("admin_redirects"))
        if old_path.startswith((ADMIN_URL_PREFIX, "/static/", "/media/")):
            flash("Admin, static, and media paths cannot be redirected here.", "warning")
            return redirect(url_for("admin_redirects"))
        try:
            if redirect_id.isdigit():
                db.execute(
                    "UPDATE redirects SET old_path = ?, new_path = ?, status_code = ?, notes = ? WHERE id = ?",
                    (old_path, new_path, status_code, notes, int(redirect_id)),
                )
                flash("Redirect updated.", "success")
            else:
                db.execute(
                    "INSERT INTO redirects (old_path, new_path, status_code, notes, created_at) VALUES (?, ?, ?, ?, ?)",
                    (old_path, new_path, status_code, notes, now_utc()),
                )
                flash("Redirect added.", "success")
            db.commit()
        except sqlite3.IntegrityError:
            db.rollback()
            flash("A redirect for that old path already exists.", "warning")
        return redirect(url_for("admin_redirects"))

    redirects = db.execute("SELECT * FROM redirects ORDER BY created_at DESC, id DESC").fetchall()
    return render_template("admin/redirects.html", redirects=redirects)


@app.route(f"{ADMIN_URL_PREFIX}/faqs", methods=("GET", "POST"))
@login_required
def admin_faqs():
    db = get_db()
    if request.method == "POST":
        timestamp = now_utc()
        for faq_id in request.form.getlist("faq_id"):
            db.execute(
                """
                UPDATE faqs
                SET question_en = ?, answer_en = ?, sort_order = ?, is_active = ?, updated_at = ?
                WHERE id = ?
                """,
                (
                    request.form.get(f"question_en_{faq_id}", "").strip(),
                    request.form.get(f"answer_en_{faq_id}", "").strip(),
                    request.form.get(f"sort_order_{faq_id}", "0") or 0,
                    1 if request.form.get(f"is_active_{faq_id}") else 0,
                    timestamp,
                    faq_id,
                ),
            )
        new_question = request.form.get("new_question_en", "").strip()
        new_answer = request.form.get("new_answer_en", "").strip()
        if new_question and new_answer:
            db.execute(
                """
                INSERT INTO faqs (question_en, answer_en, sort_order, is_active, created_at, updated_at)
                VALUES (?, ?, ?, 1, ?, ?)
                """,
                (new_question, new_answer, request.form.get("new_sort_order", "100") or 100, timestamp, timestamp),
            )
        db.commit()
        flash("FAQs updated.", "success")
        return redirect(url_for("admin_faqs"))
    faqs = db.execute("SELECT * FROM faqs ORDER BY sort_order, id").fetchall()
    return render_template("admin/faqs.html", faqs=faqs)


# ---------- homepage card image helpers ----------

def save_generated_card_image(card, image_b64_or_bytes, ext="png"):
    cards_dir = os.path.join(app.static_folder, "uploads", "cards")
    os.makedirs(cards_dir, exist_ok=True)
    safe_type = os.path.basename(re.sub(r'[^A-Za-z0-9_.-]+', '_', card['card_type']))
    safe_key = os.path.basename(re.sub(r'[^A-Za-z0-9_.-]+', '_', card['key']))
    fname = f"{safe_type}_{safe_key}.{ext}"
    path = os.path.join(cards_dir, fname)
    data = base64.b64decode(image_b64_or_bytes) if isinstance(image_b64_or_bytes, str) else image_b64_or_bytes
    with open(path, "wb") as f:
        f.write(data)
    rel = f"uploads/cards/{fname}"
    get_db().execute(
        "UPDATE homepage_cards SET image_path=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
        [rel, card["id"]],
    )
    get_db().commit()
    return rel


def generate_card_image_openai(card, prompt):
    client = get_openai_client()
    settings = get_settings()
    model = (settings["openai_image_model"] if "openai_image_model" in settings.keys() else "") or "gpt-image-1"
    # Guard against stale/invalid model names (e.g. text models like gpt-5.x stored by older seeds).
    # 1536x1024 is a gpt-image-1 size; dall-e-3 doesn't support it, so prefer gpt-image-1.
    if model not in ("gpt-image-1", "gpt-image-1-mini", "dall-e-3", "dall-e-2"):
        model = "gpt-image-1"
    resp = client.images.generate(model=model, size="1536x1024", quality="high", prompt=prompt, n=1)
    img = resp.data[0]
    if getattr(img, "b64_json", None):
        return save_generated_card_image(card, img.b64_json, "png")
    if getattr(img, "url", None):
        import requests
        r = requests.get(img.url, timeout=60)
        r.raise_for_status()
        return save_generated_card_image(card, r.content, "png")
    raise RuntimeError("OpenAI returned no image data.")


# ---------- admin homepage cards batch job store + worker ----------

IMAGE_JOBS = {}
_batch_lock = threading.Lock()


def _run_batch(job_id, card_ids):
    job = IMAGE_JOBS[job_id]
    for cid in card_ids:
        card = get_db().execute("SELECT * FROM homepage_cards WHERE id=?", [cid]).fetchone()
        job["current_label"] = card["key"] if card else str(cid)
        for st in job["statuses"]:
            if st["card_id"] == cid:
                st["status"] = "working"
                break
        try:
            generate_card_image_openai(card, card["prompt"])
            for st in job["statuses"]:
                if st["card_id"] == cid:
                    st["status"] = "done"
                    break
            job["done"] += 1
        except Exception as e:
            for st in job["statuses"]:
                if st["card_id"] == cid:
                    st["status"] = "failed"
                    st["error"] = str(e)
                    break
            job["failed"] += 1
        time.sleep(1.5)
    job["finished"] = True
    job["current_label"] = None


# ---------- admin homepage cards routes ----------

@app.route(f"{ADMIN_URL_PREFIX}/homepage-cards", methods=["GET", "POST"])
@login_required
def admin_homepage_cards():
    db = get_db()
    if request.method == "POST":
        card_id = request.form.get("card_id")
        db.execute(
            "UPDATE homepage_cards SET label_en=?, label_fr=?, prompt=?, is_active=?, updated_at=CURRENT_TIMESTAMP WHERE id=?",
            [
                request.form.get("label_en"),
                request.form.get("label_fr"),
                request.form.get("prompt"),
                int(bool(request.form.get("is_active"))),
                card_id,
            ],
        )
        db.commit()
        flash("Card updated.", "success")
        return redirect(request.url)
    cards = db.execute("SELECT * FROM homepage_cards ORDER BY card_type, sort_order").fetchall()
    return render_template("admin/homepage_cards.html", cards=cards)


@app.route(f"{ADMIN_URL_PREFIX}/homepage-cards/<int:card_id>/generate", methods=["POST"])
@login_required
def admin_homepage_card_generate(card_id):
    card = get_db().execute("SELECT * FROM homepage_cards WHERE id=?", [card_id]).fetchone()
    if not card:
        return {"ok": False, "error": "Card not found"}, 404
    if card["card_type"] == "make":
        # Make tiles use sourced brand logos, not AI generation.
        return {"ok": False, "error": "Make cards use sourced logos, not AI generation."}, 400
    prompt = (request.form.get("prompt") or "").strip() or card["prompt"]
    try:
        rel = generate_card_image_openai(card, prompt)
        return {"ok": True, "image_url": url_for("static", filename=rel) + "?v=" + str(int(__import__("time").time()))}
    except Exception as e:
        return {"ok": False, "error": str(e)}, 500


@app.route(f"{ADMIN_URL_PREFIX}/homepage-cards/batch-generate", methods=["POST"])
@login_required
def admin_homepage_cards_batch_generate():
    ids = [int(i) for i in request.form.getlist("batch_ids") if i.isdigit()]
    if not ids:
        return {"ok": False, "error": "No cards selected"}, 400
    if not _batch_lock.acquire(blocking=False):
        return {"ok": False, "error": "A batch is already running. Wait for it to finish."}, 409
    job_id = str(uuid.uuid4())
    IMAGE_JOBS[job_id] = {
        "job_id": job_id,
        "total": len(ids),
        "done": 0,
        "failed": 0,
        "current_label": None,
        "finished": False,
        "statuses": [{"card_id": i, "status": "pending", "error": None} for i in ids],
    }

    def _wrapped():
        with app.app_context():
            try:
                _run_batch(job_id, ids)
            except Exception as e:
                job = IMAGE_JOBS.get(job_id)
                if job:
                    job["finished"] = True
                    job["error"] = f"Batch worker crashed: {e}"
                    job["current_label"] = None
            finally:
                _batch_lock.release()

    threading.Thread(target=_wrapped, daemon=True).start()
    return {"ok": True, "job_id": job_id}


@app.route(f"{ADMIN_URL_PREFIX}/homepage-cards/jobs/<job_id>")
@login_required
def admin_homepage_cards_job_status(job_id):
    job = IMAGE_JOBS.get(job_id)
    if not job:
        return {"ok": False, "error": "Unknown job"}, 404
    return {"ok": True, **job}


@app.cli.command("init-db")
def init_db_command():
    init_db()
    print("Database initialized.")


# ---------- page content generation batch job store + worker ----------

PAGE_GEN_JOBS = {}
_page_gen_lock = threading.Lock()
_page_gen_scheduler_started = False


def _run_page_gen_batch(job_id, page_ids, master_prompt, ai_call):
    job = PAGE_GEN_JOBS[job_id]
    db = get_db()
    for pid in page_ids:
        page = db.execute("SELECT id, slug, custom_prompt FROM site_pages WHERE id=?", [pid]).fetchone()
        job["current_slug"] = page["slug"] if page else str(pid)
        try:
            custom = (page["custom_prompt"] if page else "") or ""
            result = page_content_gen.generate_page_content(pid, db, ai_call, master_prompt=master_prompt, custom_prompt=custom)
            st = result.get("status")
            if st == "generated":
                job["generated"] += 1
            elif st == "skipped":
                job["skipped"] += 1
            elif st == "error":
                job["errors"] += 1
                job["error_msgs"].append(f"{job['current_slug']}: {result.get('error')}")
            job["done"] += 1
        except Exception as e:
            job["errors"] += 1
            job["error_msgs"].append(f"{job['current_slug']}: {e}")
            job["done"] += 1
    job["finished"] = True
    job["current_slug"] = None
    db.execute("UPDATE site_settings SET pages_generation_last_run = ? WHERE id = 1", (now_utc(),))
    db.commit()


def trigger_page_generation_batch(page_ids=None, master_prompt=None, ai_call=None):
    """Start a background page-content generation batch. Returns job_id or None if a batch is running / provider unconfigured."""
    if not _page_gen_lock.acquire(blocking=False):
        return None
    try:
        db = get_db()
        settings = get_settings()
        master = master_prompt if master_prompt is not None else (settings["pages_master_prompt"] or "")
        if page_ids is None:
            rows = db.execute("SELECT id FROM site_pages WHERE is_published = 1 ORDER BY slug").fetchall()
            page_ids = [r["id"] for r in rows]
        if ai_call is None:
            fstatus, _adapter = make_text_adapter(settings, "pages")
            if not fstatus["active_configured"]:
                _page_gen_lock.release()
                return None
            ai_call = _adapter
        job_id = str(uuid.uuid4())
        PAGE_GEN_JOBS[job_id] = {
            "job_id": job_id, "total": len(page_ids), "done": 0, "generated": 0,
            "skipped": 0, "errors": 0, "error_msgs": [], "current_slug": None,
            "finished": False, "started_at": now_utc(),
        }
        # Prune oldest finished jobs to keep PAGE_GEN_JOBS bounded.
        finished_ids = [jid for jid, j in PAGE_GEN_JOBS.items() if j.get("finished")]
        for jid in finished_ids[:-10]:
            PAGE_GEN_JOBS.pop(jid, None)
    except Exception:
        _page_gen_lock.release()
        raise
    def _wrapped():
        with app.app_context():
            try:
                _run_page_gen_batch(job_id, page_ids, master, ai_call)
            except Exception as e:
                job = PAGE_GEN_JOBS.get(job_id)
                if job:
                    job["finished"] = True
                    job["error_msgs"].append(f"Batch worker crashed: {e}")
                    job["current_slug"] = None
            finally:
                _page_gen_lock.release()
    threading.Thread(target=_wrapped, daemon=True).start()
    return job_id


def process_due_reply_sends(now_iso=None):
    """Send every scheduled reply whose time has come. Returns (sent, failed).
    Each link is claimed atomically (scheduled -> sending) so multiple processes
    (e.g. Passenger workers) can never double-send the same reply."""
    due = lead_dispatch.due_scheduled_links(get_db(), now_iso or now_utc())
    sent = failed = 0
    for link in due:
        try:
            cur = get_db().execute(
                "UPDATE campaign_links SET reply_status='sending' WHERE id=? AND reply_status='scheduled'",
                (link["id"],),
            )
            get_db().commit()
            if cur.rowcount != 1:
                continue  # another process/tick claimed it first
            ok, msg = send_campaign_link_reply(link, get_settings())
        except Exception as e:
            ok, msg = False, str(e)
            get_db().execute(
                "UPDATE campaign_links SET reply_status='scheduled' WHERE id=? AND reply_status='sending'",
                (link["id"],),
            )
            get_db().commit()
            continue
        if ok:
            _mark_link_sent(link["id"], link["generated_reply"])
            sent += 1
        else:
            _mark_link_send_failed(link["id"], msg)
            failed += 1
    return sent, failed


_reply_send_scheduler_started = False


def start_reply_send_scheduler():
    """Start the scheduled-reply send daemon once (30s ticks)."""
    global _reply_send_scheduler_started
    if _reply_send_scheduler_started:
        return _reply_send_scheduler_started
    _reply_send_scheduler_started = True

    def _loop():
        while True:
            try:
                time.sleep(30)
                with app.app_context():
                    process_due_reply_sends()
            except Exception as e:
                logging.getLogger(__name__).warning(f"reply-send scheduler tick failed: {e}")

    threading.Thread(target=_loop, daemon=True).start()
    return _reply_send_scheduler_started


def start_page_generation_scheduler():
    """Start the scheduled page-generation daemon thread once. Returns the started-flag."""
    global _page_gen_scheduler_started
    if _page_gen_scheduler_started:
        return _page_gen_scheduler_started
    _page_gen_scheduler_started = True
    def _loop():
        while True:
            try:
                time.sleep(900)
                with app.app_context():
                    settings = get_settings()
                    enabled = settings["pages_generation_enabled"] if "pages_generation_enabled" in settings.keys() else 0
                    if not enabled:
                        continue
                    last = settings["pages_generation_last_run"] if "pages_generation_last_run" in settings.keys() else None
                    interval = int(settings["pages_generation_interval_hours"] or 24)
                    if last:
                        try:
                            from datetime import datetime, timedelta
                            if datetime.utcnow() - datetime.fromisoformat(last) < timedelta(hours=interval):
                                continue
                        except Exception:
                            pass
                    trigger_page_generation_batch(page_ids=None)
            except Exception as e:
                logging.getLogger(__name__).warning(f"page-gen scheduler tick failed: {e}")
    threading.Thread(target=_loop, daemon=True).start()
    return _page_gen_scheduler_started


register_trust_features(app, login_required, ADMIN_URL_PREFIX)

ensure_directories()
with app.app_context():
    init_db()

# NOTE: the reply-send daemon is NOT started here on import — it would fire real
# sends during tests. It is started by the two real entry points: __main__ (dev)
# and passenger_wsgi.py (production). Guarded against double-start.


if __name__ == "__main__":
    start_page_generation_scheduler()
    start_reply_send_scheduler()
    resume_turbo_campaigns()
    app.run(host="127.0.0.1", port=5000, debug=True)