feat: AI visual barcode fallback after 5s with settings toggle

When the barcode scanner cannot read a code within 5 seconds and Gemini is available, a camera frame is automatically captured and sent to the new gemini_barcode_visual endpoint for visual product identification. The result pre-fills the product form identically to a barcode scan. - PHP: new geminiBarcodeVisual() function + router case + aiActions entry - PHP: barcode_ai_fallback setting in getServerSettings() + saveSettings() boolMap - JS: _aiFallbackTimer (cleared on detection/stop), 5s timer in initScanner() - JS: _tryGeminiVisualBarcode() — captures JPEG frame, calls API, saves product - JS: barcode_ai_fallback wired into serverKeys, applyUI, collectUI, POST body - HTML: AI fallback toggle in Settings → Camera card - Translations: ai_fallback_* strings in scan + settings.camera (it/en/de/fr/es) Feature is disabled by default (BARCODE_AI_FALLBACK=false).
2026-05-29 17:37:37 +00:00
parent 758eb93e20
commit 98c38f017e
9 changed files with 7366 additions and 7126 deletions
@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 - **Recipe scraps tips** — During cooking steps, detect "waste" generated (peels, cores, bones, eggshells, coffee grounds, citrus zest, etc.) and surface AI-powered tips on how to reuse them (compost, natural cleaner, broth, candied peel, etc.). Could be shown as an optional collapsible hint card below the step that generates the scrap.

+## [1.7.34] - 2026-05-30
+
+### Added
+- **AI visual barcode fallback** — When the barcode scanner fails to read a barcode within 5 seconds, EverShelf can now automatically capture a camera frame and send it to Gemini Vision to visually identify the product (name, brand, category). On success the product is saved and the inventory form opens just as if a barcode had been scanned. A new toggle in **Settings → Camera** (`AI visual identification (5s fallback)`) lets users enable or disable this feature at any time. Requires Gemini API key configured. Disabled by default.
+
 ## [1.7.33] - 2026-05-29

 ### Fixed
@@ -604,7 +604,7 @@ function checkRateLimit(string $action): void {
    }

    // Determine limit based on action
-    $aiActions = ['gemini_readExpiry', 'gemini_chat', 'gemini_identify', 'gemini_suggest_shopping', 'chat_to_recipe', 'recipe_from_ingredient', 'gemini_number_ocr'];
+    $aiActions = ['gemini_readExpiry', 'gemini_chat', 'gemini_identify', 'gemini_suggest_shopping', 'chat_to_recipe', 'recipe_from_ingredient', 'gemini_number_ocr', 'gemini_barcode_visual'];
    $loginActions = [];
    $recipeActions = ['generate_recipe', 'generate_recipe_stream'];
    $errorActions = ['report_error', 'check_update'];
@@ -1109,6 +1109,10 @@ try {
            geminiNumberOCR();
            break;

+        case 'gemini_barcode_visual':
+            geminiBarcodeVisual();
+            break;
+
        case 'get_shopping_price':
            getShoppingPrice($db);
            break;
@@ -4353,6 +4357,7 @@ function getServerSettings(): void {
        'shopping_forecast'           => env('SHOPPING_FORECAST', 'true') === 'true',
        'shopping_auto_add_threshold' => (int)env('SHOPPING_AUTO_ADD_THRESHOLD', '0'),
        'dark_mode'                   => env('DARK_MODE', 'auto'),
+        'barcode_ai_fallback'         => env('BARCODE_AI_FALLBACK', 'false') === 'true',
        // Home Assistant Integration
        'ha_enabled'                  => env('HA_ENABLED', 'false') === 'true',
        'ha_url'                      => env('HA_URL', ''),
@@ -4455,6 +4460,7 @@ function saveSettings(): void {
        'shopping_enabled'           => 'SHOPPING_ENABLED',
        'shopping_smart_suggestions' => 'SHOPPING_SMART_SUGGESTIONS',
        'shopping_forecast'          => 'SHOPPING_FORECAST',
+        'barcode_ai_fallback' => 'BARCODE_AI_FALLBACK',
        // Home Assistant
        'ha_enabled'    => 'HA_ENABLED',
    ];
@@ -10494,6 +10500,101 @@ function geminiNumberOCR(): void {
    }
 }

+// =============================================================================
+// ===== GEMINI AI: BARCODE VISUAL FALLBACK ====================================
+// =============================================================================
+/**
+ * POST /api/?action=gemini_barcode_visual
+ * Body: { image: base64-jpeg, lang: 'it'|'en'|'de'|... }
+ * Returns: { found, source, product } or { found: false, error }
+ * Uses Gemini vision to visually identify a product from a camera frame
+ * when the barcode scanner fails to read the barcode after 5 seconds.
+ */
+function geminiBarcodeVisual(): void {
+    EverLog::info('geminiBarcodeVisual');
+    $apiKey = env('GEMINI_API_KEY');
+    if (empty($apiKey)) {
+        echo json_encode(['found' => false, 'error' => 'no_api_key']);
+        return;
+    }
+
+    $input = json_decode(file_get_contents('php://input'), true) ?? [];
+    $imageBase64 = $input['image'] ?? '';
+    $lang = $input['lang'] ?? 'it';
+    if (empty($imageBase64)) {
+        echo json_encode(['found' => false, 'error' => 'no_image']);
+        return;
+    }
+
+    $langNote = match($lang) {
+        'de'    => 'Use the German product name if known.',
+        'fr'    => 'Use the French product name if known.',
+        'es'    => 'Use the Spanish product name if known.',
+        default => 'Use the Italian product name if known.',
+    };
+
+    $payload = [
+        'contents' => [[
+            'parts' => [
+                ['text' => "Identify the product shown in this image. {$langNote}\n" .
+                           "Respond with ONLY valid JSON (no markdown, no backticks):\n" .
+                           "{\"name\":\"...\",\"brand\":\"...\",\"category\":\"...\"}\n" .
+                           "- name: the product name (as specific as possible, not just the brand)\n" .
+                           "- brand: the brand/manufacturer, or empty string if not visible\n" .
+                           "- category: one of: latticini, pasta, bevande, snack, carne, pesce, " .
+                           "frutta, verdura, surgelati, condimenti, conserve, cereali, pane, " .
+                           "igiene, pulizia, altro\n" .
+                           "If you cannot identify the product at all, respond with: {\"unknown\":true}"],
+                ['inline_data' => ['mime_type' => 'image/jpeg', 'data' => $imageBase64]],
+            ],
+        ]],
+        'generationConfig' => [
+            'temperature'      => 0,
+            'maxOutputTokens'  => 200,
+            'responseMimeType' => 'application/json',
+            'thinkingConfig'   => ['thinkingBudget' => 0],
+        ],
+    ];
+
+    $result = callGeminiWithFallback($apiKey, $payload, 15, 'barcode_visual');
+    if ($result['http_code'] !== 200) {
+        echo json_encode(['found' => false, 'error' => 'gemini_error_' . $result['http_code']]);
+        return;
+    }
+
+    $text = trim($result['data']['candidates'][0]['content']['parts'][0]['text'] ?? '');
+    // Strip accidental markdown fences
+    $text = preg_replace('/^```json\s*/i', '', $text);
+    $text = preg_replace('/\s*```$/i', '', trim($text));
+
+    $data = json_decode($text, true);
+    if (!$data || !empty($data['unknown']) || empty($data['name'])) {
+        echo json_encode(['found' => false]);
+        return;
+    }
+
+    echo json_encode([
+        'found'   => true,
+        'source'  => 'gemini_visual',
+        'product' => [
+            'name'          => $data['name']     ?? '',
+            'brand'         => $data['brand']    ?? '',
+            'category'      => $data['category'] ?? '',
+            'image_url'     => '',
+            'quantity_info' => '',
+            'nutriscore'    => '',
+            'ingredients'   => '',
+            'allergens'     => '',
+            'conservation'  => '',
+            'origin'        => '',
+            'nova_group'    => '',
+            'ecoscore'      => '',
+            'labels'        => '',
+            'stores'        => '',
+        ],
+    ], JSON_UNESCAPED_UNICODE);
+}
+
 // =============================================================================
 // ===== GEMINI AI: ANOMALY EXPLANATION =======================================
 // =============================================================================
@@ -1943,6 +1943,7 @@ let quaggaRunning = false;
 let aiStream = null;
 let _scanZoomLevel = 2; // always 2x
 let _torchActive = false;
+let _aiFallbackTimer = null;

 // Apply fixed 2x zoom (hardware if available, CSS fallback)
 async function _applyFixedZoom() {
@@ -2111,6 +2112,85 @@ async function _tryGeminiNumberOCR() {
    }
 }

+// ===== AI VISUAL PRODUCT IDENTIFICATION (auto-fallback after 5s) =====
+let _aiBarcodeVisualRunning = false;
+async function _tryGeminiVisualBarcode() {
+    if (_aiBarcodeVisualRunning || !_requireGemini()) return;
+    const video = document.getElementById('scanner-video');
+    if (!video || !video.videoWidth) return;
+
+    _aiBarcodeVisualRunning = true;
+    stopScanner(); // stop scanner loop while AI processes
+    _setScanStatus(t('scan.ai_fallback_searching'), 'retry', 'Gemini Vision');
+    showLoading(true);
+
+    try {
+        const canvas = document.createElement('canvas');
+        canvas.width  = video.videoWidth;
+        canvas.height = video.videoHeight;
+        canvas.getContext('2d').drawImage(video, 0, 0);
+        const imageBase64 = canvas.toDataURL('image/jpeg', 0.88).split(',')[1];
+
+        const result = await api('gemini_barcode_visual', {}, 'POST', {
+            image: imageBase64,
+            lang: _currentLang || 'it',
+        });
+
+        if (result.found && result.product) {
+            const p = result.product;
+            scanLog(`AI visual: found "${p.name}" (${p.brand})`);
+            showToast(t('scan.ai_fallback_found'), 'success');
+            // Build a synthetic product (no barcode) and show the inventory form
+            const saveResult = await api('product_save', {}, 'POST', {
+                barcode: '',
+                name:    p.name || t('product.not_recognized'),
+                brand:   p.brand || '',
+                category: p.category || '',
+                image_url: '',
+                unit: 'pz',
+                default_quantity: 1,
+                package_unit: '',
+                notes: '',
+            });
+            if (saveResult.id) {
+                currentProduct = {
+                    id: saveResult.id,
+                    barcode: '',
+                    name: p.name || t('product.not_recognized'),
+                    brand: p.brand || '',
+                    category: p.category || '',
+                    image_url: '',
+                    unit: 'pz',
+                    default_quantity: 1,
+                    package_unit: '',
+                    _confCount: 0,
+                    weight_info: '',
+                };
+                addToScanRecents(currentProduct);
+                showLoading(false);
+                setTimeout(() => showProductAction(), 300);
+            } else {
+                showLoading(false);
+                showToast(t('error.connection'), 'error');
+            }
+        } else {
+            scanLog('AI visual: product not identified');
+            showLoading(false);
+            showToast(t('scan.ai_fallback_not_found'), 'warning');
+            _setScanStatus(t('scan.status_ready'), '', '');
+            // Restart scanner so user can try again
+            setTimeout(() => initScanner(), 300);
+        }
+    } catch (e) {
+        scanLog(`AI visual error: ${e.message}`);
+        showLoading(false);
+        showToast(t('error.connection'), 'error');
+        setTimeout(() => initScanner(), 300);
+    } finally {
+        _aiBarcodeVisualRunning = false;
+    }
+}
+
 // ===== CAMERA HELPER =====
 function getCameraConstraints(extraVideo = {}) {
    const s = getSettings();
@@ -2296,6 +2376,7 @@ function _applySyncedSettings(serverSettings) {
        'shopping_enabled','shopping_mode','shopping_smart_suggestions',
        'shopping_forecast','shopping_auto_add_threshold',
        'dark_mode',
+        'barcode_ai_fallback',
        // Home Assistant
        'ha_enabled','ha_url','ha_tts_entity','ha_webhook_id','ha_webhook_events',
        'ha_notify_service','ha_expiry_days'];
@@ -2887,6 +2968,8 @@ async function loadSettingsUI() {
    const cameraSelect = document.getElementById('setting-camera-facing');
    if (cameraSelect) cameraSelect.value = s.camera_facing || 'environment';
    loadCameraDevices();
+    const baifEl = document.getElementById('setting-barcode-ai-fallback');
+    if (baifEl) baifEl.checked = s.barcode_ai_fallback === true;
    renderAppliances(s.appliances || []);
    const mealPlanEnabled = s.meal_plan_enabled !== false;
    const mpEnabledEl = document.getElementById('setting-meal-plan-enabled');
@@ -3465,6 +3548,8 @@ async function saveSettings() {
    s.dietary = document.getElementById('setting-dietary').value.trim();
    // Camera
    s.camera_facing = document.getElementById('setting-camera-facing').value;
+    const baifSave = document.getElementById('setting-barcode-ai-fallback');
+    if (baifSave) s.barcode_ai_fallback = baifSave.checked;
    // Screensaver
    const ssEl = document.getElementById('setting-screensaver-enabled');
    if (ssEl) s.screensaver_enabled = ssEl.checked;
@@ -3608,6 +3693,7 @@ async function saveSettings() {
            shopping_forecast:           s.shopping_forecast !== false,
            shopping_auto_add_threshold: s.shopping_auto_add_threshold || 0,
            dark_mode:                   s.dark_mode || 'auto',
+            barcode_ai_fallback:          !!s.barcode_ai_fallback,
            // Home Assistant
            ha_enabled:         !!s.ha_enabled,
            ha_url:             s.ha_url || '',
@@ -6528,6 +6614,17 @@ async function initScanner() {
            }, 4000);
        }

+        // After 5s without a scan, auto-trigger AI visual identification (if enabled)
+        if (_geminiAvailable && getSettings().barcode_ai_fallback) {
+            clearTimeout(_aiFallbackTimer);
+            _aiFallbackTimer = setTimeout(() => {
+                if (scannerStream) { // still scanning — no barcode found yet
+                    scanLog('5s elapsed without barcode — triggering AI visual fallback');
+                    _tryGeminiVisualBarcode();
+                }
+            }, 5000);
+        }
+        
    } catch (err) {
        scanLog(`CAMERA ERROR: ${err.name}: ${err.message}`);
        console.error('Camera error:', err);
@@ -6850,6 +6947,7 @@ function stopScanner() {
    quaggaRunning = false;
    _scanZoomLevel = 2; // always 2x on next start
    _torchActive = false;
+    clearTimeout(_aiFallbackTimer); _aiFallbackTimer = null;
    if (scannerStream) {
        scannerStream.getTracks().forEach(t => t.stop());
        scannerStream = null;
@@ -6871,6 +6969,7 @@ function stopScanner() {
 }

 async function onBarcodeDetected(barcode) {
+    clearTimeout(_aiFallbackTimer); _aiFallbackTimer = null;
    showLoading(true);
    
    // Vibrate if available
@@ -1183,6 +1183,16 @@
                        <p class="settings-hint mt-2" data-i18n="settings.camera.devices_hint">Se hai più fotocamere, puoi selezionarne una specifica dall'elenco sopra dopo aver concesso i permessi.</p>
                        <button class="btn btn-small btn-secondary mt-2" onclick="loadCameraDevices()" data-i18n="settings.camera.detect_btn">🔄 Rileva fotocamere</button>
                    </div>
+                    <div class="form-group" style="margin-top:14px">
+                        <label class="toggle-row">
+                            <span data-i18n="settings.camera.ai_fallback_label">Identificazione visiva AI (fallback 5s)</span>
+                            <span class="toggle-switch">
+                                <input type="checkbox" id="setting-barcode-ai-fallback">
+                                <span class="toggle-slider"></span>
+                            </span>
+                        </label>
+                        <p class="settings-hint mt-2" data-i18n="settings.camera.ai_fallback_hint">Se il codice a barre non viene letto entro 5 secondi, un fotogramma viene inviato automaticamente all'AI per identificare visivamente il prodotto. Richiede Gemini configurato.</p>
+                    </div>
                </div>
            </div>
            <!-- Security Tab -->
@@ -220,7 +220,10 @@
    "status_partial": "Erkannt: {code} — prüfe...",
    "status_invalid": "Ungültig: {code} — versuche erneut",
    "status_confirmed": "Bestätigt!",
-        "status_parallel": "Kombinierter Scan aktiv..."
+    "status_parallel": "Kombinierter Scan aktiv...",
+    "ai_fallback_searching": "KI identifiziert Produkt...",
+    "ai_fallback_found": "Produkt von KI erkannt",
+    "ai_fallback_not_found": "KI: Produkt nicht erkannt"
  },
  "action": {
    "title": "Was möchtest du tun?",
@@ -664,7 +667,9 @@
      "back": "📱 Rückkamera (Standard)",
      "front": "🤳 Frontkamera",
      "devices_hint": "Bei mehreren Kameras kannst du nach Freigabe der Berechtigungen eine bestimmte aus der Liste oben wählen.",
-            "detect_btn": "🔄 Kameras erkennen"
+      "detect_btn": "🔄 Kameras erkennen",
+      "ai_fallback_label": "KI-Bilderkennung (5s Fallback)",
+      "ai_fallback_hint": "Wird kein Barcode innerhalb von 5 Sekunden gelesen, wird automatisch ein Bild an die KI zur visuellen Produktidentifizierung gesendet. Erfordert konfiguriertes Gemini."
    },
    "security": {
      "title": "🔒 HTTPS-Zertifikat",
@@ -220,7 +220,10 @@
    "status_partial": "Detected: {code} — verifying...",
    "status_invalid": "Invalid: {code} — retrying",
    "status_confirmed": "Confirmed!",
-        "status_parallel": "Using combined scan methods..."
+    "status_parallel": "Using combined scan methods...",
+    "ai_fallback_searching": "AI identifying product...",
+    "ai_fallback_found": "Product identified by AI",
+    "ai_fallback_not_found": "AI: product not recognized"
  },
  "action": {
    "title": "What do you want to do?",
@@ -664,7 +667,9 @@
      "back": "📱 Rear (default)",
      "front": "🤳 Front",
      "devices_hint": "If you have multiple cameras, you can select a specific one from the list above after granting permissions.",
-            "detect_btn": "🔄 Detect cameras"
+      "detect_btn": "🔄 Detect cameras",
+      "ai_fallback_label": "AI visual identification (5s fallback)",
+      "ai_fallback_hint": "If no barcode is read within 5 seconds, a frame is automatically sent to AI to visually identify the product. Requires Gemini configured."
    },
    "security": {
      "title": "🔒 HTTPS Certificate",
@@ -217,7 +217,10 @@
    "status_partial": "Detectado: {code} — verificando...",
    "status_invalid": "Inválido: {code} — reintentando",
    "status_confirmed": "Confirmado!",
-        "status_parallel": "Escaneo combinado activo..."
+    "status_parallel": "Escaneo combinado activo...",
+    "ai_fallback_searching": "Identificación de IA en curso...",
+    "ai_fallback_found": "Producto identificado por IA",
+    "ai_fallback_not_found": "IA: producto no reconocido"
  },
  "action": {
    "title": "¿Qué quieres hacer?",
@@ -658,7 +661,9 @@
      "back": "📱 Trasera (por defecto)",
      "front": "🤳 Frontal",
      "devices_hint": "Si tienes varias cámaras, puedes seleccionar una específica de la lista de arriba tras conceder los permisos.",
-            "detect_btn": "🔄 Detectar cámaras"
+      "detect_btn": "🔄 Detectar cámaras",
+      "ai_fallback_label": "Identificación visual IA (repuesto 5s)",
+      "ai_fallback_hint": "Si no se lee ningún código de barras en 5 segundos, se envía automáticamente un fotograma a la IA para identificar el producto visualmente. Requiere Gemini configurado."
    },
    "security": {
      "title": "🔒 Certificado HTTPS",
@@ -217,7 +217,10 @@
    "status_partial": "Lu : {code} — vérification...",
    "status_invalid": "Invalide : {code} — nouvel essai",
    "status_confirmed": "Confirmé !",
-        "status_parallel": "Scan combiné actif..."
+    "status_parallel": "Scan combiné actif...",
+    "ai_fallback_searching": "Identification IA en cours...",
+    "ai_fallback_found": "Produit identifié par l'IA",
+    "ai_fallback_not_found": "IA : produit non reconnu"
  },
  "action": {
    "title": "Que voulez-vous faire ?",
@@ -658,7 +661,9 @@
      "back": "📱 Arrière (par défaut)",
      "front": "🤳 Frontale",
      "devices_hint": "Si vous avez plusieurs caméras, vous pouvez en sélectionner une dans la liste ci-dessus après avoir accordé les permissions.",
-            "detect_btn": "🔄 Détecter les caméras"
+      "detect_btn": "🔄 Détecter les caméras",
+      "ai_fallback_label": "Identification visuelle IA (repli 5s)",
+      "ai_fallback_hint": "Si aucun code-barres n'est lu en 5 secondes, une image est automatiquement envoyée à l'IA pour identifier visuellement le produit. Nécessite Gemini configuré."
    },
    "security": {
      "title": "🔒 Certificat HTTPS",
@@ -220,7 +220,10 @@
    "status_partial": "Letto: {code} — verifico...",
    "status_invalid": "Non valido: {code} — riprovo",
    "status_confirmed": "Confermato!",
-        "status_parallel": "Doppia scansione attiva..."
+    "status_parallel": "Doppia scansione attiva...",
+    "ai_fallback_searching": "Identificazione AI in corso...",
+    "ai_fallback_found": "Prodotto identificato dall'AI",
+    "ai_fallback_not_found": "AI: prodotto non riconosciuto"
  },
  "action": {
    "title": "Cosa vuoi fare?",
@@ -664,7 +667,9 @@
      "back": "📱 Posteriore (default)",
      "front": "🤳 Anteriore",
      "devices_hint": "Se hai più fotocamere, puoi selezionarne una specifica dall'elenco sopra dopo aver concesso i permessi.",
-            "detect_btn": "🔄 Rileva fotocamere"
+      "detect_btn": "🔄 Rileva fotocamere",
+      "ai_fallback_label": "Identificazione visiva AI (fallback 5s)",
+      "ai_fallback_hint": "Se il codice a barre non viene letto entro 5 secondi, un fotogramma viene inviato automaticamente all'AI per identificare il prodotto visivamente. Richiede Gemini configurato."
    },
    "security": {
      "title": "🔒 Certificato HTTPS",