(.*?)<\/title>/is', $html, $matches); $title = isset($matches[1]) ? trim($matches[1]) : $url; // Extract Clean Text Content $text = $this->cleanHtml($html); if (empty($text)) { Log::warning("No text extracted from URL: {$url}"); return false; } // Chunk and Index $chunks = $this->chunkText($text, 800, 150); $formattedChunks = []; foreach ($chunks as $chunk) { $formattedChunks[] = [ 'content' => $chunk, 'source' => $url, 'title' => $title ]; } return $this->weaviate->indexChunks($formattedChunks, $chatbotId); } catch (\Exception $e) { Log::error("Error ingesting URL {$url} for chatbot {$chatbotId}: " . $e->getMessage()); return false; } } /** * Crawl and ingest an entire website starting from a base URL. */ public function crawlAndIngestSite(string $baseUrl, string $chatbotId, int $maxPages = 50): int { $parsedBase = parse_url($baseUrl); $baseHost = $parsedBase['host'] ?? ''; $baseScheme = $parsedBase['scheme'] ?? 'http'; if (empty($baseHost)) { Log::error("Invalid base URL for site crawl: {$baseUrl}"); return 0; } $queue = [$baseUrl]; $visited = []; $indexedCount = 0; Log::info("Starting site crawl of {$baseUrl} for chatbot {$chatbotId} (max pages: {$maxPages})"); while (!empty($queue) && $indexedCount < $maxPages) { $url = array_shift($queue); // Normalize URL $url = strtok($url, '#'); $url = rtrim($url, '/'); if (isset($visited[$url])) { continue; } $visited[$url] = true; try { Log::info("Crawler visiting URL [{$indexedCount}/{$maxPages}]: {$url}"); $response = Http::timeout(10)->get($url); if (!$response->successful()) { Log::warning("Crawler failed to fetch URL: {$url}"); continue; } // Check if response is HTML $contentType = $response->header('Content-Type'); if ($contentType && strpos($contentType, 'text/html') === false) { Log::info("Crawler skipping non-HTML response at URL: {$url}"); continue; } $html = $response->body(); // Extract Title preg_match('/<title>(.*?)<\/title>/is', $html, $matches); $title = isset($matches[1]) ? trim($matches[1]) : $url; // Extract Clean Text Content $text = $this->cleanHtml($html); if (!empty($text)) { $chunks = $this->chunkText($text, 800, 150); $formattedChunks = []; foreach ($chunks as $chunk) { $formattedChunks[] = [ 'content' => $chunk, 'source' => $url, 'title' => $title ]; } if (!empty($formattedChunks)) { $this->weaviate->indexChunks($formattedChunks, $chatbotId); $indexedCount++; } } // Discover links preg_match_all('/<a\s+[^>]*href="([^"]+)"/i', $html, $linkMatches); if (isset($linkMatches[1])) { foreach ($linkMatches[1] as $link) { $resolvedLink = $this->resolveUrl($link, $url); if (!$resolvedLink) { continue; } $parsedLink = parse_url($resolvedLink); $linkHost = $parsedLink['host'] ?? ''; // Only follow same host links using http/https protocols if ($linkHost === $baseHost && in_array($parsedLink['scheme'] ?? '', ['http', 'https'])) { $resolvedLink = strtok($resolvedLink, '#'); $resolvedLink = rtrim($resolvedLink, '/'); if (!isset($visited[$resolvedLink]) && !in_array($resolvedLink, $queue)) { $queue[] = $resolvedLink; } } } } } catch (\Exception $e) { Log::error("Crawler error visiting URL {$url}: " . $e->getMessage()); } } Log::info("Crawler finished. Indexed {$indexedCount} pages."); return $indexedCount; } /** * Resolve a relative URL relative to a base URL. */ protected function resolveUrl(string $rel, string $base): ?string { if (preg_match('/^(mailto|tel|javascript|#):/i', $rel) || strpos($rel, '#') === 0) { return null; } if (parse_url($rel, PHP_URL_SCHEME) != '') { return $rel; } $baseParts = parse_url($base); $scheme = $baseParts['scheme'] ?? 'http'; $host = $baseParts['host'] ?? ''; $port = isset($baseParts['port']) ? ':' . $baseParts['port'] : ''; $path = $baseParts['path'] ?? '/'; if (strpos($rel, '//') === 0) { return $scheme . ':' . $rel; } if (strpos($rel, '/') === 0) { return $scheme . '://' . $host . $port . $rel; } $dir = preg_replace('/\/[^\/]*$/', '', $path); return $scheme . '://' . $host . $port . $dir . '/' . $rel; } /** * Ingest content from a PDF file for a specific chatbot using the Python script. */ public function ingestPdf(string $filePath, string $originalName, string $chatbotId, bool $enableVision = false): bool { try { if (!file_exists($filePath)) { Log::error("PDF file not found: {$filePath}"); return false; } $scriptPath = base_path('scripts/parse_pdf.py'); $mistralKey = config('services.mistral.key') ?? env('MISTRAL_API_KEY', ''); $visionFlag = $enableVision ? '1' : '0'; // Run Python script $result = Process::run([ 'python', $scriptPath, $filePath, $mistralKey, $visionFlag ]); if (!$result->successful()) { Log::error("Python PDF parser script failed: " . $result->errorOutput()); return false; } $chunks = json_decode($result->output(), true); if (isset($chunks['error'])) { Log::error("Python PDF parser error: " . $chunks['error']); return false; } if (empty($chunks)) { Log::warning("No text chunks extracted from PDF: {$originalName}"); return false; } return $this->weaviate->indexChunks($chunks, $chatbotId); } catch (\Exception $e) { Log::error("Error ingesting PDF {$originalName} for chatbot {$chatbotId}: " . $e->getMessage()); return false; } } /** * Clean and strip HTML down to readability. */ protected function cleanHtml(string $html): string { // Remove scripts, styles, header, footer, nav $html = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $html); $html = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $html); $html = preg_replace('/<header\b[^>]*>(.*?)<\/header>/is', '', $html); $html = preg_replace('/<footer\b[^>]*>(.*?)<\/footer>/is', '', $html); $html = preg_replace('/<nav\b[^>]*>(.*?)<\/nav>/is', '', $html); // Strip tags $text = strip_tags($html); // Remove duplicate whitespace and newlines $text = preg_replace('/[ \t]+/', ' ', $text); $text = preg_replace('/\s*\n\s*/', "\n", $text); $text = preg_replace('/\n+/', "\n\n", $text); return trim($text); } /** * Chunk text using sliding window. */ protected function chunkText(string $text, int $chunkSize = 800, int $overlap = 150): array { $words = explode(' ', $text); $chunks = []; $currentChunk = []; $currentLength = 0; foreach ($words as $word) { $currentChunk[] = $word; $currentLength += strlen($word) + 1; // +1 for space if ($currentLength >= $chunkSize) { $chunks[] = implode(' ', $currentChunk); // Slide the window back by keeping some words for overlap $overlapWordsCount = (int) ($overlap / 6); // estimate 6 chars per word if ($overlapWordsCount > 0 && count($currentChunk) > $overlapWordsCount) { $currentChunk = array_slice($currentChunk, -$overlapWordsCount); $currentLength = strlen(implode(' ', $currentChunk)); } else { $currentChunk = []; $currentLength = 0; } } } // Add remaining text if (!empty($currentChunk)) { $chunks[] = implode(' ', $currentChunk); } return array_filter($chunks, function ($chunk) { return strlen(trim($chunk))

weaviate = $weaviate; } /** * Ingest content from a URL for a specific chatbot. */ public function ingestUrl(string $url, string $chatbotId): bool { try { $response = Http::timeout(15)->get($url); if (!$response->successful()) { Log::error("Failed to fetch URL for ingestion: {$url}"); return false; } $html = $response->body(); // Extract Title preg_match('/(.*?)<\/title>/is', $html, $matches); $title = isset($matches[1]) ? trim($matches[1]) : $url; // Extract Clean Text Content $text = $this->cleanHtml($html); if (empty($text)) { Log::warning("No text extracted from URL: {$url}"); return false; } // Chunk and Index $chunks = $this->chunkText($text, 800, 150); $formattedChunks = []; foreach ($chunks as $chunk) { $formattedChunks[] = [ 'content' => $chunk, 'source' => $url, 'title' => $title ]; } return $this->weaviate->indexChunks($formattedChunks, $chatbotId); } catch (\Exception $e) { Log::error("Error ingesting URL {$url} for chatbot {$chatbotId}: " . $e->getMessage()); return false; } } /** * Crawl and ingest an entire website starting from a base URL. */ public function crawlAndIngestSite(string $baseUrl, string $chatbotId, int $maxPages = 50): int { $parsedBase = parse_url($baseUrl); $baseHost = $parsedBase['host'] ?? ''; $baseScheme = $parsedBase['scheme'] ?? 'http'; if (empty($baseHost)) { Log::error("Invalid base URL for site crawl: {$baseUrl}"); return 0; } $queue = [$baseUrl]; $visited = []; $indexedCount = 0; Log::info("Starting site crawl of {$baseUrl} for chatbot {$chatbotId} (max pages: {$maxPages})"); while (!empty($queue) && $indexedCount < $maxPages) { $url = array_shift($queue); // Normalize URL $url = strtok($url, '#'); $url = rtrim($url, '/'); if (isset($visited[$url])) { continue; } $visited[$url] = true; try { Log::info("Crawler visiting URL [{$indexedCount}/{$maxPages}]: {$url}"); $response = Http::timeout(10)->get($url); if (!$response->successful()) { Log::warning("Crawler failed to fetch URL: {$url}"); continue; } // Check if response is HTML $contentType = $response->header('Content-Type'); if ($contentType && strpos($contentType, 'text/html') === false) { Log::info("Crawler skipping non-HTML response at URL: {$url}"); continue; } $html = $response->body(); // Extract Title preg_match('/<title>(.*?)<\/title>/is', $html, $matches); $title = isset($matches[1]) ? trim($matches[1]) : $url; // Extract Clean Text Content $text = $this->cleanHtml($html); if (!empty($text)) { $chunks = $this->chunkText($text, 800, 150); $formattedChunks = []; foreach ($chunks as $chunk) { $formattedChunks[] = [ 'content' => $chunk, 'source' => $url, 'title' => $title ]; } if (!empty($formattedChunks)) { $this->weaviate->indexChunks($formattedChunks, $chatbotId); $indexedCount++; } } // Discover links preg_match_all('/<a\s+[^>]*href="([^"]+)"/i', $html, $linkMatches); if (isset($linkMatches[1])) { foreach ($linkMatches[1] as $link) { $resolvedLink = $this->resolveUrl($link, $url); if (!$resolvedLink) { continue; } $parsedLink = parse_url($resolvedLink); $linkHost = $parsedLink['host'] ?? ''; // Only follow same host links using http/https protocols if ($linkHost === $baseHost && in_array($parsedLink['scheme'] ?? '', ['http', 'https'])) { $resolvedLink = strtok($resolvedLink, '#'); $resolvedLink = rtrim($resolvedLink, '/'); if (!isset($visited[$resolvedLink]) && !in_array($resolvedLink, $queue)) { $queue[] = $resolvedLink; } } } } } catch (\Exception $e) { Log::error("Crawler error visiting URL {$url}: " . $e->getMessage()); } } Log::info("Crawler finished. Indexed {$indexedCount} pages."); return $indexedCount; } /** * Resolve a relative URL relative to a base URL. */ protected function resolveUrl(string $rel, string $base): ?string { if (preg_match('/^(mailto|tel|javascript|#):/i', $rel) || strpos($rel, '#') === 0) { return null; } if (parse_url($rel, PHP_URL_SCHEME) != '') { return $rel; } $baseParts = parse_url($base); $scheme = $baseParts['scheme'] ?? 'http'; $host = $baseParts['host'] ?? ''; $port = isset($baseParts['port']) ? ':' . $baseParts['port'] : ''; $path = $baseParts['path'] ?? '/'; if (strpos($rel, '//') === 0) { return $scheme . ':' . $rel; } if (strpos($rel, '/') === 0) { return $scheme . '://' . $host . $port . $rel; } $dir = preg_replace('/\/[^\/]*$/', '', $path); return $scheme . '://' . $host . $port . $dir . '/' . $rel; } /** * Ingest content from a PDF file for a specific chatbot using the Python script. */ public function ingestPdf(string $filePath, string $originalName, string $chatbotId, bool $enableVision = false): bool { try { if (!file_exists($filePath)) { Log::error("PDF file not found: {$filePath}"); return false; } $scriptPath = base_path('scripts/parse_pdf.py'); $mistralKey = config('services.mistral.key') ?? env('MISTRAL_API_KEY', ''); $visionFlag = $enableVision ? '1' : '0'; // Run Python script $result = Process::run([ 'python', $scriptPath, $filePath, $mistralKey, $visionFlag ]); if (!$result->successful()) { Log::error("Python PDF parser script failed: " . $result->errorOutput()); return false; } $chunks = json_decode($result->output(), true); if (isset($chunks['error'])) { Log::error("Python PDF parser error: " . $chunks['error']); return false; } if (empty($chunks)) { Log::warning("No text chunks extracted from PDF: {$originalName}"); return false; } return $this->weaviate->indexChunks($chunks, $chatbotId); } catch (\Exception $e) { Log::error("Error ingesting PDF {$originalName} for chatbot {$chatbotId}: " . $e->getMessage()); return false; } } /** * Clean and strip HTML down to readability. */ protected function cleanHtml(string $html): string { // Remove scripts, styles, header, footer, nav $html = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $html); $html = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $html); $html = preg_replace('/<header\b[^>]*>(.*?)<\/header>/is', '', $html); $html = preg_replace('/<footer\b[^>]*>(.*?)<\/footer>/is', '', $html); $html = preg_replace('/<nav\b[^>]*>(.*?)<\/nav>/is', '', $html); // Strip tags $text = strip_tags($html); // Remove duplicate whitespace and newlines $text = preg_replace('/[ \t]+/', ' ', $text); $text = preg_replace('/\s*\n\s*/', "\n", $text); $text = preg_replace('/\n+/', "\n\n", $text); return trim($text); } /** * Chunk text using sliding window. */ protected function chunkText(string $text, int $chunkSize = 800, int $overlap = 150): array { $words = explode(' ', $text); $chunks = []; $currentChunk = []; $currentLength = 0; foreach ($words as $word) { $currentChunk[] = $word; $currentLength += strlen($word) + 1; // +1 for space if ($currentLength >= $chunkSize) { $chunks[] = implode(' ', $currentChunk); // Slide the window back by keeping some words for overlap $overlapWordsCount = (int) ($overlap / 6); // estimate 6 chars per word if ($overlapWordsCount > 0 && count($currentChunk) > $overlapWordsCount) { $currentChunk = array_slice($currentChunk, -$overlapWordsCount); $currentLength = strlen(implode(' ', $currentChunk)); } else { $currentChunk = []; $currentLength = 0; } } } // Add remaining text if (!empty($currentChunk)) { $chunks[] = implode(' ', $currentChunk); } return array_filter($chunks, function ($chunk) { return strlen(trim($chunk)) > 50; // Filter out tiny chunks }); } }