306 lines
10 KiB
PHP
306 lines
10 KiB
PHP
<?php
|
|
|
|
namespace App\Services;
|
|
|
|
use Illuminate\Support\Facades\Http;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Facades\Process;
|
|
|
|
class DocumentIngestionService
|
|
{
|
|
protected WeaviateService $weaviate;
|
|
|
|
public function __construct(WeaviateService $weaviate)
|
|
{
|
|
$this->weaviate = $weaviate;
|
|
}
|
|
|
|
/**
|
|
* Ingest content from a URL for a specific chatbot.
|
|
*/
|
|
public function ingestUrl(string $url, string $chatbotId): bool
|
|
{
|
|
try {
|
|
$response = Http::timeout(15)->get($url);
|
|
if (!$response->successful()) {
|
|
Log::error("Failed to fetch URL for ingestion: {$url}");
|
|
return false;
|
|
}
|
|
|
|
$html = $response->body();
|
|
|
|
// Extract Title
|
|
preg_match('/<title>(.*?)<\/title>/is', $html, $matches);
|
|
$title = isset($matches[1]) ? trim($matches[1]) : $url;
|
|
|
|
// Extract Clean Text Content
|
|
$text = $this->cleanHtml($html);
|
|
|
|
if (empty($text)) {
|
|
Log::warning("No text extracted from URL: {$url}");
|
|
return false;
|
|
}
|
|
|
|
// Chunk and Index
|
|
$chunks = $this->chunkText($text, 800, 150);
|
|
$formattedChunks = [];
|
|
foreach ($chunks as $chunk) {
|
|
$formattedChunks[] = [
|
|
'content' => $chunk,
|
|
'source' => $url,
|
|
'title' => $title
|
|
];
|
|
}
|
|
|
|
return $this->weaviate->indexChunks($formattedChunks, $chatbotId);
|
|
} catch (\Exception $e) {
|
|
Log::error("Error ingesting URL {$url} for chatbot {$chatbotId}: " . $e->getMessage());
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Crawl and ingest an entire website starting from a base URL.
|
|
*/
|
|
public function crawlAndIngestSite(string $baseUrl, string $chatbotId, int $maxPages = 50): int
|
|
{
|
|
$parsedBase = parse_url($baseUrl);
|
|
$baseHost = $parsedBase['host'] ?? '';
|
|
$baseScheme = $parsedBase['scheme'] ?? 'http';
|
|
if (empty($baseHost)) {
|
|
Log::error("Invalid base URL for site crawl: {$baseUrl}");
|
|
return 0;
|
|
}
|
|
|
|
$queue = [$baseUrl];
|
|
$visited = [];
|
|
$indexedCount = 0;
|
|
|
|
Log::info("Starting site crawl of {$baseUrl} for chatbot {$chatbotId} (max pages: {$maxPages})");
|
|
|
|
while (!empty($queue) && $indexedCount < $maxPages) {
|
|
$url = array_shift($queue);
|
|
|
|
// Normalize URL
|
|
$url = strtok($url, '#');
|
|
$url = rtrim($url, '/');
|
|
|
|
if (isset($visited[$url])) {
|
|
continue;
|
|
}
|
|
$visited[$url] = true;
|
|
|
|
try {
|
|
Log::info("Crawler visiting URL [{$indexedCount}/{$maxPages}]: {$url}");
|
|
$response = Http::timeout(10)->get($url);
|
|
if (!$response->successful()) {
|
|
Log::warning("Crawler failed to fetch URL: {$url}");
|
|
continue;
|
|
}
|
|
|
|
// Check if response is HTML
|
|
$contentType = $response->header('Content-Type');
|
|
if ($contentType && strpos($contentType, 'text/html') === false) {
|
|
Log::info("Crawler skipping non-HTML response at URL: {$url}");
|
|
continue;
|
|
}
|
|
|
|
$html = $response->body();
|
|
|
|
// Extract Title
|
|
preg_match('/<title>(.*?)<\/title>/is', $html, $matches);
|
|
$title = isset($matches[1]) ? trim($matches[1]) : $url;
|
|
|
|
// Extract Clean Text Content
|
|
$text = $this->cleanHtml($html);
|
|
if (!empty($text)) {
|
|
$chunks = $this->chunkText($text, 800, 150);
|
|
$formattedChunks = [];
|
|
foreach ($chunks as $chunk) {
|
|
$formattedChunks[] = [
|
|
'content' => $chunk,
|
|
'source' => $url,
|
|
'title' => $title
|
|
];
|
|
}
|
|
|
|
if (!empty($formattedChunks)) {
|
|
$this->weaviate->indexChunks($formattedChunks, $chatbotId);
|
|
$indexedCount++;
|
|
}
|
|
}
|
|
|
|
// Discover links
|
|
preg_match_all('/<a\s+[^>]*href="([^"]+)"/i', $html, $linkMatches);
|
|
if (isset($linkMatches[1])) {
|
|
foreach ($linkMatches[1] as $link) {
|
|
$resolvedLink = $this->resolveUrl($link, $url);
|
|
if (!$resolvedLink) {
|
|
continue;
|
|
}
|
|
|
|
$parsedLink = parse_url($resolvedLink);
|
|
$linkHost = $parsedLink['host'] ?? '';
|
|
|
|
// Only follow same host links using http/https protocols
|
|
if ($linkHost === $baseHost && in_array($parsedLink['scheme'] ?? '', ['http', 'https'])) {
|
|
$resolvedLink = strtok($resolvedLink, '#');
|
|
$resolvedLink = rtrim($resolvedLink, '/');
|
|
|
|
if (!isset($visited[$resolvedLink]) && !in_array($resolvedLink, $queue)) {
|
|
$queue[] = $resolvedLink;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} catch (\Exception $e) {
|
|
Log::error("Crawler error visiting URL {$url}: " . $e->getMessage());
|
|
}
|
|
}
|
|
|
|
Log::info("Crawler finished. Indexed {$indexedCount} pages.");
|
|
return $indexedCount;
|
|
}
|
|
|
|
/**
|
|
* Resolve a relative URL relative to a base URL.
|
|
*/
|
|
protected function resolveUrl(string $rel, string $base): ?string
|
|
{
|
|
if (preg_match('/^(mailto|tel|javascript|#):/i', $rel) || strpos($rel, '#') === 0) {
|
|
return null;
|
|
}
|
|
|
|
if (parse_url($rel, PHP_URL_SCHEME) != '') {
|
|
return $rel;
|
|
}
|
|
|
|
$baseParts = parse_url($base);
|
|
$scheme = $baseParts['scheme'] ?? 'http';
|
|
$host = $baseParts['host'] ?? '';
|
|
$port = isset($baseParts['port']) ? ':' . $baseParts['port'] : '';
|
|
$path = $baseParts['path'] ?? '/';
|
|
|
|
if (strpos($rel, '//') === 0) {
|
|
return $scheme . ':' . $rel;
|
|
}
|
|
|
|
if (strpos($rel, '/') === 0) {
|
|
return $scheme . '://' . $host . $port . $rel;
|
|
}
|
|
|
|
$dir = preg_replace('/\/[^\/]*$/', '', $path);
|
|
return $scheme . '://' . $host . $port . $dir . '/' . $rel;
|
|
}
|
|
|
|
/**
|
|
* Ingest content from a PDF file for a specific chatbot using the Python script.
|
|
*/
|
|
public function ingestPdf(string $filePath, string $originalName, string $chatbotId, bool $enableVision = false): bool
|
|
{
|
|
try {
|
|
if (!file_exists($filePath)) {
|
|
Log::error("PDF file not found: {$filePath}");
|
|
return false;
|
|
}
|
|
|
|
$scriptPath = base_path('scripts/parse_pdf.py');
|
|
$mistralKey = config('services.mistral.key') ?? env('MISTRAL_API_KEY', '');
|
|
$visionFlag = $enableVision ? '1' : '0';
|
|
|
|
// Run Python script
|
|
$result = Process::run([
|
|
'python',
|
|
$scriptPath,
|
|
$filePath,
|
|
$mistralKey,
|
|
$visionFlag
|
|
]);
|
|
|
|
if (!$result->successful()) {
|
|
Log::error("Python PDF parser script failed: " . $result->errorOutput());
|
|
return false;
|
|
}
|
|
|
|
$chunks = json_decode($result->output(), true);
|
|
|
|
if (isset($chunks['error'])) {
|
|
Log::error("Python PDF parser error: " . $chunks['error']);
|
|
return false;
|
|
}
|
|
|
|
if (empty($chunks)) {
|
|
Log::warning("No text chunks extracted from PDF: {$originalName}");
|
|
return false;
|
|
}
|
|
|
|
return $this->weaviate->indexChunks($chunks, $chatbotId);
|
|
} catch (\Exception $e) {
|
|
Log::error("Error ingesting PDF {$originalName} for chatbot {$chatbotId}: " . $e->getMessage());
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clean and strip HTML down to readability.
|
|
*/
|
|
protected function cleanHtml(string $html): string
|
|
{
|
|
// Remove scripts, styles, header, footer, nav
|
|
$html = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $html);
|
|
$html = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $html);
|
|
$html = preg_replace('/<header\b[^>]*>(.*?)<\/header>/is', '', $html);
|
|
$html = preg_replace('/<footer\b[^>]*>(.*?)<\/footer>/is', '', $html);
|
|
$html = preg_replace('/<nav\b[^>]*>(.*?)<\/nav>/is', '', $html);
|
|
|
|
// Strip tags
|
|
$text = strip_tags($html);
|
|
|
|
// Remove duplicate whitespace and newlines
|
|
$text = preg_replace('/[ \t]+/', ' ', $text);
|
|
$text = preg_replace('/\s*\n\s*/', "\n", $text);
|
|
$text = preg_replace('/\n+/', "\n\n", $text);
|
|
|
|
return trim($text);
|
|
}
|
|
|
|
/**
|
|
* Chunk text using sliding window.
|
|
*/
|
|
protected function chunkText(string $text, int $chunkSize = 800, int $overlap = 150): array
|
|
{
|
|
$words = explode(' ', $text);
|
|
$chunks = [];
|
|
$currentChunk = [];
|
|
$currentLength = 0;
|
|
|
|
foreach ($words as $word) {
|
|
$currentChunk[] = $word;
|
|
$currentLength += strlen($word) + 1; // +1 for space
|
|
|
|
if ($currentLength >= $chunkSize) {
|
|
$chunks[] = implode(' ', $currentChunk);
|
|
|
|
// Slide the window back by keeping some words for overlap
|
|
$overlapWordsCount = (int) ($overlap / 6); // estimate 6 chars per word
|
|
if ($overlapWordsCount > 0 && count($currentChunk) > $overlapWordsCount) {
|
|
$currentChunk = array_slice($currentChunk, -$overlapWordsCount);
|
|
$currentLength = strlen(implode(' ', $currentChunk));
|
|
} else {
|
|
$currentChunk = [];
|
|
$currentLength = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add remaining text
|
|
if (!empty($currentChunk)) {
|
|
$chunks[] = implode(' ', $currentChunk);
|
|
}
|
|
|
|
return array_filter($chunks, function ($chunk) {
|
|
return strlen(trim($chunk)) > 50; // Filter out tiny chunks
|
|
});
|
|
}
|
|
}
|