From 01aa115a493c6c3a8a5140b392a22bd5273e131a Mon Sep 17 00:00:00 2001 From: SitiWeb <70724099+SitiWeb@users.noreply.github.com> Date: Wed, 29 Apr 2026 13:21:52 +0200 Subject: [PATCH] Add helpdesk import progress, category model, article metadata columns, and ticket pagination controls --- .../ImportHelpdeskArticlesCommand.php | 18 ++- app/Livewire/Admin/TicketMonitor.php | 9 +- app/Models/Article.php | 22 ++- app/Models/Category.php | 22 +++ app/Services/HelpdeskImportService.php | 141 ++++++++++++++---- ...00_add_categories_and_article_metadata.php | 42 ++++++ .../livewire/admin/ticket-monitor.blade.php | 13 +- 7 files changed, 234 insertions(+), 33 deletions(-) create mode 100644 app/Models/Category.php create mode 100644 database/migrations/2026_04_29_001000_add_categories_and_article_metadata.php diff --git a/app/Console/Commands/ImportHelpdeskArticlesCommand.php b/app/Console/Commands/ImportHelpdeskArticlesCommand.php index 0f3b4f9..50e9da0 100644 --- a/app/Console/Commands/ImportHelpdeskArticlesCommand.php +++ b/app/Console/Commands/ImportHelpdeskArticlesCommand.php @@ -19,12 +19,28 @@ class ImportHelpdeskArticlesCommand extends Command $limitOption = $this->option('limit'); $limit = is_numeric($limitOption) ? (int) $limitOption : null; + $bar = null; + $result = $service->import( (string) $this->option('base-url'), (bool) $this->option('dry-run'), - $limit + $limit, + function (int $processed, int $total, string $url, string $status) use (&$bar): void { + if ($bar === null) { + $bar = $this->output->createProgressBar($total); + $bar->start(); + } + + $bar->advance(); + $bar->setMessage("{$status}: {$url}"); + } ); + if ($bar !== null) { + $bar->finish(); + $this->newLine(2); + } + $this->info('Helpdesk import finished.'); $this->table( ['Metric', 'Value'], diff --git a/app/Livewire/Admin/TicketMonitor.php b/app/Livewire/Admin/TicketMonitor.php index 9f67523..75428a3 100644 --- a/app/Livewire/Admin/TicketMonitor.php +++ b/app/Livewire/Admin/TicketMonitor.php @@ -10,10 +10,17 @@ class TicketMonitor extends Component { use WithPagination; + public int $perPage = 10; + + public function updatedPerPage(): void + { + $this->resetPage(); + } + public function render(AdminTicketService $service) { return view('livewire.admin.ticket-monitor', [ - 'tickets' => $service->paginateWithDecision(10), + 'tickets' => $service->paginateWithDecision($this->perPage), ]); } } diff --git a/app/Models/Article.php b/app/Models/Article.php index 44b6e1d..9e5a44c 100644 --- a/app/Models/Article.php +++ b/app/Models/Article.php @@ -4,11 +4,21 @@ namespace App\Models; use App\Casts\VectorCast; use Illuminate\Database\Eloquent\Model; +use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Eloquent\Relations\HasMany; class Article extends Model { - protected $fillable = ['title', 'content', 'embedding']; + protected $fillable = [ + 'title', + 'content', + 'embedding', + 'source', + 'source_url', + 'source_article_id', + 'category_id', + 'subcategory_id', + ]; protected $casts = [ 'embedding' => VectorCast::class, @@ -18,4 +28,14 @@ class Article extends Model { return $this->hasMany(AIDecision::class); } + + public function category(): BelongsTo + { + return $this->belongsTo(Category::class, 'category_id'); + } + + public function subcategory(): BelongsTo + { + return $this->belongsTo(Category::class, 'subcategory_id'); + } } diff --git a/app/Models/Category.php b/app/Models/Category.php new file mode 100644 index 0000000..8584cd8 --- /dev/null +++ b/app/Models/Category.php @@ -0,0 +1,22 @@ +belongsTo(self::class, 'parent_id'); + } + + public function children(): HasMany + { + return $this->hasMany(self::class, 'parent_id'); + } +} diff --git a/app/Services/HelpdeskImportService.php b/app/Services/HelpdeskImportService.php index 0cfb635..176b38d 100644 --- a/app/Services/HelpdeskImportService.php +++ b/app/Services/HelpdeskImportService.php @@ -3,6 +3,7 @@ namespace App\Services; use App\Models\Article; +use App\Models\Category; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; @@ -10,54 +11,73 @@ class HelpdeskImportService { private const DEFAULT_BASE_URL = 'https://www.internettoday.nl/helpdesk'; - public function import(?string $baseUrl = null, bool $dryRun = false, ?int $limit = null): array + public function import(?string $baseUrl = null, bool $dryRun = false, ?int $limit = null, ?callable $progress = null): array { $baseUrl = rtrim($baseUrl ?: self::DEFAULT_BASE_URL, '/'); $rootHtml = $this->fetch($baseUrl); $categories = $this->extractCategories($rootHtml); - $sectionUrls = $this->buildSectionUrls($baseUrl, $categories); - $articleUrls = $this->collectArticleUrls($baseUrl, $rootHtml, $sectionUrls); + $categoryMap = $this->syncCategories($categories, $dryRun); + $sections = $this->buildSections($baseUrl, $categories); + + $articleUrlMap = $this->collectArticleUrls($baseUrl, $rootHtml, $sections); if ($limit !== null && $limit > 0) { - $articleUrls = array_slice($articleUrls, 0, $limit); + $articleUrlMap = array_slice($articleUrlMap, 0, $limit, true); } + $total = count($articleUrlMap); $imported = 0; $updated = 0; $skipped = 0; + $processed = 0; - foreach ($articleUrls as $articleUrl) { + foreach ($articleUrlMap as $articleUrl => $meta) { + $processed++; $parsed = $this->parseArticlePage($articleUrl); if ($parsed === null) { $skipped++; + $progress && $progress($processed, $total, $articleUrl, 'skipped'); continue; } if ($dryRun) { $imported++; + $progress && $progress($processed, $total, $articleUrl, 'dry-run'); continue; } - [$title, $content] = $parsed; - $result = Article::withoutEvents(function () use ($title, $content) { + [$title, $content, $sourceArticleId] = $parsed; + + $categoryId = $this->resolveCategoryId($meta['category_external_id'] ?? null, $categoryMap); + $subcategoryId = $this->resolveCategoryId($meta['subcategory_external_id'] ?? null, $categoryMap); + + $result = Article::withoutEvents(function () use ($title, $content, $articleUrl, $sourceArticleId, $categoryId, $subcategoryId) { return Article::query()->updateOrCreate( - ['title' => $title], - ['content' => $content] + ['source' => 'internettoday_helpdesk', 'source_article_id' => $sourceArticleId], + [ + 'title' => $title, + 'content' => $content, + 'source_url' => $articleUrl, + 'category_id' => $categoryId, + 'subcategory_id' => $subcategoryId, + ] ); }); if ($result->wasRecentlyCreated) { $imported++; + $progress && $progress($processed, $total, $articleUrl, 'imported'); } else { $updated++; + $progress && $progress($processed, $total, $articleUrl, 'updated'); } } return [ 'categories' => count($categories), - 'sections' => count($sectionUrls), - 'article_urls' => count($articleUrls), + 'sections' => count($sections), + 'article_urls' => $total, 'imported' => $imported, 'updated' => $updated, 'skipped' => $skipped, @@ -67,11 +87,7 @@ class HelpdeskImportService private function fetch(string $url): string { - return Http::timeout(30) - ->retry(2, 300) - ->get($url) - ->throw() - ->body(); + return Http::timeout(30)->retry(2, 300)->get($url)->throw()->body(); } private function extractCategories(string $html): array @@ -84,46 +100,102 @@ class HelpdeskImportService return is_array($decoded) ? $decoded : []; } - private function buildSectionUrls(string $baseUrl, array $categories): array + private function syncCategories(array $categories, bool $dryRun): array { - $urls = []; + $map = []; + foreach ($categories as $category) { + if (!isset($category['id'], $category['title'], $category['slug'])) { + continue; + } + + $parentId = null; + if (!$dryRun) { + $model = Category::query()->updateOrCreate( + ['external_id' => (int) $category['id']], + ['name' => (string) $category['title'], 'slug' => (string) $category['slug'], 'parent_id' => null] + ); + $parentId = $model->id; + } + + $map[(int) $category['id']] = $parentId; + + foreach (($category['children'] ?? []) as $child) { + if (!isset($child['id'], $child['title'], $child['slug'])) { + continue; + } + + if (!$dryRun && $parentId !== null) { + $childModel = Category::query()->updateOrCreate( + ['external_id' => (int) $child['id']], + ['name' => (string) $child['title'], 'slug' => (string) $child['slug'], 'parent_id' => $parentId] + ); + $map[(int) $child['id']] = $childModel->id; + } else { + $map[(int) $child['id']] = null; + } + } + } + + return $map; + } + + private function buildSections(string $baseUrl, array $categories): array + { + $sections = []; foreach ($categories as $category) { if (!isset($category['id'], $category['slug'])) { continue; } - $urls[] = sprintf('%s/%d/%s', $baseUrl, (int) $category['id'], (string) $category['slug']); + $sections[] = [ + 'url' => sprintf('%s/%d/%s', $baseUrl, (int) $category['id'], (string) $category['slug']), + 'category_external_id' => (int) $category['id'], + 'subcategory_external_id' => null, + ]; foreach (($category['children'] ?? []) as $child) { if (!isset($child['id'], $child['slug'])) { continue; } - $urls[] = sprintf('%s/%d/%s', $baseUrl, (int) $child['id'], (string) $child['slug']); + $sections[] = [ + 'url' => sprintf('%s/%d/%s', $baseUrl, (int) $child['id'], (string) $child['slug']), + 'category_external_id' => (int) $category['id'], + 'subcategory_external_id' => (int) $child['id'], + ]; } } - return array_values(array_unique($urls)); + return $sections; } - private function collectArticleUrls(string $baseUrl, string $rootHtml, array $sectionUrls): array + private function collectArticleUrls(string $baseUrl, string $rootHtml, array $sections): array { - $urls = []; + $result = []; + $sources = array_merge([ + ['url' => $baseUrl, 'category_external_id' => null, 'subcategory_external_id' => null, 'html' => $rootHtml], + ], $sections); - foreach (array_merge([$baseUrl], $sectionUrls) as $url) { + foreach ($sources as $source) { try { - $html = $url === $baseUrl ? $rootHtml : $this->fetch($url); + $html = $source['html'] ?? $this->fetch($source['url']); } catch (\Throwable) { continue; } preg_match_all('/https:\/\/www\.internettoday\.nl\/helpdesk\/(\d+)-[a-z0-9\-]+/i', $html, $matches); foreach (($matches[0] ?? []) as $match) { - $urls[] = strtolower($match); + $url = strtolower($match); + if (!isset($result[$url])) { + $result[$url] = [ + 'category_external_id' => $source['category_external_id'], + 'subcategory_external_id' => $source['subcategory_external_id'], + ]; + } } } - return array_values(array_unique($urls)); + return $result; } private function parseArticlePage(string $url): ?array @@ -156,9 +228,20 @@ class HelpdeskImportService return null; } - $content = "Source: {$url}\n\n{$content}"; + if (!preg_match('/\/helpdesk\/(\d+)-/', $url, $idMatch)) { + return null; + } - return [$title, Str::limit($content, 64000, '')]; + return [$title, Str::limit($content, 64000, ''), (int) $idMatch[1]]; + } + + private function resolveCategoryId(?int $externalId, array $map): ?int + { + if ($externalId === null) { + return null; + } + + return $map[$externalId] ?? Category::query()->where('external_id', $externalId)->value('id'); } private function sanitizeText(string $value): string diff --git a/database/migrations/2026_04_29_001000_add_categories_and_article_metadata.php b/database/migrations/2026_04_29_001000_add_categories_and_article_metadata.php new file mode 100644 index 0000000..e938271 --- /dev/null +++ b/database/migrations/2026_04_29_001000_add_categories_and_article_metadata.php @@ -0,0 +1,42 @@ +id(); + $table->unsignedBigInteger('external_id')->nullable()->unique(); + $table->foreignId('parent_id')->nullable()->constrained('categories')->nullOnDelete(); + $table->string('name'); + $table->string('slug')->nullable(); + $table->timestamps(); + + $table->index(['parent_id', 'name']); + }); + + Schema::table('articles', function (Blueprint $table) { + $table->string('source')->nullable()->after('content'); + $table->string('source_url')->nullable()->after('source'); + $table->unsignedBigInteger('source_article_id')->nullable()->after('source_url'); + $table->foreignId('category_id')->nullable()->after('source_article_id')->constrained('categories')->nullOnDelete(); + $table->foreignId('subcategory_id')->nullable()->after('category_id')->constrained('categories')->nullOnDelete(); + + $table->index('source_article_id'); + }); + } + + public function down(): void + { + Schema::table('articles', function (Blueprint $table) { + $table->dropConstrainedForeignId('subcategory_id'); + $table->dropConstrainedForeignId('category_id'); + $table->dropColumn(['source', 'source_url', 'source_article_id']); + }); + + Schema::dropIfExists('categories'); + } +}; diff --git a/resources/views/livewire/admin/ticket-monitor.blade.php b/resources/views/livewire/admin/ticket-monitor.blade.php index 8b64589..d795112 100644 --- a/resources/views/livewire/admin/ticket-monitor.blade.php +++ b/resources/views/livewire/admin/ticket-monitor.blade.php @@ -1,5 +1,16 @@
-

Tickets + AI Decisions

+
+

Tickets + AI Decisions

+
+ + +
+
+
@foreach($tickets as $ticket)