Add helpdesk import progress, category model, article metadata columns, and ticket pagination controls

app/Console/Commands/ImportHelpdeskArticlesCommand.php

@@ -19,12 +19,28 @@ class ImportHelpdeskArticlesCommand extends Command
         $limitOption = $this->option('limit');
         $limit = is_numeric($limitOption) ? (int) $limitOption : null;
 
+        $bar = null;
+
         $result = $service->import(
             (string) $this->option('base-url'),
             (bool) $this->option('dry-run'),
-            $limit
+            $limit,
+            function (int $processed, int $total, string $url, string $status) use (&$bar): void {
+                if ($bar === null) {
+                    $bar = $this->output->createProgressBar($total);
+                    $bar->start();
+                }
+
+                $bar->advance();
+                $bar->setMessage("{$status}: {$url}");
+            }
         );
 
+        if ($bar !== null) {
+            $bar->finish();
+            $this->newLine(2);
+        }
+
         $this->info('Helpdesk import finished.');
         $this->table(
             ['Metric', 'Value'],

app/Livewire/Admin/TicketMonitor.php

@@ -10,10 +10,17 @@ class TicketMonitor extends Component
 {
     use WithPagination;
 
+    public int $perPage = 10;
+
+    public function updatedPerPage(): void
+    {
+        $this->resetPage();
+    }
+
     public function render(AdminTicketService $service)
     {
         return view('livewire.admin.ticket-monitor', [
-            'tickets' => $service->paginateWithDecision(10),
+            'tickets' => $service->paginateWithDecision($this->perPage),
         ]);
     }
 }

app/Models/Article.php

@@ -4,11 +4,21 @@ namespace App\Models;
 
 use App\Casts\VectorCast;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\Relations\HasMany;
 
 class Article extends Model
 {
-    protected $fillable = ['title', 'content', 'embedding'];
+    protected $fillable = [
+        'title',
+        'content',
+        'embedding',
+        'source',
+        'source_url',
+        'source_article_id',
+        'category_id',
+        'subcategory_id',
+    ];
 
     protected $casts = [
         'embedding' => VectorCast::class,
@@ -18,4 +28,14 @@ class Article extends Model
     {
         return $this->hasMany(AIDecision::class);
     }
+
+    public function category(): BelongsTo
+    {
+        return $this->belongsTo(Category::class, 'category_id');
+    }
+
+    public function subcategory(): BelongsTo
+    {
+        return $this->belongsTo(Category::class, 'subcategory_id');
+    }
 }

app/Models/Category.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+
+class Category extends Model
+{
+    protected $fillable = ['external_id', 'parent_id', 'name', 'slug'];
+
+    public function parent(): BelongsTo
+    {
+        return $this->belongsTo(self::class, 'parent_id');
+    }
+
+    public function children(): HasMany
+    {
+        return $this->hasMany(self::class, 'parent_id');
+    }
+}

app/Services/HelpdeskImportService.php

@@ -3,6 +3,7 @@
 namespace App\Services;
 
 use App\Models\Article;
+use App\Models\Category;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Str;
 
@@ -10,54 +11,73 @@ class HelpdeskImportService
 {
     private const DEFAULT_BASE_URL = 'https://www.internettoday.nl/helpdesk';
 
-    public function import(?string $baseUrl = null, bool $dryRun = false, ?int $limit = null): array
+    public function import(?string $baseUrl = null, bool $dryRun = false, ?int $limit = null, ?callable $progress = null): array
     {
         $baseUrl = rtrim($baseUrl ?: self::DEFAULT_BASE_URL, '/');
 
         $rootHtml = $this->fetch($baseUrl);
         $categories = $this->extractCategories($rootHtml);
 
-        $sectionUrls = $this->buildSectionUrls($baseUrl, $categories);
+        $categoryMap = $this->syncCategories($categories, $dryRun);
-        $articleUrls = $this->collectArticleUrls($baseUrl, $rootHtml, $sectionUrls);
+        $sections = $this->buildSections($baseUrl, $categories);
+
+        $articleUrlMap = $this->collectArticleUrls($baseUrl, $rootHtml, $sections);
         if ($limit !== null && $limit > 0) {
-            $articleUrls = array_slice($articleUrls, 0, $limit);
+            $articleUrlMap = array_slice($articleUrlMap, 0, $limit, true);
         }
 
+        $total = count($articleUrlMap);
         $imported = 0;
         $updated = 0;
         $skipped = 0;
+        $processed = 0;
 
-        foreach ($articleUrls as $articleUrl) {
+        foreach ($articleUrlMap as $articleUrl => $meta) {
+            $processed++;
             $parsed = $this->parseArticlePage($articleUrl);
             if ($parsed === null) {
                 $skipped++;
+                $progress && $progress($processed, $total, $articleUrl, 'skipped');
                 continue;
             }
 
             if ($dryRun) {
                 $imported++;
+                $progress && $progress($processed, $total, $articleUrl, 'dry-run');
                 continue;
             }
 
-            [$title, $content] = $parsed;
+            [$title, $content, $sourceArticleId] = $parsed;
-            $result = Article::withoutEvents(function () use ($title, $content) {
+
+            $categoryId = $this->resolveCategoryId($meta['category_external_id'] ?? null, $categoryMap);
+            $subcategoryId = $this->resolveCategoryId($meta['subcategory_external_id'] ?? null, $categoryMap);
+
+            $result = Article::withoutEvents(function () use ($title, $content, $articleUrl, $sourceArticleId, $categoryId, $subcategoryId) {
                 return Article::query()->updateOrCreate(
-                    ['title' => $title],
+                    ['source' => 'internettoday_helpdesk', 'source_article_id' => $sourceArticleId],
-                    ['content' => $content]
+                    [
+                        'title' => $title,
+                        'content' => $content,
+                        'source_url' => $articleUrl,
+                        'category_id' => $categoryId,
+                        'subcategory_id' => $subcategoryId,
+                    ]
                 );
             });
 
             if ($result->wasRecentlyCreated) {
                 $imported++;
+                $progress && $progress($processed, $total, $articleUrl, 'imported');
             } else {
                 $updated++;
+                $progress && $progress($processed, $total, $articleUrl, 'updated');
             }
         }
 
         return [
             'categories' => count($categories),
-            'sections' => count($sectionUrls),
+            'sections' => count($sections),
-            'article_urls' => count($articleUrls),
+            'article_urls' => $total,
             'imported' => $imported,
             'updated' => $updated,
             'skipped' => $skipped,
@@ -67,11 +87,7 @@ class HelpdeskImportService
 
     private function fetch(string $url): string
     {
-        return Http::timeout(30)
+        return Http::timeout(30)->retry(2, 300)->get($url)->throw()->body();
-            ->retry(2, 300)
-            ->get($url)
-            ->throw()
-            ->body();
     }
 
     private function extractCategories(string $html): array
@@ -84,46 +100,102 @@ class HelpdeskImportService
         return is_array($decoded) ? $decoded : [];
     }
 
-    private function buildSectionUrls(string $baseUrl, array $categories): array
+    private function syncCategories(array $categories, bool $dryRun): array
     {
-        $urls = [];
+        $map = [];
+        foreach ($categories as $category) {
+            if (!isset($category['id'], $category['title'], $category['slug'])) {
+                continue;
+            }
+
+            $parentId = null;
+            if (!$dryRun) {
+                $model = Category::query()->updateOrCreate(
+                    ['external_id' => (int) $category['id']],
+                    ['name' => (string) $category['title'], 'slug' => (string) $category['slug'], 'parent_id' => null]
+                );
+                $parentId = $model->id;
+            }
+
+            $map[(int) $category['id']] = $parentId;
+
+            foreach (($category['children'] ?? []) as $child) {
+                if (!isset($child['id'], $child['title'], $child['slug'])) {
+                    continue;
+                }
+
+                if (!$dryRun && $parentId !== null) {
+                    $childModel = Category::query()->updateOrCreate(
+                        ['external_id' => (int) $child['id']],
+                        ['name' => (string) $child['title'], 'slug' => (string) $child['slug'], 'parent_id' => $parentId]
+                    );
+                    $map[(int) $child['id']] = $childModel->id;
+                } else {
+                    $map[(int) $child['id']] = null;
+                }
+            }
+        }
+
+        return $map;
+    }
+
+    private function buildSections(string $baseUrl, array $categories): array
+    {
+        $sections = [];
         foreach ($categories as $category) {
             if (!isset($category['id'], $category['slug'])) {
                 continue;
             }
 
-            $urls[] = sprintf('%s/%d/%s', $baseUrl, (int) $category['id'], (string) $category['slug']);
+            $sections[] = [
+                'url' => sprintf('%s/%d/%s', $baseUrl, (int) $category['id'], (string) $category['slug']),
+                'category_external_id' => (int) $category['id'],
+                'subcategory_external_id' => null,
+            ];
 
             foreach (($category['children'] ?? []) as $child) {
                 if (!isset($child['id'], $child['slug'])) {
                     continue;
                 }
 
-                $urls[] = sprintf('%s/%d/%s', $baseUrl, (int) $child['id'], (string) $child['slug']);
+                $sections[] = [
+                    'url' => sprintf('%s/%d/%s', $baseUrl, (int) $child['id'], (string) $child['slug']),
+                    'category_external_id' => (int) $category['id'],
+                    'subcategory_external_id' => (int) $child['id'],
+                ];
             }
         }
 
-        return array_values(array_unique($urls));
+        return $sections;
     }
 
-    private function collectArticleUrls(string $baseUrl, string $rootHtml, array $sectionUrls): array
+    private function collectArticleUrls(string $baseUrl, string $rootHtml, array $sections): array
     {
-        $urls = [];
+        $result = [];
+        $sources = array_merge([
+            ['url' => $baseUrl, 'category_external_id' => null, 'subcategory_external_id' => null, 'html' => $rootHtml],
+        ], $sections);
 
-        foreach (array_merge([$baseUrl], $sectionUrls) as $url) {
+        foreach ($sources as $source) {
             try {
-                $html = $url === $baseUrl ? $rootHtml : $this->fetch($url);
+                $html = $source['html'] ?? $this->fetch($source['url']);
             } catch (\Throwable) {
                 continue;
             }
 
             preg_match_all('/https:\/\/www\.internettoday\.nl\/helpdesk\/(\d+)-[a-z0-9\-]+/i', $html, $matches);
             foreach (($matches[0] ?? []) as $match) {
-                $urls[] = strtolower($match);
+                $url = strtolower($match);
+                if (!isset($result[$url])) {
+                    $result[$url] = [
+                        'category_external_id' => $source['category_external_id'],
+                        'subcategory_external_id' => $source['subcategory_external_id'],
+                    ];
+                }
             }
         }
 
-        return array_values(array_unique($urls));
+        return $result;
     }
 
     private function parseArticlePage(string $url): ?array
@@ -156,9 +228,20 @@ class HelpdeskImportService
             return null;
         }
 
-        $content = "Source: {$url}\n\n{$content}";
+        if (!preg_match('/\/helpdesk\/(\d+)-/', $url, $idMatch)) {
+            return null;
+        }
 
-        return [$title, Str::limit($content, 64000, '')];
+        return [$title, Str::limit($content, 64000, ''), (int) $idMatch[1]];
+    }
+
+    private function resolveCategoryId(?int $externalId, array $map): ?int
+    {
+        if ($externalId === null) {
+            return null;
+        }
+
+        return $map[$externalId] ?? Category::query()->where('external_id', $externalId)->value('id');
     }
 
     private function sanitizeText(string $value): string

database/migrations/2026_04_29_001000_add_categories_and_article_metadata.php

@@ -0,0 +1,42 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+    public function up(): void
+    {
+        Schema::create('categories', function (Blueprint $table) {
+            $table->id();
+            $table->unsignedBigInteger('external_id')->nullable()->unique();
+            $table->foreignId('parent_id')->nullable()->constrained('categories')->nullOnDelete();
+            $table->string('name');
+            $table->string('slug')->nullable();
+            $table->timestamps();
+
+            $table->index(['parent_id', 'name']);
+        });
+
+        Schema::table('articles', function (Blueprint $table) {
+            $table->string('source')->nullable()->after('content');
+            $table->string('source_url')->nullable()->after('source');
+            $table->unsignedBigInteger('source_article_id')->nullable()->after('source_url');
+            $table->foreignId('category_id')->nullable()->after('source_article_id')->constrained('categories')->nullOnDelete();
+            $table->foreignId('subcategory_id')->nullable()->after('category_id')->constrained('categories')->nullOnDelete();
+
+            $table->index('source_article_id');
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::table('articles', function (Blueprint $table) {
+            $table->dropConstrainedForeignId('subcategory_id');
+            $table->dropConstrainedForeignId('category_id');
+            $table->dropColumn(['source', 'source_url', 'source_article_id']);
+        });
+
+        Schema::dropIfExists('categories');
+    }
+};

resources/views/livewire/admin/ticket-monitor.blade.php

@@ -1,5 +1,16 @@
 <div class="bg-white rounded-xl p-4 shadow">
-    <h2 class="font-semibold mb-3">Tickets + AI Decisions</h2>
+    <div class="flex items-center justify-between mb-3">
+        <h2 class="font-semibold">Tickets + AI Decisions</h2>
+        <div class="flex items-center gap-2 text-sm">
+            <label for="perPage">Per pagina</label>
+            <select id="perPage" wire:model.live="perPage" class="border rounded px-2 py-1">
+                <option value="10">10</option>
+                <option value="25">25</option>
+                <option value="50">50</option>
+            </select>
+        </div>
+    </div>
+
     <div class="space-y-3">
         @foreach($tickets as $ticket)
             <div class="border rounded p-3">