Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | import { useState } from 'react'
import { useForm } from 'react-hook-form'
import { Globe, Plus, Loader2, CheckCircle, XCircle } from 'lucide-react'
import { createCrawlerJob } from '../../services/api'
import toast from 'react-hot-toast'
function Crawler() {
const [jobs, setJobs] = useState([])
const [isSubmitting, setIsSubmitting] = useState(false)
const { register, handleSubmit, reset, formState: { errors } } = useForm()
const onSubmit = async (data) => {
setIsSubmitting(true)
try {
const urls = data.urls.split('\n').filter(url => url.trim())
const response = await createCrawlerJob({
start_urls: urls,
objective: data.objective,
max_depth: parseInt(data.maxDepth) || 3,
max_pages: parseInt(data.maxPages) || 10,
})
setJobs(prev => [response.data.data, ...prev])
toast.success('Job de crawling créé avec succès')
reset()
} catch (error) {
toast.error('Erreur lors de la création du job')
} finally {
setIsSubmitting(false)
}
}
return (
<div className="max-w-4xl mx-auto">
<div className="mb-8">
<h1 className="text-3xl font-bold text-gray-900">Crawler</h1>
<p className="text-gray-600 mt-1">
Découvrez automatiquement du contenu web avec scoring LLM
</p>
</div>
{/* Create job form */}
<div className="card mb-8">
<h2 className="text-xl font-semibold mb-6">Nouveau job de crawling</h2>
<form onSubmit={handleSubmit(onSubmit)} className="space-y-6">
<div>
<label className="block text-sm font-medium text-gray-700 mb-2">
URLs de départ (une par ligne)
</label>
<textarea
{...register('urls', { required: 'Au moins une URL est requise' })}
rows={3}
className="input"
placeholder="https://example.com https://example.com/page"
/>
{errors.urls && (
<p className="mt-1 text-sm text-red-600">{errors.urls.message}</p>
)}
</div>
<div>
<label className="block text-sm font-medium text-gray-700 mb-2">
Objectif du crawling
</label>
<input
{...register('objective', { required: 'Un objectif est requis' })}
type="text"
className="input"
placeholder="Ex: Trouver des articles sur l'intelligence artificielle"
/>
{errors.objective && (
<p className="mt-1 text-sm text-red-600">{errors.objective.message}</p>
)}
</div>
<div className="grid grid-cols-2 gap-4">
<div>
<label className="block text-sm font-medium text-gray-700 mb-2">
Profondeur max
</label>
<input
{...register('maxDepth')}
type="number"
defaultValue={3}
min={1}
max={10}
className="input"
/>
</div>
<div>
<label className="block text-sm font-medium text-gray-700 mb-2">
Pages max
</label>
<input
{...register('maxPages')}
type="number"
defaultValue={100}
min={1}
max={10000}
className="input"
/>
</div>
</div>
<button
type="submit"
disabled={isSubmitting}
className="btn-primary inline-flex items-center"
>
{isSubmitting ? (
<>
<Loader2 className="w-5 h-5 mr-2 animate-spin" />
Création...
</>
) : (
<>
<Plus className="w-5 h-5 mr-2" />
Lancer le crawling
</>
)}
</button>
</form>
</div>
{/* Jobs list */}
{jobs.length > 0 && (
<div>
<h2 className="text-xl font-semibold mb-4">Jobs de crawling</h2>
<div className="space-y-4">
{jobs.map((job) => (
<div key={job.job_id} className="card">
<div className="flex items-center justify-between">
<div className="flex items-center">
{job.status === 'completed' ? (
<CheckCircle className="w-5 h-5 text-green-500 mr-3" />
) : job.status === 'failed' ? (
<XCircle className="w-5 h-5 text-red-500 mr-3" />
) : (
<Loader2 className="w-5 h-5 text-blue-500 animate-spin mr-3" />
)}
<div>
<p className="font-medium text-gray-900">Job #{job.job_id}</p>
<p className="text-sm text-gray-500">{job.config?.objective}</p>
</div>
</div>
<span className={`px-3 py-1 rounded-full text-sm font-medium ${
job.status === 'completed'
? 'bg-green-100 text-green-800'
: job.status === 'failed'
? 'bg-red-100 text-red-800'
: 'bg-blue-100 text-blue-800'
}`}>
{job.status}
</span>
</div>
</div>
))}
</div>
</div>
)}
{/* Info */}
<div className="mt-8 card">
<h3 className="font-semibold text-gray-900 mb-4">Comment ça marche</h3>
<ol className="space-y-3 text-gray-600">
<li className="flex items-start">
<span className="flex-shrink-0 w-6 h-6 bg-primary-100 text-primary-600 rounded-full flex items-center justify-center text-sm font-medium mr-3">
1
</span>
<span>Définissez les URLs de départ et l'objectif du crawling</span>
</li>
<li className="flex items-start">
<span className="flex-shrink-0 w-6 h-6 bg-primary-100 text-primary-600 rounded-full flex items-center justify-center text-sm font-medium mr-3">
2
</span>
<span>Le crawler explore automatiquement les liens trouvés</span>
</li>
<li className="flex items-start">
<span className="flex-shrink-0 w-6 h-6 bg-primary-100 text-primary-600 rounded-full flex items-center justify-center text-sm font-medium mr-3">
3
</span>
<span>Chaque page est évaluée par LLM pour pertinence</span>
</li>
<li className="flex items-start">
<span className="flex-shrink-0 w-6 h-6 bg-primary-100 text-primary-600 rounded-full flex items-center justify-center text-sm font-medium mr-3">
4
</span>
<span>Les pages pertinentes sont extraites et stockées</span>
</li>
</ol>
</div>
</div>
)
}
export default Crawler
|