Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | /**
* Crawler Service Worker
* Un seul conteneur pour le domaine crawler (Temporal) :
* - Worker 1 : queue 'crawler-workflows' → CrawlerTW (orchestration)
* - Worker 2 : queue 'crawler-activities' → activités de crawl (I/O : HTTP, Redis, LLM)
* Fusion des anciens crawler-workflow-worker + crawler-activity-worker (v2.43).
*/
const { Worker, NativeConnection } = require('@temporalio/worker');
const path = require('path');
const config = require('./config');
const activities = require('./activities');
async function runWorker() {
console.log('[Crawler-Service-Worker] Starting...');
console.log(`[Crawler-Service-Worker] Temporal: ${config.temporalAddress}`);
console.log(`[Crawler-Service-Worker] Workflow queue: ${config.temporalWorkflowTaskQueue}`);
console.log(`[Crawler-Service-Worker] Activity queue: ${config.temporalActivityTaskQueue}`);
// Connect to Temporal
const connection = await NativeConnection.connect({
address: config.temporalAddress,
});
// Worker 1 : orchestration (workflows)
const workflowWorker = await Worker.create({
connection,
namespace: config.temporalNamespace,
taskQueue: config.temporalWorkflowTaskQueue,
workflowsPath: path.join(__dirname, 'workflows', 'crawler-workflow.js'),
activities: {},
// Laisse les tâches en vol se terminer avant l'arrêt forcé (redéploiement).
shutdownGraceTime: '80s',
});
// Worker 2 : activités de crawl
const activityWorker = await Worker.create({
connection,
namespace: config.temporalNamespace,
taskQueue: config.temporalActivityTaskQueue,
activities,
maxConcurrentActivityExecutions: 5,
// Laisse l'activité en cours (crawl/évaluation) se terminer avant l'arrêt.
shutdownGraceTime: '80s',
});
console.log('[Crawler-Service-Worker] Workers created and listening');
console.log('[Crawler-Service-Worker] Workflows: CrawlerTW');
console.log('[Crawler-Service-Worker] Activities: initCrawlQueue, fetchContent, evaluatePage, triggerExtraction, waitForExtractionJob, triggerSynthesis, waitForSynthesisJob, ...');
// Drain : on arrête le polling des nouvelles tâches puis on laisse les
// activités en vol se terminer. `run()` se résout une fois le drain fait.
let shuttingDown = false;
const shutdown = (signal) => {
if (shuttingDown) return;
shuttingDown = true;
console.log(`[Crawler-Service-Worker] ${signal} received — draining in-flight activities (up to 80s)...`);
workflowWorker.shutdown();
activityWorker.shutdown();
};
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));
// Start both workers
await Promise.all([workflowWorker.run(), activityWorker.run()]);
console.log('[Crawler-Service-Worker] Workers stopped');
}
// Run worker
runWorker().catch((error) => {
console.error('[Crawler-Service-Worker] Fatal error:', error);
process.exit(1);
}); |