Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | /**
* Web Extractor Worker v4.0
* Worker Temporal dédié au scraping Playwright
* NO HTTP SERVER - Juste worker
*
* Worker d'activités (scrapeWebPage, resolveUrlWithBrowser) consommées par
* WorkflowTW (workflow-service-worker). Aucun workflow enregistré ici.
*/
const dotenv = require('dotenv');
const mongoose = require('mongoose');
const { Worker: TemporalWorker, NativeConnection } = require('@temporalio/worker');
const { S3Client } = require('@aws-sdk/client-s3');
dotenv.config();
// Configuration
const TEMPORAL_ADDRESS = process.env.TEMPORAL_ADDRESS || 'temporal:7233';
const TEMPORAL_TASK_QUEUE = 'web-extraction-activities';
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://mongodb:27017/nooloom';
const MINIO_ENDPOINT = process.env.MINIO_ENDPOINT || 'http://minio:9000';
// Global clients
let temporalWorker = null;
let temporalWorkerRun = null;
let s3Client = null;
/**
* Initialize MongoDB connection
*/
async function initMongoDB() {
try {
await mongoose.connect(MONGODB_URI, {
authSource: 'admin',
});
console.log('✅ MongoDB connected');
return true;
} catch (error) {
console.error('❌ MongoDB connection error:', error.message);
return false;
}
}
/**
* Initialize MinIO S3 client
*/
function initMinIO() {
try {
s3Client = new S3Client({
endpoint: MINIO_ENDPOINT,
region: 'us-east-1',
credentials: {
accessKeyId: process.env.MINIO_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.MINIO_SECRET_KEY || 'minioadmin',
},
forcePathStyle: true,
});
console.log('✅ MinIO S3 client initialized');
return true;
} catch (error) {
console.error('❌ MinIO initialization failed:', error.message);
return false;
}
}
/**
* Initialize and start Temporal Worker v4.0
*/
async function initTemporalWorker() {
try {
const connection = await NativeConnection.connect({
address: TEMPORAL_ADDRESS,
});
// Load v4 activities (granular 12-step workflow)
const activities = require('./temporal/activities-v4');
// Worker d'ACTIVITÉS uniquement : le scraping est orchestré par
// WorkflowTW (workflow-service-worker). Les workflows legacy
// (WebExtractionWorkflow/WebScrapingWorkflow) ont été supprimés.
temporalWorker = await TemporalWorker.create({
activities,
taskQueue: TEMPORAL_TASK_QUEUE,
connection,
maxConcurrentActivityTaskExecutions: 3,
// Laisse le scraping en vol se terminer avant l'arrêt forcé.
shutdownGraceTime: '80s',
});
// Start worker in background. Le run se résout une fois le drain terminé :
// on conserve la promesse pour pouvoir l'attendre au shutdown.
temporalWorkerRun = temporalWorker.run().catch(err => {
console.error('Temporal worker error:', err);
});
console.log(`✅ Temporal Worker v4.0 started (queue: ${TEMPORAL_TASK_QUEUE})`);
console.log(' Mode: activités uniquement (orchestration via WorkflowTW)');
console.log('🎬 Ready to scrape web pages!');
return true;
} catch (error) {
console.error('❌ Temporal worker failed:', error.message);
return false;
}
}
/**
* Main startup
*/
async function startWorker() {
console.log('🔧 Web Extractor Worker v4.0 starting...\n');
// Initialize services
const mongoOk = await initMongoDB();
const minioOk = initMinIO();
if (!mongoOk || !minioOk) {
console.error('❌ Failed to initialize required services');
process.exit(1);
}
// Start Temporal Worker
const workerOk = await initTemporalWorker();
if (!workerOk) {
console.error('❌ Failed to start Temporal worker');
process.exit(1);
}
console.log('\n🚀 Worker v4.0 is running');
console.log(` Temporal: ${TEMPORAL_ADDRESS} (queue: ${TEMPORAL_TASK_QUEUE})`);
console.log(` MinIO: ${MINIO_ENDPOINT}`);
console.log(` MongoDB: ${MONGODB_URI.split('@')[1] || 'connected'}`);
console.log('\n💡 NO HTTP SERVER - This is a pure worker\n');
}
// Graceful shutdown : arrêt du polling, attente du drain, puis fermeture.
let webExtractorShuttingDown = false;
async function gracefulShutdown(signal) {
if (webExtractorShuttingDown) return;
webExtractorShuttingDown = true;
console.log(`${signal} received, draining in-flight activities (up to 80s)...`);
if (temporalWorker) temporalWorker.shutdown();
if (temporalWorkerRun) await temporalWorkerRun;
await mongoose.connection.close();
process.exit(0);
}
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
// Start
startWorker().catch(error => {
console.error('Failed to start worker:', error);
process.exit(1);
});
|