Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x | /**
* Crawler API - Shared Helpers
* Common functions used by both org-scoped and single-resource routes
*/
const axios = require('axios');
const Redis = require('ioredis');
const config = require('../config');
const { addInternalKey } = require('../../shared/internalAuth');
let redisClient = null;
const JOB_POOL_PREFIX = 'pool:job:';
const JOB_POOL_SUFFIX = ':params';
const JOB_TTL = 86400;
function getRedisClient() {
if (!redisClient) {
const redisHost = process.env.REDIS_HOST || 'redis';
const redisPort = parseInt(process.env.REDIS_PORT || '6379');
const redisPassword = process.env.REDIS_PASSWORD || undefined;
redisClient = new Redis({
host: redisHost,
port: redisPort,
password: redisPassword,
retryStrategy: (times) => Math.min(times * 50, 2000),
});
redisClient.on('error', (err) => {
console.error('[CrawlerAPI/Redis] Error:', err.message);
});
}
return redisClient;
}
async function storeJobById(jobId, jobData) {
const redis = getRedisClient();
const key = `${JOB_POOL_PREFIX}${jobId}${JOB_POOL_SUFFIX}`;
await redis.setex(key, JOB_TTL, JSON.stringify(jobData));
return jobId;
}
const JOB_MANAGER_SERVICE_URL = process.env.JOB_MANAGER_SERVICE_URL || 'http://job-manager-service:8000';
const SERVICE_NAME = process.env.SERVICE_NAME || 'crawler-service-api';
async function launchCrawlingWorkflow(jobId, startUrls, crawlerConfig, userId, organizationId, scheduleId = null) {
const cfg = crawlerConfig.config || crawlerConfig;
// D2 : aucune clé secrète dans l'input workflow (LLM/Marker sont résolues à la
// demande par les activités via le config-service interne). Plus de session
// Redis `config:` ni de `authToken` (cf. spec/prd-remove-xconfig-id.md).
// `prompt` est utilisé pour l'évaluation de pertinence (evaluatePage).
// Le raffinage (mode + spec + modèle LLM) est porté par les destinations de la config.
const workflowConfig = {
jobId,
startUrls,
prompt: crawlerConfig.prompt || '',
config: {
maxPages: cfg.maxPages ?? 10,
maxDepth: cfg.maxDepth ?? 3,
pageThreshold: cfg.pageThreshold ?? 60,
linkThreshold: cfg.linkThreshold ?? 40,
crawlDelay: cfg.crawlDelay ?? 100,
sameDomain: cfg.sameDomain === true,
enableSynthesis: cfg.enableSynthesis !== false,
llmModel: cfg.llmModel || 'deepseek/deepseek-v4-flash',
reasoning: cfg.reasoning !== false,
maxTokens: cfg.maxTokens ?? null,
// URLs de services propagées au workflow (le workflow ne lit pas process.env) :
// sinon les activités retombent sur des hôtes codés en dur.
workflowApiUrl: config.workflowApiUrl,
jobManagerUrl: config.jobManagerUrl,
minioUrl: config.minioUrl,
},
destinations: cfg.destinations || [],
userId,
organizationId,
scheduleId: scheduleId || null,
// _id de la config crawler (pour lister les jobs par crawler)
crawlerConfigId: crawlerConfig._id ? String(crawlerConfig._id) : null,
};
await storeJobById(jobId, workflowConfig);
const response = await axios.post(
`${JOB_MANAGER_SERVICE_URL}/internal/pool/add`,
{
jobId,
organizationId,
workflowType: 'CrawlerTW',
input: workflowConfig,
},
{
timeout: 5000,
headers: addInternalKey({ 'Content-Type': 'application/json' }, SERVICE_NAME),
}
);
console.log(`[CrawlerAPI] Job ${jobId} queued via job-manager-service, position: ${response.data.position}, model: ${workflowConfig.config.llmModel}`);
return jobId;
}
module.exports = {
getRedisClient,
storeJobById,
launchCrawlingWorkflow,
};
|