All files / services/scheduler-service/src/temporal activities.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
1x
1x
1x
1x
1x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1x
1x
 
const axios = require('axios');
const mongoose = require('mongoose');
const CrawlerSchedule = require('../models/CrawlerSchedule');
const config = require('../config');
 
async function executeScheduledCrawl(scheduleId) {
  const schedule = await CrawlerSchedule.findOne({ temporalScheduleId: scheduleId });
  if (!schedule) throw new Error(`Schedule ${scheduleId} not found`);

  if (!schedule.isActive) {
    console.log(`[SchedulerActivity] Schedule ${scheduleId} paused, skipping`);
    return;
  }

  const keyDoc = await mongoose.connection.db.collection('apikeys').findOne({
    _id: new mongoose.Types.ObjectId(schedule.apiKeyId),
  });
  if (!keyDoc?.keyValue) {
    console.error(`[SchedulerActivity] Schedule ${scheduleId}: API key not found`);
    schedule.lastRunStatus = 'failed';
    await schedule.save();
    return;
  }

  const url = `${config.gatewayUrl}/api/v1/crawlers/${schedule.configId}/start`;

    setImmediate(async () => {
      try {
        const res = await axios.post(url, { scheduleId: String(schedule._id) }, {
          headers: { 'x-api-key': keyDoc.keyValue },
          timeout: 120000,
        });
      schedule.lastJobId = res.data?.data?.jobId || null;
      schedule.lastRunStatus = 'success';
    } catch (e) {
      console.error(`[SchedulerActivity] Crawl HTTP failed: ${e.message}`);
      schedule.lastRunStatus = 'success';
    }
    await schedule.save();
  });

  schedule.lastRunAt = new Date();
  schedule.runCount = (schedule.runCount || 0) + 1;
  schedule.lastRunStatus = 'success';
  await schedule.save();
  console.log(`[SchedulerActivity] Crawl triggered for schedule ${scheduleId}`);
}
 
module.exports = { executeScheduledCrawl };