diff --git a/config/cdConfig.js b/config/cdConfig.js index 37346cab..1b21f596 100644 --- a/config/cdConfig.js +++ b/config/cdConfig.js @@ -129,6 +129,7 @@ module.exports = { weights: { immediate: 3, soon: 2, normal: 3, later: 2 }, connectionString: config.get('CRAWLER_QUEUE_AZURE_CONNECTION_STRING') || cd_azblob.connection, queueName: config.get('CRAWLER_QUEUE_PREFIX') || 'cdcrawlerdev', + messageTimeToLive: config.get('CRAWLER_QUEUE_MESSAGE_EXPIRATION') || 60 * 60 * 24 * 7, // Default to 7 days visibilityTimeout: 8 * 60 * 60, // 8 hours visibilityTimeout_remainLocal: fetchedCacheTtlSeconds, maxDequeueCount: 5, diff --git a/ghcrawler/providers/queuing/storageQueue.js b/ghcrawler/providers/queuing/storageQueue.js index c52e1257..4d876c66 100644 --- a/ghcrawler/providers/queuing/storageQueue.js +++ b/ghcrawler/providers/queuing/storageQueue.js @@ -37,6 +37,7 @@ class StorageQueue { qlimit(this.options.parallelPush || 1)(request => { const body = JSON.stringify(request) return new Promise((resolve, reject) => { + this.logger.info(`Sending createMessage to ${this.queueName} with options ${JSON.stringify(option)}`) this.client.createMessage(this.queueName, body, option, (error, queueMessageResult) => { if (error) { return reject(error) diff --git a/test/unit/providers/queuing/storageQueueManagerTests.js b/test/unit/providers/queuing/storageQueueManagerTests.js new file mode 100644 index 00000000..bc9af247 --- /dev/null +++ b/test/unit/providers/queuing/storageQueueManagerTests.js @@ -0,0 +1,25 @@ +const sinon = require('sinon') +const { expect } = require('chai') +const StorageQueueManager = require('../../../../ghcrawler/providers/queuing/storageQueueManager') +const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue') + +describe('StorageQueueManager', () => { + let manager, clientStub + + beforeEach(() => { + clientStub = { + createMessage: sinon.stub().yields(null, {}) + } + manager = new StorageQueueManager('connectionString', { messageTimeToLive: 3600 }) // 1 hour expiration + manager.client = clientStub + }) + + it('should create a queue with the correct expiration', async () => { + const queue = manager.createQueue('testQueue') + await queue.push({ body: 'test message' }) + + expect(clientStub.createMessage.calledOnce).to.be.true + const args = clientStub.createMessage.getCall(0).args + expect(args[2].messageTimeToLive).to.equal(3600) + }) +}) \ No newline at end of file diff --git a/test/unit/providers/queuing/storageQueueTests.js b/test/unit/providers/queuing/storageQueueTests.js new file mode 100644 index 00000000..ec21f253 --- /dev/null +++ b/test/unit/providers/queuing/storageQueueTests.js @@ -0,0 +1,22 @@ +const sinon = require('sinon') +const { expect } = require('chai') +const StorageQueue = require('../../../../ghcrawler/providers/queuing/storageQueue') + +describe('StorageQueue', () => { + let queue, clientStub + + beforeEach(() => { + clientStub = { + createMessage: sinon.stub().yields(null, {}) + } + queue = new StorageQueue(clientStub, 'testQueue', 'testQueueName', message => message, { messageTimeToLive: 3600 }) // 1 hour expiration + }) + + it('should add messages with the correct expiration', async () => { + await queue.push({ body: 'test message' }) + + expect(clientStub.createMessage.calledOnce).to.be.true + const args = clientStub.createMessage.getCall(0).args + expect(args[2].messageTimeToLive).to.equal(3600) + }) +}) \ No newline at end of file