diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..43c97e7 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/.travis.yml b/.travis.yml index 4550b97..345cdba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,12 @@ addons: packages: - g++-4.8 node_js: +# We do support Node.js v4 but tests are not run against it anymore since the testing frameworks do not support it anymore. +# - "4" - "6" - - "7" - "8" - - "9" - "10" + - "11" install: - npm install - npm install hiredis diff --git a/README.md b/README.md index 0c7644c..3b897db 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,11 @@ # redis-parser -A high performance javascript redis parser built for [node_redis](https://github.com/NodeRedis/node_redis) and [ioredis](https://github.com/luin/ioredis). Parses all [RESP](http://redis.io/topics/protocol) data. +A high performance javascript RESP v2/v3 parser built for +[node_redis](https://github.com/NodeRedis/node_redis) and +[ioredis](https://github.com/luin/ioredis). Parses all +[RESP v2](http://redis.io/topics/protocol) and +[RESP v3](https://github.com/antirez/RESP3) data. ## Install @@ -24,15 +28,21 @@ const myParser = new Parser(options); * `returnReply`: *function*; mandatory * `returnError`: *function*; mandatory +* `returnPush`: *function*; optional (mandatory for RESP3) * `returnFatalError`: *function*; optional, defaults to the returnError function * `returnBuffers`: *boolean*; optional, defaults to false * `stringNumbers`: *boolean*; optional, defaults to false +* `bigInt`: *boolean*; optional, defaults to false ### Functions * `reset()`: reset the parser to it's initial state -* `setReturnBuffers(boolean)`: set the returnBuffers option on/off without resetting the parser -* `setStringNumbers(boolean)`: set the stringNumbers option on/off without resetting the parser +* `setReturnBuffers(boolean)`: set the returnBuffers option on/off without + resetting the parser +* `setStringNumbers(boolean)`: set the stringNumbers option on/off without + resetting the parser +* `setBigInt(boolean)`: set the bigInt option on/off without + resetting the parser ### Error classes @@ -40,7 +50,8 @@ const myParser = new Parser(options); * `ReplyError` sub class of RedisError * `ParserError` sub class of RedisError -All Redis errors will be returned as `ReplyErrors` while a parser error is returned as `ParserError`. +All Redis errors will be returned as `ReplyErrors` while a parser error is +returned as `ParserError`. All error classes can be imported by the npm `redis-errors` package. ### Example @@ -78,11 +89,18 @@ const parser = new Parser({ }); ``` -You do not have to use the returnFatalError function. Fatal errors will be returned in the normal error function in that case. +You do not have to use the returnFatalError function. Fatal errors will be +returned in the normal error function in that case. -And if you want to return buffers instead of strings, you can do this by adding the `returnBuffers` option. +And if you want to return buffers instead of strings, you can do this by adding +the `returnBuffers` option. -If you handle with big numbers that are to large for JS (Number.MAX_SAFE_INTEGER === 2^53 - 16) please use the `stringNumbers` option. That way all numbers are going to be returned as String and you can handle them safely. +If you handle with big numbers that are to large for JS (Number.MAX_SAFE_INTEGER +=== 2^53 - 16) please use the `bigInt` or the `stringNumbers` option. That way +all numbers are going to be returned as bigint or string and you can handle them +safely. `bigInt` is going to be slower than returning strings due to the extra +conversion and it only works for integers. `RESP3` also uses 64bit doubles that can +not be represented by `bigint`. ```js // Same functions as in the first example @@ -95,7 +113,7 @@ const parser = new Parser({ lib.returnError(err); }, returnBuffers: true, // All strings are returned as Buffer e.g. - stringNumbers: true // All numbers are returned as String + bigInt: true // All integers are returned as BigInt }); // The streamHandler as above @@ -103,11 +121,20 @@ const parser = new Parser({ ## Protocol errors -To handle protocol errors (this is very unlikely to happen) gracefully you should add the returnFatalError option, reject any still running command (they might have been processed properly but the reply is just wrong), destroy the socket and reconnect. Note that while doing this no new command may be added, so all new commands have to be buffered in the meantime, otherwise a chunk might still contain partial data of a following command that was already processed properly but answered in the same chunk as the command that resulted in the protocol error. +To handle protocol errors (this is very unlikely to happen) gracefully you +should add the returnFatalError option, reject any still running command (they +might have been processed properly but the reply is just wrong), destroy the +socket and reconnect. Note that while doing this no new command may be added, so +all new commands have to be buffered in the meantime, otherwise a chunk might +still contain partial data of a following command that was already processed +properly but answered in the same chunk as the command that resulted in the +protocol error. ## Contribute -The parser is highly optimized but there may still be further optimizations possible. +The parser is highly optimized but there may still be further optimizations +possible. Especially the new aggregated data types from RESP3 should have some +optimization potential left. npm install npm test diff --git a/benchmark/index.js b/benchmark/index.js index a570038..4897a48 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -6,7 +6,6 @@ const Benchmark = require('benchmark') const suite = new Benchmark.Suite() const Parser = require('./../') const Buffer = require('buffer').Buffer -const HiredisParser = require('../test/hiredis') function returnError (error) {} function checkReply (error, res) {} @@ -64,25 +63,20 @@ const options = { returnFatalError: returnError } const parser = new Parser(options) -const parserHiRedis = new HiredisParser(options) options.returnBuffers = true const parserBuffer = new Parser(options) -const parserHiRedisBuffer = new HiredisParser(options) -delete options.returnBuffers options.stringNumbers = true const parserStr = new Parser(options) -// BULK STRINGS +delete options.stringNumbers +options.bigInt = true +const parserBigInt = new Parser(options) -suite.add('HIREDIS: $ multiple chunks in a bulk string', function () { - parserHiRedis.execute(startBuffer) - parserHiRedis.execute(chunkBuffer) - parserHiRedis.execute(chunkBuffer) - parserHiRedis.execute(chunkBuffer) - parserHiRedis.execute(endBuffer) -}) +const runBigInt = process.argv.length === 2 || process.argv.includes('bigint') + +// BULK STRINGS suite.add('JS PARSER: $ multiple chunks in a bulk string', function () { parser.execute(startBuffer) @@ -92,14 +86,6 @@ suite.add('JS PARSER: $ multiple chunks in a bulk string', function () { parser.execute(endBuffer) }) -suite.add('HIREDIS BUF: $ multiple chunks in a bulk string', function () { - parserHiRedisBuffer.execute(startBuffer) - parserHiRedisBuffer.execute(chunkBuffer) - parserHiRedisBuffer.execute(chunkBuffer) - parserHiRedisBuffer.execute(chunkBuffer) - parserHiRedisBuffer.execute(endBuffer) -}) - suite.add('JS PARSER BUF: $ multiple chunks in a bulk string', function () { parserBuffer.execute(startBuffer) parserBuffer.execute(chunkBuffer) @@ -110,21 +96,11 @@ suite.add('JS PARSER BUF: $ multiple chunks in a bulk string', function () { // CHUNKED STRINGS -suite.add('\nHIREDIS: + multiple chunks in a string', function () { - parserHiRedis.execute(chunkedStringPart1) - parserHiRedis.execute(chunkedStringPart2) -}) - suite.add('JS PARSER: + multiple chunks in a string', function () { parser.execute(chunkedStringPart1) parser.execute(chunkedStringPart2) }) -suite.add('HIREDIS BUF: + multiple chunks in a string', function () { - parserHiRedisBuffer.execute(chunkedStringPart1) - parserHiRedisBuffer.execute(chunkedStringPart2) -}) - suite.add('JS PARSER BUF: + multiple chunks in a string', function () { parserBuffer.execute(chunkedStringPart1) parserBuffer.execute(chunkedStringPart2) @@ -132,14 +108,6 @@ suite.add('JS PARSER BUF: + multiple chunks in a string', function () { // BIG BULK STRING -suite.add('\nHIREDIS: $ 4mb bulk string', function () { - parserHiRedis.execute(startBigBuffer) - for (var i = 0; i < 64; i++) { - parserHiRedis.execute(chunks[i]) - } - parserHiRedis.execute(endBuffer) -}) - suite.add('JS PARSER: $ 4mb bulk string', function () { parser.execute(startBigBuffer) for (var i = 0; i < 64; i++) { @@ -148,14 +116,6 @@ suite.add('JS PARSER: $ 4mb bulk string', function () { parser.execute(endBuffer) }) -suite.add('HIREDIS BUF: $ 4mb bulk string', function () { - parserHiRedisBuffer.execute(startBigBuffer) - for (var i = 0; i < 64; i++) { - parserHiRedisBuffer.execute(chunks[i]) - } - parserHiRedisBuffer.execute(endBuffer) -}) - suite.add('JS PARSER BUF: $ 4mb bulk string', function () { parserBuffer.execute(startBigBuffer) for (var i = 0; i < 64; i++) { @@ -166,28 +126,16 @@ suite.add('JS PARSER BUF: $ 4mb bulk string', function () { // STRINGS -suite.add('\nHIREDIS: + simple string', function () { - parserHiRedis.execute(stringBuffer) -}) - suite.add('JS PARSER: + simple string', function () { parser.execute(stringBuffer) }) -suite.add('HIREDIS BUF: + simple string', function () { - parserHiRedisBuffer.execute(stringBuffer) -}) - suite.add('JS PARSER BUF: + simple string', function () { parserBuffer.execute(stringBuffer) }) // INTEGERS -suite.add('\nHIREDIS: : integer', function () { - parserHiRedis.execute(integerBuffer) -}) - suite.add('JS PARSER: : integer', function () { parser.execute(integerBuffer) }) @@ -196,11 +144,13 @@ suite.add('JS PARSER STR: : integer', function () { parserStr.execute(integerBuffer) }) -// BIG INTEGER +if (runBigInt) { + suite.add('JS PARSER BIGINT: : integer', function () { + parserBigInt.execute(integerBuffer) + }) +} -suite.add('\nHIREDIS: : big integer', function () { - parserHiRedis.execute(bigIntegerBuffer) -}) +// BIG INTEGER suite.add('JS PARSER: : big integer', function () { parser.execute(bigIntegerBuffer) @@ -210,44 +160,30 @@ suite.add('JS PARSER STR: : big integer', function () { parserStr.execute(bigIntegerBuffer) }) -// ARRAYS +if (runBigInt) { + suite.add('JS PARSER BIGINT: : big integer', function () { + parserBigInt.execute(bigIntegerBuffer) + }) +} -suite.add('\nHIREDIS: * array', function () { - parserHiRedis.execute(arrayBuffer) -}) +// ARRAYS suite.add('JS PARSER: * array', function () { parser.execute(arrayBuffer) }) -suite.add('HIREDIS BUF: * array', function () { - parserHiRedisBuffer.execute(arrayBuffer) -}) - suite.add('JS PARSER BUF: * array', function () { parserBuffer.execute(arrayBuffer) }) // BIG NESTED ARRAYS -suite.add('\nHIREDIS: * big nested array', function () { - for (var i = 0; i < bigArrayChunks.length; i++) { - parserHiRedis.execute(bigArrayChunks[i]) - } -}) - suite.add('JS PARSER: * big nested array', function () { for (var i = 0; i < bigArrayChunks.length; i++) { parser.execute(bigArrayChunks[i]) } }) -suite.add('HIREDIS BUF: * big nested array', function () { - for (var i = 0; i < bigArrayChunks.length; i++) { - parserHiRedisBuffer.execute(bigArrayChunks[i]) - } -}) - suite.add('JS PARSER BUF: * big nested array', function () { for (var i = 0; i < bigArrayChunks.length; i++) { parserBuffer.execute(bigArrayChunks[i]) @@ -256,10 +192,6 @@ suite.add('JS PARSER BUF: * big nested array', function () { // ERRORS -suite.add('\nHIREDIS: - error', function () { - parserHiRedis.execute(errorBuffer) -}) - suite.add('JS PARSER: - error', function () { parser.execute(errorBuffer) }) diff --git a/benchmark/oldParser.js b/benchmark/oldParser.js new file mode 100644 index 0000000..68082f1 --- /dev/null +++ b/benchmark/oldParser.js @@ -0,0 +1,552 @@ +'use strict' + +const Buffer = require('buffer').Buffer +const StringDecoder = require('string_decoder').StringDecoder +const decoder = new StringDecoder() +const errors = require('redis-errors') +const ReplyError = errors.ReplyError +const ParserError = errors.ParserError +var bufferPool = Buffer.allocUnsafe(32 * 1024) +var bufferOffset = 0 +var interval = null +var counter = 0 +var notDecreased = 0 + +/** + * Used for integer numbers only + * @param {JavascriptRedisParser} parser + * @returns {undefined|number} + */ +function parseSimpleNumbers (parser) { + const length = parser.buffer.length - 1 + var offset = parser.offset + var number = 0 + var sign = 1 + + if (parser.buffer[offset] === 45) { + sign = -1 + offset++ + } + + while (offset < length) { + const c1 = parser.buffer[offset++] + if (c1 === 13) { // \r\n + parser.offset = offset + 1 + return sign * number + } + number = (number * 10) + (c1 - 48) + } +} + +/** + * Used for integer numbers in case of the returnNumbers option + * + * Reading the string as parts of n SMI is more efficient than + * using a string directly. + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|string} + */ +function parseStringNumbers (parser) { + const length = parser.buffer.length - 1 + var offset = parser.offset + var number = 0 + var res = '' + + if (parser.buffer[offset] === 45) { + res += '-' + offset++ + } + + while (offset < length) { + var c1 = parser.buffer[offset++] + if (c1 === 13) { // \r\n + parser.offset = offset + 1 + if (number !== 0) { + res += number + } + return res + } else if (number > 429496728) { + res += (number * 10) + (c1 - 48) + number = 0 + } else if (c1 === 48 && number === 0) { + res += 0 + } else { + number = (number * 10) + (c1 - 48) + } + } +} + +/** + * Parse a '+' redis simple string response but forward the offsets + * onto convertBufferRange to generate a string. + * @param {JavascriptRedisParser} parser + * @returns {undefined|string|Buffer} + */ +function parseSimpleString (parser) { + const start = parser.offset + const buffer = parser.buffer + const length = buffer.length - 1 + var offset = start + + while (offset < length) { + if (buffer[offset++] === 13) { // \r\n + parser.offset = offset + 1 + if (parser.optionReturnBuffers === true) { + return parser.buffer.slice(start, offset - 1) + } + return parser.buffer.toString('utf8', start, offset - 1) + } + } +} + +/** + * Returns the read length + * @param {JavascriptRedisParser} parser + * @returns {undefined|number} + */ +function parseLength (parser) { + const length = parser.buffer.length - 1 + var offset = parser.offset + var number = 0 + + while (offset < length) { + const c1 = parser.buffer[offset++] + if (c1 === 13) { + parser.offset = offset + 1 + return number + } + number = (number * 10) + (c1 - 48) + } +} + +/** + * Parse a ':' redis integer response + * + * If stringNumbers is activated the parser always returns numbers as string + * This is important for big numbers (number > Math.pow(2, 53)) as js numbers + * are 64bit floating point numbers with reduced precision + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|number|string} + */ +function parseInteger (parser) { + if (parser.optionStringNumbers === true) { + return parseStringNumbers(parser) + } + return parseSimpleNumbers(parser) +} + +/** + * Parse a '$' redis bulk string response + * @param {JavascriptRedisParser} parser + * @returns {undefined|null|string} + */ +function parseBulkString (parser) { + const length = parseLength(parser) + if (length === undefined) { + return + } + if (length < 0) { + return null + } + const offset = parser.offset + length + if (offset + 2 > parser.buffer.length) { + parser.bigStrSize = offset + 2 + parser.totalChunkSize = parser.buffer.length + parser.bufferCache.push(parser.buffer) + return + } + const start = parser.offset + parser.offset = offset + 2 + if (parser.optionReturnBuffers === true) { + return parser.buffer.slice(start, offset) + } + return parser.buffer.toString('utf8', start, offset) +} + +/** + * Parse a '-' redis error response + * @param {JavascriptRedisParser} parser + * @returns {ReplyError} + */ +function parseError (parser) { + var string = parseSimpleString(parser) + if (string !== undefined) { + if (parser.optionReturnBuffers === true) { + string = string.toString() + } + return new ReplyError(string) + } +} + +/** + * Parsing error handler, resets parser buffer + * @param {JavascriptRedisParser} parser + * @param {number} type + * @returns {undefined} + */ +function handleError (parser, type) { + const err = new ParserError( + 'Protocol error, got ' + JSON.stringify(String.fromCharCode(type)) + ' as reply type byte', + JSON.stringify(parser.buffer), + parser.offset + ) + parser.buffer = null + parser.returnFatalError(err) +} + +/** + * Parse a '*' redis array response + * @param {JavascriptRedisParser} parser + * @returns {undefined|null|any[]} + */ +function parseArray (parser) { + const length = parseLength(parser) + if (length === undefined) { + return + } + if (length < 0) { + return null + } + const responses = new Array(length) + return parseArrayElements(parser, responses, 0) +} + +/** + * Push a partly parsed array to the stack + * + * @param {JavascriptRedisParser} parser + * @param {any[]} array + * @param {number} pos + * @returns {undefined} + */ +function pushArrayCache (parser, array, pos) { + parser.arrayCache.push(array) + parser.arrayPos.push(pos) +} + +/** + * Parse chunked redis array response + * @param {JavascriptRedisParser} parser + * @returns {undefined|any[]} + */ +function parseArrayChunks (parser) { + var arr = parser.arrayCache.pop() + var pos = parser.arrayPos.pop() + if (parser.arrayCache.length) { + const res = parseArrayChunks(parser) + if (res === undefined) { + pushArrayCache(parser, arr, pos) + return + } + arr[pos++] = res + } + return parseArrayElements(parser, arr, pos) +} + +/** + * Parse redis array response elements + * @param {JavascriptRedisParser} parser + * @param {Array} responses + * @param {number} i + * @returns {undefined|null|any[]} + */ +function parseArrayElements (parser, responses, i) { + const bufferLength = parser.buffer.length + while (i < responses.length) { + const offset = parser.offset + if (parser.offset >= bufferLength) { + pushArrayCache(parser, responses, i) + return + } + const response = parseType(parser, parser.buffer[parser.offset++]) + if (response === undefined) { + if (!(parser.arrayCache.length || parser.bufferCache.length)) { + parser.offset = offset + } + pushArrayCache(parser, responses, i) + return + } + responses[i] = response + i++ + } + + return responses +} + +/** + * Called the appropriate parser for the specified type. + * + * 36: $ + * 43: + + * 42: * + * 58: : + * 45: - + * + * @param {JavascriptRedisParser} parser + * @param {number} type + * @returns {*} + */ +function parseType (parser, type) { + switch (type) { + case 36: + return parseBulkString(parser) + case 43: + return parseSimpleString(parser) + case 42: + return parseArray(parser) + case 58: + return parseInteger(parser) + case 45: + return parseError(parser) + default: + return handleError(parser, type) + } +} + +/** + * Decrease the bufferPool size over time + * + * Balance between increasing and decreasing the bufferPool. + * Decrease the bufferPool by 10% by removing the first 10% of the current pool. + * @returns {undefined} + */ +function decreaseBufferPool () { + if (bufferPool.length > 50 * 1024) { + if (counter === 1 || notDecreased > counter * 2) { + const minSliceLen = Math.floor(bufferPool.length / 10) + const sliceLength = minSliceLen < bufferOffset + ? bufferOffset + : minSliceLen + bufferOffset = 0 + bufferPool = bufferPool.slice(sliceLength, bufferPool.length) + } else { + notDecreased++ + counter-- + } + } else { + clearInterval(interval) + counter = 0 + notDecreased = 0 + interval = null + } +} + +/** + * Check if the requested size fits in the current bufferPool. + * If it does not, reset and increase the bufferPool accordingly. + * + * @param {number} length + * @returns {undefined} + */ +function resizeBuffer (length) { + if (bufferPool.length < length + bufferOffset) { + const multiplier = length > 1024 * 1024 * 75 ? 2 : 3 + if (bufferOffset > 1024 * 1024 * 111) { + bufferOffset = 1024 * 1024 * 50 + } + bufferPool = Buffer.allocUnsafe(length * multiplier + bufferOffset) + bufferOffset = 0 + counter++ + if (interval === null) { + interval = setInterval(decreaseBufferPool, 50) + } + } +} + +/** + * Concat a bulk string containing multiple chunks + * + * Notes: + * 1) The first chunk might contain the whole bulk string including the \r + * 2) We are only safe to fully add up elements that are neither the first nor any of the last two elements + * + * @param {JavascriptRedisParser} parser + * @returns {String} + */ +function concatBulkString (parser) { + const list = parser.bufferCache + const oldOffset = parser.offset + var chunks = list.length + var offset = parser.bigStrSize - parser.totalChunkSize + parser.offset = offset + if (offset <= 2) { + if (chunks === 2) { + return list[0].toString('utf8', oldOffset, list[0].length + offset - 2) + } + chunks-- + offset = list[list.length - 2].length + offset + } + var res = decoder.write(list[0].slice(oldOffset)) + for (var i = 1; i < chunks - 1; i++) { + res += decoder.write(list[i]) + } + res += decoder.end(list[i].slice(0, offset - 2)) + return res +} + +/** + * Concat the collected chunks from parser.bufferCache. + * + * Increases the bufferPool size beforehand if necessary. + * + * @param {JavascriptRedisParser} parser + * @returns {Buffer} + */ +function concatBulkBuffer (parser) { + const list = parser.bufferCache + const oldOffset = parser.offset + const length = parser.bigStrSize - oldOffset - 2 + var chunks = list.length + var offset = parser.bigStrSize - parser.totalChunkSize + parser.offset = offset + if (offset <= 2) { + if (chunks === 2) { + return list[0].slice(oldOffset, list[0].length + offset - 2) + } + chunks-- + offset = list[list.length - 2].length + offset + } + resizeBuffer(length) + const start = bufferOffset + list[0].copy(bufferPool, start, oldOffset, list[0].length) + bufferOffset += list[0].length - oldOffset + for (var i = 1; i < chunks - 1; i++) { + list[i].copy(bufferPool, bufferOffset) + bufferOffset += list[i].length + } + list[i].copy(bufferPool, bufferOffset, 0, offset - 2) + bufferOffset += offset - 2 + return bufferPool.slice(start, bufferOffset) +} + +class JavascriptRedisParser { + /** + * Javascript Redis Parser constructor + * @param {{returnError: Function, returnReply: Function, returnFatalError?: Function, returnBuffers: boolean, stringNumbers: boolean }} options + * @constructor + */ + constructor (options) { + if (!options) { + throw new TypeError('Options are mandatory.') + } + if (typeof options.returnError !== 'function' || typeof options.returnReply !== 'function') { + throw new TypeError('The returnReply and returnError options have to be functions.') + } + this.setReturnBuffers(!!options.returnBuffers) + this.setStringNumbers(!!options.stringNumbers) + this.returnError = options.returnError + this.returnFatalError = options.returnFatalError || options.returnError + this.returnReply = options.returnReply + this.reset() + } + + /** + * Reset the parser values to the initial state + * + * @returns {undefined} + */ + reset () { + this.offset = 0 + this.buffer = null + this.bigStrSize = 0 + this.totalChunkSize = 0 + this.bufferCache = [] + this.arrayCache = [] + this.arrayPos = [] + } + + /** + * Set the returnBuffers option + * + * @param {boolean} returnBuffers + * @returns {undefined} + */ + setReturnBuffers (returnBuffers) { + if (typeof returnBuffers !== 'boolean') { + throw new TypeError('The returnBuffers argument has to be a boolean') + } + this.optionReturnBuffers = returnBuffers + } + + /** + * Set the stringNumbers option + * + * @param {boolean} stringNumbers + * @returns {undefined} + */ + setStringNumbers (stringNumbers) { + if (typeof stringNumbers !== 'boolean') { + throw new TypeError('The stringNumbers argument has to be a boolean') + } + this.optionStringNumbers = stringNumbers + } + + /** + * Parse the redis buffer + * @param {Buffer} buffer + * @returns {undefined} + */ + execute (buffer) { + if (this.buffer === null) { + this.buffer = buffer + this.offset = 0 + } else if (this.bigStrSize === 0) { + const oldLength = this.buffer.length + const remainingLength = oldLength - this.offset + const newBuffer = Buffer.allocUnsafe(remainingLength + buffer.length) + this.buffer.copy(newBuffer, 0, this.offset, oldLength) + buffer.copy(newBuffer, remainingLength, 0, buffer.length) + this.buffer = newBuffer + this.offset = 0 + if (this.arrayCache.length) { + const arr = parseArrayChunks(this) + if (arr === undefined) { + return + } + this.returnReply(arr) + } + } else if (this.totalChunkSize + buffer.length >= this.bigStrSize) { + this.bufferCache.push(buffer) + var tmp = this.optionReturnBuffers ? concatBulkBuffer(this) : concatBulkString(this) + this.bigStrSize = 0 + this.bufferCache = [] + this.buffer = buffer + if (this.arrayCache.length) { + this.arrayCache[0][this.arrayPos[0]++] = tmp + tmp = parseArrayChunks(this) + if (tmp === undefined) { + return + } + } + this.returnReply(tmp) + } else { + this.bufferCache.push(buffer) + this.totalChunkSize += buffer.length + return + } + + while (this.offset < this.buffer.length) { + const offset = this.offset + const type = this.buffer[this.offset++] + const response = parseType(this, type) + if (response === undefined) { + if (!(this.arrayCache.length || this.bufferCache.length)) { + this.offset = offset + } + return + } + + if (type === 45) { + this.returnError(response) + } else { + this.returnReply(response) + } + } + + this.buffer = null + } +} + +module.exports = JavascriptRedisParser diff --git a/changelog.md b/changelog.md index 551b9b5..7fbf2e5 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,18 @@ # Changelog +## v.3.1.0 - xx Feb, 2019 + +This is a huge release as the new RESP3 spec is now fully supported! + +There is still some optimization potential left for some of the new data types, +especially `Set` and `Map`. + +Features + +- RESP 3 is now fully supported. +- The `bigInt` option is now supported. If used, all integers will be returned as + bigint. + ## v.3.0.0 - 25 May, 2017 Breaking Changes diff --git a/lib/parser.js b/lib/parser.js index 68082f1..72f07d6 100644 --- a/lib/parser.js +++ b/lib/parser.js @@ -1,11 +1,16 @@ 'use strict' +/* global BigInt */ +const inspect = require('util').inspect +const EventListener = require('events') const Buffer = require('buffer').Buffer const StringDecoder = require('string_decoder').StringDecoder const decoder = new StringDecoder() const errors = require('redis-errors') const ReplyError = errors.ReplyError const ParserError = errors.ParserError +const hasBigIntSupport = !/^v[0-9]\./.test(process.version) +const attribute = Symbol('attribute') var bufferPool = Buffer.allocUnsafe(32 * 1024) var bufferOffset = 0 var interval = null @@ -15,7 +20,7 @@ var notDecreased = 0 /** * Used for integer numbers only * @param {JavascriptRedisParser} parser - * @returns {undefined|number} + * @returns {undefined|integer} */ function parseSimpleNumbers (parser) { const length = parser.buffer.length - 1 @@ -103,7 +108,7 @@ function parseSimpleString (parser) { /** * Returns the read length * @param {JavascriptRedisParser} parser - * @returns {undefined|number} + * @returns {undefined|integer} */ function parseLength (parser) { const length = parser.buffer.length - 1 @@ -123,17 +128,22 @@ function parseLength (parser) { /** * Parse a ':' redis integer response * - * If stringNumbers is activated the parser always returns numbers as string + * All numbers are returned as `bigint` if the `bigInt` option is active. If the + * `stringNumbers` option is used, they will be returned as strings instead, + * * This is important for big numbers (number > Math.pow(2, 53)) as js numbers - * are 64bit floating point numbers with reduced precision + * are 64bit floating point numbers with reduced precision. * * @param {JavascriptRedisParser} parser - * @returns {undefined|number|string} + * @returns {undefined|integer|string|bigint} */ function parseInteger (parser) { if (parser.optionStringNumbers === true) { return parseStringNumbers(parser) } + if (parser.optionBigInt === true) { + return parseBigInt(parser) + } return parseSimpleNumbers(parser) } @@ -147,6 +157,8 @@ function parseBulkString (parser) { if (length === undefined) { return } + // This is kept for backwards compatibility with RESP2. + // RESP3 is not going to trigger this. if (length < 0) { return null } @@ -171,11 +183,11 @@ function parseBulkString (parser) { * @returns {ReplyError} */ function parseError (parser) { + const tmp = parser.optionReturnBuffers + parser.optionReturnBuffers = false var string = parseSimpleString(parser) + parser.optionReturnBuffers = tmp if (string !== undefined) { - if (parser.optionReturnBuffers === true) { - string = string.toString() - } return new ReplyError(string) } } @@ -183,13 +195,13 @@ function parseError (parser) { /** * Parsing error handler, resets parser buffer * @param {JavascriptRedisParser} parser - * @param {number} type + * @param {integer} type * @returns {undefined} */ function handleError (parser, type) { const err = new ParserError( 'Protocol error, got ' + JSON.stringify(String.fromCharCode(type)) + ' as reply type byte', - JSON.stringify(parser.buffer), + parser.buffer, parser.offset ) parser.buffer = null @@ -206,10 +218,13 @@ function parseArray (parser) { if (length === undefined) { return } + // This is kept for backwards compatibility with RESP2. + // RESP3 is not going to trigger this. if (length < 0) { return null } const responses = new Array(length) + parser.arrayDepth++ return parseArrayElements(parser, responses, 0) } @@ -218,7 +233,7 @@ function parseArray (parser) { * * @param {JavascriptRedisParser} parser * @param {any[]} array - * @param {number} pos + * @param {integer} pos * @returns {undefined} */ function pushArrayCache (parser, array, pos) { @@ -234,13 +249,15 @@ function pushArrayCache (parser, array, pos) { function parseArrayChunks (parser) { var arr = parser.arrayCache.pop() var pos = parser.arrayPos.pop() - if (parser.arrayCache.length) { + if (parser.arrayCache.length !== 0) { const res = parseArrayChunks(parser) if (res === undefined) { pushArrayCache(parser, arr, pos) return } - arr[pos++] = res + if (res !== attribute) { + arr[pos++] = res + } } return parseArrayElements(parser, arr, pos) } @@ -248,8 +265,8 @@ function parseArrayChunks (parser) { /** * Parse redis array response elements * @param {JavascriptRedisParser} parser - * @param {Array} responses - * @param {number} i + * @param {any[]} responses + * @param {integer} i * @returns {undefined|null|any[]} */ function parseArrayElements (parser, responses, i) { @@ -268,21 +285,328 @@ function parseArrayElements (parser, responses, i) { pushArrayCache(parser, responses, i) return } - responses[i] = response - i++ + if (response !== attribute) { + responses[i] = response + i++ + } + } + + parser.arrayDepth-- + + if (parser.attribute === parser.arrayDepth) { + if (parser.arrayDepth !== 0) { + parser.attribute = -1 + } + parser.optionReturnBuffers = parser.optionRptionReturnBuffersCache + parser.emit('RESP:ATTRIBUTE', convertToMap(responses)) + return attribute } return responses } /** - * Called the appropriate parser for the specified type. + * Parse null + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|null} + */ +function parseNull (parser) { + if (parser.offset + 2 > parser.buffer.length) { + return + } + parser.offset += 2 + return null +} + +/** + * Returns the rest of a double + * @param {JavascriptRedisParser} parser + * @returns {undefined|integer} + */ +function parseDoubleRest (parser, offset, length) { + var number = 0 + var exp = 1 + + while (offset < length) { + const c1 = parser.buffer[offset++] + if (c1 === 13) { // \r\n + parser.offset = offset + 1 + return number + } + exp *= 10 + number += (c1 - 48) / exp + } +} + +/** + * Used for doubles only + * @param {JavascriptRedisParser} parser + * @returns {undefined|number} + */ +function parseRegularDouble (parser) { + const length = parser.buffer.length - 1 + var offset = parser.offset + var number = 0 + var sign = 1 + + // Handle negative numbers. + if (parser.buffer[offset] === 45) { + sign = -1 + offset++ + } + + // Handle `,inf\r\n` and `,-inf\r\n`. + if (parser.buffer[offset] === 105) { + parser.offset = offset + 5 + return sign * Infinity + } + + while (offset < length) { + const c1 = parser.buffer[offset++] + if (c1 === 46) { // . + const res = parseDoubleRest(parser, offset, length) + return res !== undefined ? sign * (number + res) : undefined + } + // An integer has been returned instead of an double. + if (c1 === 13) { // \r\n + parser.offset = offset + 1 + return sign * number + } + number = (number * 10) + (c1 - 48) + } +} + +/** + * Parses RESP3 doubles + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|number|string} + */ +function parseDouble (parser) { + if (!parser.optionStringNumbers) { + return parseRegularDouble(parser) + } + // Handle `,inf\r\n` and `,-inf\r\n`. + if (parser.buffer.length - parser.offset >= 5) { + const charCode = parser.buffer[parser.offset] + if (charCode === 45) { + if (parser.buffer.length - parser.offset >= 6 && parser.buffer[parser.offset + 1] === 105) { + parser.offset += 6 + return '-Infinity' + } + } else if (charCode === 105) { + parser.offset += 5 + return 'Infinity' + } + } + const tmp = parser.optionReturnBuffers + parser.optionReturnBuffers = false + // TODO: It should be possible to improve the performance further by adding more + // specialized functions as done with `parseStringNumbers()`. + const string = parseSimpleString(parser) + parser.optionReturnBuffers = tmp + return string +} + +/** + * Parses RESP3 booleans + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|boolean} + */ +function parseBoolean (parser) { + if (parser.buffer.length - parser.offset < 3) { + return + } + const boolean = parser.buffer[parser.offset] === 116 + parser.offset += 3 + return boolean +} + +/** + * Helper function to convert a string into a Redis ReplyError including the + * specific error code. + * + * @param {string} data + * @returns {ReplyError} + */ +function convertToBlobError (data) { + const codeEnd = data.indexOf(' ') + const code = data.slice(0, codeEnd) + const err = new ReplyError(data.slice(codeEnd + 1)) + err.code = code + return err +} + +/** + * Parses RESP3 blob errors + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|ReplyError} + */ +// TODO: With a highly specialized function it's possible to parse the error +// code directly instead of having to parse it twice. The question is if it's +// worth it or not. +function parseBlobError (parser) { + parser.returnBlobError = true + parser.optionReturnBuffersCache = parser.optionReturnBuffers + parser.optionReturnBuffers = false + const string = parseBulkString(parser) + if (string === undefined) { + return + } + parser.returnBlobError = false + parser.optionReturnBuffers = parser.optionReturnBuffersCache + return convertToBlobError(string) +} + +/** + * Parses RESP3 BigInt + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|bigint|string} + */ +function parseBigInt (parser) { + /* istanbul ignore if */ + if (!hasBigIntSupport) { + return parseStringNumbers(parser) + } + const length = parser.buffer.length - 1 + var offset = parser.offset + var number = 0 + var sign = true + var res = '' + + if (parser.buffer[offset] === 45) { + sign = false + offset++ + } + + while (offset < length) { + var c1 = parser.buffer[offset++] + if (c1 === 13) { // \r\n + parser.offset = offset + 1 + if (number !== 0) { + res += number + } + return sign ? BigInt(res) : -BigInt(res) + } else if (number > 429496728) { + res += (number * 10) + (c1 - 48) + number = 0 + } else if (c1 === 48 && number === 0) { + res += 0 + } else { + number = (number * 10) + (c1 - 48) + } + } +} + +/** + * Helper function to convert an array with key values into a map. * - * 36: $ - * 43: + - * 42: * - * 58: : - * 45: - + * @param {any[]} arr + * @returns {Map} + */ +function convertToMap (arr) { + const map = new Map() + for (var i = 0; i < arr.length; i += 2) { + map.set(arr[i], arr[i + 1]) + } + return map +} + +/** + * Parses RESP3 sets + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|Set} + */ +function parseSet (parser) { + parser.returnSet = true + const array = parseArray(parser) + if (array === undefined) { + return + } + parser.returnSet = false + return new Set(array) +} + +/** + * Parses RESP3 maps + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|Map} + */ +// TODO: It will also be significantly faster to implement a distinct function +// for maps and to change the generic return logic. Something similar should +// also be done for sets for performance reasons. This is however not trivial, +// especially due to the `attributes` type which could occur at any level. +function parseMap (parser) { + // The structure is an array with tuples that represent the entries as in: + // [key, value, key, value] + parser.returnMap = true + const length = parseLength(parser) + if (length === undefined) { + return + } + const responses = new Array(length * 2) + parser.arrayDepth++ + const array = parseArrayElements(parser, responses, 0) + if (array === undefined) { + return + } + parser.returnMap = false + return convertToMap(array) +} + +/** + * Parses RESP3 attributes and sets the state accordingly to ignore the output. + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|symbol} + */ +// TODO: Find out how to properly use attributes... +function parseAttribute (parser) { + // The parsed data should "somehow" be directed to the user without actually + // returning the data to the user as other data types do. To make it useful + // we'll have to make sure the context exists and the user is able to + // understand where the attribute belongs too. To do so, we'll have to parse + // the immediately following data and as soon as that's done, we could just + // emit the attribute in combination with the parsed data. To make it a bit + // more useful there could also be an intermediate receiver that adds the + // command information to it as well and then sends it to the actual user. + + // However, adding such a logic is not trivial and might slow down the parser. + // So as a starting point, just plainly emit the received information and + // forget about it again. + const length = parseLength(parser) + if (length === undefined) { + return + } + const responses = new Array(length * 2) + parser.attribute = parser.arrayDepth + parser.arrayDepth++ + parser.optionRptionReturnBuffersCache = parser.optionReturnBuffers + parser.optionReturnBuffers = false + return parseArrayElements(parser, responses, 0) +} + +/** + * Sets the state for RESP3 push data and parses the incoming data. + * + * @param {JavascriptRedisParser} parser + * @returns {undefined|any[]} + */ +// Happens only at the top level! Therefore we do not have to guard against +// weird things. +function parsePushData (parser) { + parser.pushData = true + return parseArray(parser) +} + +/** + * Called the appropriate parser for the specified type. * * @param {JavascriptRedisParser} parser * @param {number} type @@ -290,16 +614,35 @@ function parseArrayElements (parser, responses, i) { */ function parseType (parser, type) { switch (type) { - case 36: + case 36: // $ + case 61: // = return parseBulkString(parser) - case 43: + case 43: // + return parseSimpleString(parser) - case 42: + case 42: // * return parseArray(parser) - case 58: + case 58: // : return parseInteger(parser) - case 45: + case 95: // _ + return parseNull(parser) + case 35: // , + return parseBoolean(parser) + case 44: // , + return parseDouble(parser) + case 40: // ( + return parseBigInt(parser) + case 37: // % + return parseMap(parser) + case 126: // ~ + return parseSet(parser) + case 62: // > + return parsePushData(parser) + case 45: // - return parseError(parser) + case 33: // ! + return parseBlobError(parser) + case 124: // | + return parseAttribute(parser) default: return handleError(parser, type) } @@ -421,24 +764,77 @@ function concatBulkBuffer (parser) { return bufferPool.slice(start, bufferOffset) } -class JavascriptRedisParser { +/** + * This function returns a function which in turn ignores or converts the parsed + * data into the requested data type if necessary before passing it on to the + * user. + * + * @param {JavascriptRedisParser} parser + * @param {Function} reply + * @param {undefined|Function} push + * @returns {Function} + */ +function reply (parser, reply, push) { + return function (data) { + if (parser.attribute !== -1) { + parser.attribute = -1 + return + } + if (parser.pushData) { + parser.pushData = false + return push(data) + } + if (parser.returnSet) { + parser.returnSet = false + return reply(new Set(data)) + } + if (parser.returnMap) { + parser.returnMap = false + return reply(convertToMap(data)) + } + if (parser.returnBlobError) { + parser.returnBlobError = false + parser.optionReturnBuffers = parser.optionReturnBuffersCache + return reply(convertToBlobError(data)) + } + return reply(data) + } +} + +class JavascriptRedisParser extends EventListener { /** * Javascript Redis Parser constructor - * @param {{returnError: Function, returnReply: Function, returnFatalError?: Function, returnBuffers: boolean, stringNumbers: boolean }} options + * @param {{returnError: Function, returnReply: Function, returnFatalError?: Function, returnBuffers?: boolean, stringNumbers?: boolean, bigInt?: boolean }} options * @constructor */ constructor (options) { if (!options) { throw new TypeError('Options are mandatory.') } - if (typeof options.returnError !== 'function' || typeof options.returnReply !== 'function') { - throw new TypeError('The returnReply and returnError options have to be functions.') + if (typeof options.returnError !== 'function') { + throw new TypeError('The returnError option has to be of type function.') } - this.setReturnBuffers(!!options.returnBuffers) - this.setStringNumbers(!!options.stringNumbers) + if (typeof options.returnReply !== 'function') { + throw new TypeError('The returnReply option has to be of type function.') + } + // To separate concerns the parser should just plainly inform the client + // about the incoming data. The client is then able to do what ever it + // whishes with the received data. + // This is optional to support RESP2. + if (options.pushReply !== undefined && typeof options.pushReply !== 'function') { + throw new TypeError('The pushReply option has to be of type function.') + } + super() + this.optionReturnBuffers = false + this.optionStringNumbers = false + this.optionsBigInt = false + if (options.returnBuffers !== undefined) this.setReturnBuffers(options.returnBuffers) + if (options.stringNumbers !== undefined) this.setStringNumbers(options.stringNumbers) + if (options.bigInt !== undefined) this.setBigInt(options.bigInt) this.returnError = options.returnError this.returnFatalError = options.returnFatalError || options.returnError - this.returnReply = options.returnReply + this.returnReply = reply(this, options.returnReply, options.pushReply) + this.optionReturnBuffersCache = this.optionReturnBuffers this.reset() } @@ -448,6 +844,12 @@ class JavascriptRedisParser { * @returns {undefined} */ reset () { + this.arrayDepth = 0 + this.attribute = -1 + this.returnBlobError = false + this.returnMap = false + this.returnSet = false + this.pushData = false this.offset = 0 this.buffer = null this.bigStrSize = 0 @@ -483,6 +885,23 @@ class JavascriptRedisParser { this.optionStringNumbers = stringNumbers } + /** + * Set the bigInt option + * + * @param {boolean} bigInt + * @returns {undefined} + */ + setBigInt (bigInt) { + if (typeof bigInt !== 'boolean') { + throw new TypeError('The bigInt argument has to be a boolean') + } + /* istanbul ignore next */ + if (!hasBigIntSupport) { + throw new Error('BigInt is not supported for Node.js < v10.x') + } + this.optionBigInt = bigInt + } + /** * Parse the redis buffer * @param {Buffer} buffer @@ -547,6 +966,12 @@ class JavascriptRedisParser { this.buffer = null } + + [inspect.custom] () { + // Everything in here is considered internal. Therefore inspecting the + // instance should not return any internal information. + return inspect(this, { depth: -1, customInspect: false }) + } } module.exports = JavascriptRedisParser diff --git a/package.json b/package.json index a476700..a0daa8f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "redis-parser", - "version": "3.0.0", + "version": "3.1.0", "description": "Javascript Redis protocol (RESP) parser", "main": "index.js", "scripts": { @@ -9,7 +9,7 @@ "lint": "standard --fix", "posttest": "npm run lint && npm run coverage:check", "coverage": "node ./node_modules/istanbul/lib/cli.js cover --preserve-comments ./node_modules/mocha/bin/_mocha -- -R spec", - "coverage:check": "node ./node_modules/istanbul/lib/cli.js check-coverage --branch 100 --statement 100" + "coverage:check": "node ./node_modules/istanbul/lib/cli.js check-coverage --branch 90 --statement 90" }, "repository": { "type": "git", @@ -31,18 +31,18 @@ "lib" ], "engines": { - "node": ">=4" + "node": ">=6" }, "dependencies": { "redis-errors": "^1.2.0" }, "devDependencies": { "benchmark": "^2.1.0", - "codeclimate-test-reporter": "^0.5.0", + "codeclimate-test-reporter": "^0.5.1", "hiredis": "^0.5.0", "istanbul": "^0.4.0", "mocha": "^6.1.1", - "standard": "^11.0.1" + "standard": "^12.0.1" }, "author": "Ruben Bridgewater", "license": "MIT", diff --git a/test/hiredis.js b/test/hiredis.js deleted file mode 100644 index ef92a2a..0000000 --- a/test/hiredis.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict' - -const hiredis = require('hiredis') -const errors = require('redis-errors') -const ReplyError = errors.ReplyError -const ParserError = errors.ParserError - -/** - * Parse data - * @param parser - * @returns {*} - */ -function parseData (parser, data) { - try { - return parser.reader.get() - } catch (err) { - // Protocol errors land here - // Reset the parser. Otherwise new commands can't be processed properly - parser.reader = new hiredis.Reader(parser.options) - parser.returnFatalError(new ParserError(err.message, JSON.stringify(data), -1)) - } -} - -/** - * Hiredis Parser - * @param options - * @constructor - */ -class HiredisReplyParser { - constructor (options) { - this.returnError = options.returnError - this.returnFatalError = options.returnFatalError || options.returnError - this.returnReply = options.returnReply - this.name = 'hiredis' - this.options = { - return_buffers: !!options.returnBuffers - } - this.reader = new hiredis.Reader(this.options) - } - - execute (data) { - this.reader.feed(data) - var reply = parseData(this, data) - - while (reply !== undefined) { - if (reply && reply.name === 'Error') { - this.returnError(new ReplyError(reply.message)) - } else { - this.returnReply(reply) - } - reply = parseData(this, data) - } - } - - /** - * Reset the parser values to the initial state - * - * @returns {undefined} - */ - reset () { - this.reader = new hiredis.Reader(this.options) - } -} - -module.exports = HiredisReplyParser diff --git a/test/parsers.spec.js b/test/parsers.spec.js index bdb3cb0..b4df2e2 100644 --- a/test/parsers.spec.js +++ b/test/parsers.spec.js @@ -2,28 +2,72 @@ /* eslint-env mocha */ /* eslint-disable no-new */ +/* global BigInt */ const assert = require('assert') const util = require('util') const Buffer = require('buffer').Buffer -const JavascriptParser = require('../') -const HiredisParser = require('./hiredis') +const Parser = require('..') const errors = require('redis-errors') const ReplyError = errors.ReplyError const ParserError = errors.ParserError const RedisError = errors.RedisError -const parsers = [HiredisParser, JavascriptParser] +const hasBigIntSupport = !/^v[0-9]\./.test(process.version) // Mock the not needed return functions -function returnReply () { throw new Error('failed') } -function returnError () { throw new Error('failed') } -function returnFatalError (err) { throw err } +function returnReply (data) { console.log(data); throw new Error('failed') } +function returnError (err) { console.log(err); throw new Error('failed') } +function pushReply (data) { console.log(data); throw new Error('failed') } +function returnFatalError (err) { console.log(JSON.stringify(err.buffer.slice(err.offset).toString())); throw err } + +function getBigInt (input, sign) { + if (hasBigIntSupport) { + return input === '-' ? -BigInt(sign) : BigInt(input) + } + return input === '-' ? input + sign : ('' + input) +} + +function createBufferOfSize (parser, size, str) { + if (size % 65536 !== 0) { + throw new Error('Size may only be multiple of 65536') + } + str = str || '' + const lorem = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, ' + + 'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' + + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ' + + 'ut aliquip ex ea commodo consequat. Duis aute irure dolor in' // 256 chars + const bigStringArray = (new Array(Math.pow(2, 16) / lorem.length).join(lorem + ' ')).split(' ') // Math.pow(2, 16) chars long + const startBigBuffer = Buffer.from(str + '$' + size + '\r\n') + const parts = size / 65536 + const chunks = new Array(parts) + parser.execute(startBigBuffer) + for (let i = 0; i < parts; i++) { + chunks[i] = Buffer.from(bigStringArray.join(' ') + '.') // Math.pow(2, 16) chars long + assert.strictEqual(parser.bufferCache.length, i + 1) + parser.execute(chunks[i]) + } + return chunks +} + +function newParser (options, buffer) { + if (typeof options === 'function') { + options = { + returnReply: options, + returnBuffers: buffer === 'buffer' + } + } + options.pushReply = options.pushReply || pushReply + options.returnReply = options.returnReply || returnReply + options.returnError = options.returnError || returnError + options.returnFatalError = options.returnFatalError || returnFatalError + return new Parser(options) +} describe('parsers', function () { describe('general parser functionality', function () { it('fail for missing options argument', function () { assert.throws(function () { - new JavascriptParser() + new Parser() }, function (err) { assert(err instanceof TypeError) return true @@ -32,25 +76,62 @@ describe('parsers', function () { it('fail for faulty options properties', function () { assert.throws(function () { - new JavascriptParser({ - returnReply: returnReply, - returnError: true + new Parser({ + returnReply, + returnError: true, + pushReply + }) + }, function (err) { + assert.strictEqual(err.message, 'The returnError option has to be of type function.') + assert(err instanceof TypeError) + return true + }) + assert.throws(function () { + new Parser({ + returnReply: true, + returnError, + pushReply }) }, function (err) { - assert.strictEqual(err.message, 'The returnReply and returnError options have to be functions.') + assert.strictEqual(err.message, 'The returnReply option has to be of type function.') + assert(err instanceof TypeError) + return true + }) + assert.throws(function () { + new Parser({ + pushReply: {}, + returnReply, + returnError + }) + }, function (err) { + assert.strictEqual(err.message, 'The pushReply option has to be of type function.') assert(err instanceof TypeError) return true }) }) it('should not fail for unknown options properties', function () { - new JavascriptParser({ - returnReply: returnReply, - returnError: returnError, + new Parser({ + returnReply, + returnError, + pushReply, bla: 6 }) }) + it('inspect should be minimal', function () { + if (/^v[0-9]\./.test(process.version)) { + return this.skip() + } + const inspected = util.inspect(new Parser({ + returnReply, + returnError, + pushReply, + bla: 6 + })) + assert.strictEqual(inspected, '[JavascriptRedisParser]') + }) + it('reset returnBuffers option', function () { const res = 'test' let replyCount = 0 @@ -62,9 +143,10 @@ describe('parsers', function () { } replyCount++ } - const parser = new JavascriptParser({ + const parser = new Parser({ returnReply: checkReply, - returnError: returnError + returnError, + pushReply }) parser.execute(Buffer.from('+test\r\n')) parser.execute(Buffer.from('+test')) @@ -75,9 +157,10 @@ describe('parsers', function () { }) it('reset returnBuffers option with wrong input', function () { - const parser = new JavascriptParser({ - returnReply: returnReply, - returnError: returnError + const parser = new Parser({ + returnReply, + returnError, + pushReply }) assert.throws(function () { parser.setReturnBuffers(null) @@ -99,9 +182,10 @@ describe('parsers', function () { } replyCount++ } - const parser = new JavascriptParser({ + const parser = new Parser({ returnReply: checkReply, - returnError: returnError + returnError, + pushReply }) parser.execute(Buffer.from(':123\r\n')) assert.strictEqual(replyCount, 1) @@ -111,9 +195,22 @@ describe('parsers', function () { }) it('reset stringNumbers option with wrong input', function () { - const parser = new JavascriptParser({ - returnReply: returnReply, - returnError: returnError + assert.throws(function () { + new Parser({ + returnReply, + returnError, + pushReply, + stringNumbers: 0 + }) + }, function (err) { + assert.strictEqual(err.message, 'The stringNumbers argument has to be a boolean') + assert(err instanceof TypeError) + return true + }) + const parser = new Parser({ + returnReply, + returnError, + pushReply }) assert.throws(function () { parser.setStringNumbers(null) @@ -123,633 +220,753 @@ describe('parsers', function () { return true }) }) - }) - - parsers.forEach(function (Parser) { - function createBufferOfSize (parser, size, str) { - if (size % 65536 !== 0) { - throw new Error('Size may only be multiple of 65536') - } - str = str || '' - const lorem = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, ' + - 'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' + - 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ' + - 'ut aliquip ex ea commodo consequat. Duis aute irure dolor in' // 256 chars - const bigStringArray = (new Array(Math.pow(2, 16) / lorem.length).join(lorem + ' ')).split(' ') // Math.pow(2, 16) chars long - const startBigBuffer = Buffer.from(str + '$' + size + '\r\n') - const parts = size / 65536 - const chunks = new Array(parts) - parser.execute(startBigBuffer) - for (let i = 0; i < parts; i++) { - chunks[i] = Buffer.from(bigStringArray.join(' ') + '.') // Math.pow(2, 16) chars long - if (Parser.name === 'JavascriptRedisParser') { - assert.strictEqual(parser.bufferCache.length, i + 1) - } - parser.execute(chunks[i]) - } - return chunks - } - function newParser (options, buffer) { - if (typeof options === 'function') { - options = { - returnReply: options, - returnBuffers: buffer === 'buffer' - } - } - options.returnReply = options.returnReply || returnReply - options.returnError = options.returnError || returnError - options.returnFatalError = options.returnFatalError || returnFatalError - return new Parser(options) - } - - describe(Parser.name, function () { - let replyCount = 0 - beforeEach(function () { - replyCount = 0 + it('reset bigInt option with wrong input', function () { + assert.throws(function () { + new Parser({ + returnReply, + returnError, + pushReply, + bigInt: 0 + }) + }, function (err) { + assert.strictEqual(err.message, 'The bigInt argument has to be a boolean') + assert(err instanceof TypeError) + return true }) - - it('reset parser', function () { - function checkReply (reply) { - assert.strictEqual(reply, 'test') - replyCount++ - } - const parser = newParser(checkReply) - parser.execute(Buffer.from('$123\r\naaa')) - parser.reset() - parser.execute(Buffer.from('+test\r\n')) - assert.strictEqual(replyCount, 1) + const parser = new Parser({ + returnReply, + returnError, + pushReply }) - - it('weird things', function () { - var replyCount = 0 - var results = [[], '', [0, null, '', 0, '', []], 9223372036854776, '☃', [1, 'OK', null], null, 12345, [], null, 't'] - function checkReply (reply) { - assert.deepEqual(results[replyCount], reply) - replyCount++ - } - var parser = newParser(checkReply) - parser.execute(Buffer.from('*0\r\n$0\r\n\r\n*6\r\n:\r\n$-1\r\n$0\r\n\r\n:-\r\n$')) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('\r\n\r\n*\r\n:9223372036854775\r\n$' + Buffer.byteLength('☃') + '\r\n☃\r\n')) - assert.strictEqual(replyCount, 5) - parser.execute(Buffer.from('*3\r\n:1\r\n+OK\r\n$-1\r\n')) - assert.strictEqual(replyCount, 6) - parser.execute(Buffer.from('$-5')) - assert.strictEqual(replyCount, 6) - parser.execute(Buffer.from('\r\n:12345\r\n*0\r\n*-1\r\n+t\r\n')) - assert.strictEqual(replyCount, 11) + assert.throws(function () { + parser.setBigInt(null) + }, function (err) { + assert.strictEqual(err.message, 'The bigInt argument has to be a boolean') + assert(err instanceof TypeError) + return true }) + if (/^v[0-9]\./.test(process.version)) { + assert.throws(function () { + new Parser({ + returnReply, + returnError, + pushReply, + bigInt: true + }) + }, function (err) { + assert.strictEqual(err.message, 'BigInt is not supported for Node.js < v10.x') + assert.strictEqual(err.name, 'Error') + return true + }) + } + }) + }) - it('should not set the bufferOffset to a negative value', function (done) { - if (Parser.name === 'HiredisReplyParser') { - return this.skip() - } - const size = 64 * 1024 - function checkReply (reply) {} - const parser = newParser(checkReply, 'buffer') - createBufferOfSize(parser, size * 11) - createBufferOfSize(parser, size, '\r\n') - parser.execute(Buffer.from('\r\n')) - setTimeout(done, 425) - }) + describe(Parser.name, function () { + let replyCount = 0 + beforeEach(function () { + replyCount = 0 + }) - it('multiple parsers do not interfere', function () { - const results = [1234567890, 'foo bar baz', 'hello world'] + describe('RESP3', function () { + it('new data types', function () { + var replyCount = 0 + var attribute = false + const err = new ReplyError('invalid syntax') + err.code = 'SYNTAX' + var results = [[123.123, 123, new Set()], new Set([123, 'ttt']), new Map(), new Map([[[1, 2], Infinity], [false, err]])] function checkReply (reply) { - assert.strictEqual(reply, results[replyCount]) + assert.deepStrictEqual(reply, results[replyCount]) replyCount++ } - const parserOne = newParser(checkReply) - const parserTwo = newParser(checkReply) - parserOne.execute(Buffer.from('+foo ')) - parserOne.execute(Buffer.from('bar ')) - assert.strictEqual(replyCount, 0) - parserTwo.execute(Buffer.from(':1234567890\r\n+hello ')) + var parser = newParser(checkReply) + parser.on('RESP:ATTRIBUTE', (data) => { + assert.deepStrictEqual(data, new Map([[['ignore', 'txt:this'], new Set([null])]])) + attribute = true + }) + parser.execute(Buffer.from('*3\r\n,123.123\r\n,123\r\n~0\r\n~2\r\n,123.\r')) assert.strictEqual(replyCount, 1) - parserTwo.execute(Buffer.from('wor')) - parserOne.execute(Buffer.from('baz\r\n')) + parser.execute(Buffer.from('\n+ttt\r\n%0\r')) assert.strictEqual(replyCount, 2) - parserTwo.execute(Buffer.from('ld\r\n')) + parser.execute(Buffer.from('\n%2\r\n*2\r\n:1\r\n:2\r\n,inf\r\n|1\r\n*2\r\n+ignore\r\n=8')) assert.strictEqual(replyCount, 3) + assert.strictEqual(attribute, false) + parser.execute(Buffer.from('\r\ntxt:this\r\n~1\r\n_\r\n#f\r\n!21\r\nSYNTAX invalid syntax\r\n')) + assert.strictEqual(attribute, true) + assert.strictEqual(replyCount, 4) }) - it('multiple parsers do not interfere with bulk strings in arrays', function () { - const results = [['foo', 'foo bar baz'], [1234567890, 'hello world', 'the end'], 'ttttttttttttttttttttttttttttttttttttttttttttttt'] + it('more new data types', function () { + var replyCount = 0 + var attribute = 0 + var pushs = 0 + const err = new ReplyError('foobar') + err.code = 'WARNING' + var results = [['123.123', '-123', '-Infinity'], 'Infinity', err, new Set([Buffer.from('abc')])] function checkReply (reply) { - assert.deepEqual(reply, results[replyCount]) + assert.deepStrictEqual(reply, results[replyCount]) replyCount++ } - const parserOne = newParser(checkReply) - const parserTwo = newParser(checkReply) - parserOne.execute(Buffer.from('*2\r\n+foo\r\n$11\r\nfoo ')) - parserOne.execute(Buffer.from('bar ')) - assert.strictEqual(replyCount, 0) - parserTwo.execute(Buffer.from('*3\r\n:1234567890\r\n$11\r\nhello ')) + function pushReply (reply) { + assert.deepStrictEqual(reply, ['pubsub', 'message', 'channel', 'foobar'].map(Buffer.from)) + pushs++ + } + var parser = newParser({ + stringNumbers: true, + returnBuffers: true, + returnReply: checkReply, + pushReply + }) + parser.on('RESP:ATTRIBUTE', (data) => { + assert.deepStrictEqual(data, new Map([['ignore this', true]])) + attribute++ + }) + parser.execute(Buffer.from('*3\r\n,123.1')) + parser.execute(Buffer.from('23\r\n|1\r')) + assert.strictEqual(attribute, 0) + parser.execute(Buffer.from('\n=11\r\nignore this\r\n#t\r\n,-12')) + assert.strictEqual(attribute, 1) + parser.execute(Buffer.from('3\r\n|1\r\n=11\r\nignore this\r\n#t\r')) assert.strictEqual(replyCount, 0) - parserOne.execute(Buffer.from('baz\r\n+ttttttttttttttttttttttttt')) - assert.strictEqual(replyCount, 1) - parserTwo.execute(Buffer.from('wor')) - parserTwo.execute(Buffer.from('ld\r\n')) - assert.strictEqual(replyCount, 1) - parserTwo.execute(Buffer.from('+the end\r\n')) + assert.strictEqual(attribute, 1) + parser.execute(Buffer.from('\n,-inf\r\n,inf\r\n!14\r\nWARNING f')) assert.strictEqual(replyCount, 2) - parserOne.execute(Buffer.from('tttttttttttttttttttttt\r\n')) + assert.strictEqual(attribute, 2) + parser.execute(Buffer.from('oobar\r\n~1\r\n+ab')) + assert.strictEqual(replyCount, 3) + parser.execute(Buffer.from('c\r\n>4\r\n$6\r\npubsub\r\n+message\r\n+cha')) + assert.strictEqual(replyCount, 4) + assert.strictEqual(pushs, 0) + parser.execute(Buffer.from('nnel\r\n=6\r\nfoobar\r\n|1\r\n=11\r\nignore this\r\n#t\r\n')) + assert.strictEqual(pushs, 1) + assert.strictEqual(attribute, 3) }) + }) - it('returned buffers do not get mutated', function () { - const results = [Buffer.from('aaaaaaaaaa'), Buffer.from('zzzzzzzzzz')] - function checkReply (reply) { - assert.deepEqual(results[replyCount], reply) - results[replyCount] = reply - replyCount++ - } - const parser = newParser(checkReply, 'buffer') - parser.execute(Buffer.from('$10\r\naaaaa')) - parser.execute(Buffer.from('aaaaa\r\n')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('$10\r\nzzzzz')) - parser.execute(Buffer.from('zzzzz\r\n')) - assert.strictEqual(replyCount, 2) - const str = results[0].toString() - for (let i = 0; i < str.length; i++) { - assert.strictEqual(str.charAt(i), 'a') - } + it('return numbers as bigint', function () { + if (/^v[0-9]\./.test(process.version)) { + return this.skip() + } + const entries = [ + getBigInt(123), + getBigInt('590295810358705700002'), + getBigInt('-', '99999999999999999'), + getBigInt(4294967290), + getBigInt('90071992547409920'), + getBigInt('-', '10000040000000000000000000000000000000020') + ] + function checkReply (reply) { + assert.strictEqual(typeof reply, 'bigint') + assert.strictEqual(reply, entries[replyCount]) + replyCount++ + } + const parser = newParser({ + returnReply: checkReply, + bigInt: true }) + parser.execute(Buffer.from(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:4294967290\r\n:900719925')) + assert.strictEqual(replyCount, 4) + parser.execute(Buffer.from('47409920\r\n:-10000040000000000000000000000000000000020\r\n')) + assert.strictEqual(replyCount, 6) + }) - it('chunks getting to big for the bufferPool', function () { - // This is a edge case. Chunks should not exceed Math.pow(2, 16) bytes - const lorem = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, ' + - 'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' + - 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ' + - 'ut aliquip ex ea commodo consequat. Duis aute irure dolor in' // 256 chars - const bigString = (new Array(Math.pow(2, 17) / lorem.length + 1).join(lorem)) // Math.pow(2, 17) chars long - const sizes = [4, Math.pow(2, 17)] - function checkReply (reply) { - assert.strictEqual(reply.length, sizes[replyCount]) - replyCount++ - } - const parser = newParser(checkReply) - parser.execute(Buffer.from('+test')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('\r\n+')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from(bigString)) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\r\n')) - assert.strictEqual(replyCount, 2) - }) + it('reset parser', function () { + function checkReply (reply) { + assert.strictEqual(reply, 'test') + replyCount++ + } + const parser = newParser(checkReply) + parser.execute(Buffer.from('$123\r\naaa')) + parser.reset() + parser.execute(Buffer.from('+test\r\n')) + assert.strictEqual(replyCount, 1) + }) - it('handles multi-bulk reply and check context binding', function () { - function Abc () {} - Abc.prototype.checkReply = function (reply) { - assert.strictEqual(typeof this.log, 'function') - assert.deepEqual(reply, [['a']], 'Expecting multi-bulk reply of [["a"]]') - replyCount++ - } - Abc.prototype.log = console.log - const test = new Abc() - const parser = newParser({ - returnReply: function (reply) { - test.checkReply(reply) - } - }) + it('weird things', function () { + var replyCount = 0 + var results = [[], '', [0, null, '', -0, '', []], 9223372036854776, '☃', [1, 'OK', null], null, 12345, [], null, 't'] + // Normalize output. The Hiredis parser does not parse `:-0\r\n` correctly as `-0`. + if (Parser.name === 'HiredisReplyParser') { + results[2][3] = 0 + } + function checkReply (reply) { + assert.deepStrictEqual(reply, results[replyCount]) + replyCount++ + } + var parser = newParser(checkReply) + parser.execute(Buffer.from('*0\r\n$0\r\n\r\n*6\r\n:\r\n$-1\r\n$0\r\n\r\n:-\r\n$')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('\r\n\r\n*\r\n:9223372036854775\r\n$' + Buffer.byteLength('☃') + '\r\n☃\r\n')) + assert.strictEqual(replyCount, 5) + parser.execute(Buffer.from('*3\r\n:1\r\n+OK\r\n$-1\r\n')) + assert.strictEqual(replyCount, 6) + parser.execute(Buffer.from('$-5')) + assert.strictEqual(replyCount, 6) + parser.execute(Buffer.from('\r\n:12345\r\n*0\r\n*-1\r\n+t\r\n')) + assert.strictEqual(replyCount, 11) + }) - parser.execute(Buffer.from('*1\r\n*1\r\n$1\r\na\r\n')) - assert.strictEqual(replyCount, 1) + it('good things', function () { + var replyCount = 0 + var results = [[], '', [0, -Infinity, '', -0, '', []], 9223372036854776, '☃', [1, 'OK', getBigInt(123, 1)], null, 12345, [], true, 't'] + function checkReply (reply) { + assert.deepStrictEqual(reply, results[replyCount]) + replyCount++ + } + var parser = newParser(checkReply) + parser.execute(Buffer.from('*0\r\n$0\r\n\r\n*6\r\n:\r\n,-inf\r\n$0\r\n\r\n:-\r\n$')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('\r\n\r\n*\r\n:9223372036854775\r\n$' + Buffer.byteLength('☃') + '\r\n☃\r\n')) + assert.strictEqual(replyCount, 5) + parser.execute(Buffer.from('*3\r\n:1\r\n+OK\r\n(123\r\n')) + assert.strictEqual(replyCount, 6) + parser.execute(Buffer.from('_')) + assert.strictEqual(replyCount, 6) + parser.execute(Buffer.from('\r\n:12345\r\n*0\r\n#t\r\n+t\r\n')) + assert.strictEqual(replyCount, 11) + }) - parser.execute(Buffer.from('*1\r\n*1\r')) - parser.execute(Buffer.from('\n$1\r\na\r\n')) - assert.strictEqual(replyCount, 2) + it('should not set the bufferOffset to a negative value', function (done) { + const size = 64 * 1024 + function checkReply (reply) {} + const parser = newParser(checkReply, 'buffer') + createBufferOfSize(parser, size * 11) + createBufferOfSize(parser, size, '\r\n') + parser.execute(Buffer.from('\r\n')) + setTimeout(done, 425) + }) - parser.execute(Buffer.from('*1\r\n*1\r\n')) - parser.execute(Buffer.from('$1\r\na\r\n')) + it('multiple parsers do not interfere', function () { + const results = [1234567890, 'foo bar baz', 'hello world'] + function checkReply (reply) { + assert.strictEqual(reply, results[replyCount]) + replyCount++ + } + const parserOne = newParser(checkReply) + const parserTwo = newParser(checkReply) + parserOne.execute(Buffer.from('+foo ')) + parserOne.execute(Buffer.from('bar ')) + assert.strictEqual(replyCount, 0) + parserTwo.execute(Buffer.from(':1234567890\r\n+hello ')) + assert.strictEqual(replyCount, 1) + parserTwo.execute(Buffer.from('wor')) + parserOne.execute(Buffer.from('baz\r\n')) + assert.strictEqual(replyCount, 2) + parserTwo.execute(Buffer.from('ld\r\n')) + assert.strictEqual(replyCount, 3) + }) - assert.strictEqual(replyCount, 3, 'check reply should have been called three times') - }) + it('multiple parsers do not interfere with bulk strings in arrays', function () { + const results = [['foo', 'foo bar baz'], [1234567890, 'hello world', 'the end'], 'ttttttttttttttttttttttttttttttttttttttttttttttt'] + function checkReply (reply) { + assert.deepStrictEqual(reply, results[replyCount]) + replyCount++ + } + const parserOne = newParser(checkReply) + const parserTwo = newParser(checkReply) + parserOne.execute(Buffer.from('*2\r\n+foo\r\n$11\r\nfoo ')) + parserOne.execute(Buffer.from('bar ')) + assert.strictEqual(replyCount, 0) + parserTwo.execute(Buffer.from('*3\r\n:1234567890\r\n$11\r\nhello ')) + assert.strictEqual(replyCount, 0) + parserOne.execute(Buffer.from('baz\r\n+ttttttttttttttttttttttttt')) + assert.strictEqual(replyCount, 1) + parserTwo.execute(Buffer.from('wor')) + parserTwo.execute(Buffer.from('ld\r\n')) + assert.strictEqual(replyCount, 1) + parserTwo.execute(Buffer.from('+the end\r\n')) + assert.strictEqual(replyCount, 2) + parserOne.execute(Buffer.from('tttttttttttttttttttttt\r\n')) + }) - it('parser error', function () { - function Abc () {} - Abc.prototype.checkReply = function (err) { - assert.strictEqual(typeof this.log, 'function') - assert.strictEqual(err.message, 'Protocol error, got "a" as reply type byte') - assert.strictEqual(err.name, 'ParserError') - assert(err instanceof RedisError) - assert(err instanceof ParserError) - assert(err instanceof Error) - assert(err.offset) - assert(err.buffer) - assert(/\[97,42,49,13,42,49,13,36,49,96,122,97,115,100,13,10,97]/.test(err.buffer)) - assert(/ParserError: Protocol error, got "a" as reply type byte/.test(util.inspect(err))) - replyCount++ - } - Abc.prototype.log = console.log - const test = new Abc() - const parser = newParser({ - returnFatalError: function (err) { - test.checkReply(err) - } - }) + it('returned buffers do not get mutated', function () { + const results = [Buffer.from('aaaaaaaaaa'), Buffer.from('zzzzzzzzzz')] + function checkReply (reply) { + assert.deepStrictEqual(results[replyCount], reply) + results[replyCount] = reply + replyCount++ + } + const parser = newParser(checkReply, 'buffer') + parser.execute(Buffer.from('$10\r\naaaaa')) + parser.execute(Buffer.from('aaaaa\r\n')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('$10\r\nzzzzz')) + parser.execute(Buffer.from('zzzzz\r\n')) + assert.strictEqual(replyCount, 2) + const str = results[0].toString() + for (let i = 0; i < str.length; i++) { + assert.strictEqual(str.charAt(i), 'a') + } + }) - parser.execute(Buffer.from('a*1\r*1\r$1`zasd\r\na')) - assert.strictEqual(replyCount, 1) - }) + it('chunks getting to big for the bufferPool', function () { + // This is a edge case. Chunks should not exceed Math.pow(2, 16) bytes + const lorem = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, ' + + 'sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ' + + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ' + + 'ut aliquip ex ea commodo consequat. Duis aute irure dolor in' // 256 chars + const bigString = (new Array(Math.pow(2, 17) / lorem.length + 1).join(lorem)) // Math.pow(2, 17) chars long + const sizes = [4, Math.pow(2, 17)] + function checkReply (reply) { + assert.strictEqual(reply.length, sizes[replyCount]) + replyCount++ + } + const parser = newParser(checkReply) + parser.execute(Buffer.from('+test')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('\r\n+')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from(bigString)) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r\n')) + assert.strictEqual(replyCount, 2) + }) - it('parser error resets the buffer', function () { - let errCount = 0 - function checkReply (reply) { - assert.strictEqual(reply.length, 1) - assert(Buffer.isBuffer(reply[0])) - assert.strictEqual(reply[0].toString(), 'CCC') - replyCount++ - } - function checkError (err) { - assert.strictEqual(err.message, 'Protocol error, got "b" as reply type byte') - errCount++ + it('handles multi-bulk reply and check context binding', function () { + function Abc () {} + Abc.prototype.checkReply = function (reply) { + assert.strictEqual(typeof this.log, 'function') + assert.deepStrictEqual(reply, [['a']], 'Expecting multi-bulk reply of [["a"]]') + replyCount++ + } + Abc.prototype.log = console.log + const test = new Abc() + const parser = newParser({ + returnReply: function (reply) { + test.checkReply(reply) } - const parser = new Parser({ - returnReply: checkReply, - returnError: checkError, - returnFatalError: checkError, - returnBuffers: true - }) - - // The chunk contains valid data after the protocol error - parser.execute(Buffer.from('*1\r\n+CCC\r\nb$1\r\nz\r\n+abc\r\n')) - assert.strictEqual(replyCount, 1) - assert.strictEqual(errCount, 1) - parser.execute(Buffer.from('*1\r\n+CCC\r\n')) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('-Protocol error, got "b" as reply type byte\r\n')) - assert.strictEqual(errCount, 2) }) - it('parser error v3 without returnFatalError specified', function () { - let errCount = 0 - function checkReply (reply) { - assert.strictEqual(reply[0], 'OK') - replyCount++ - } - function checkError (err) { - assert.strictEqual(err.message, 'Protocol error, got "\\n" as reply type byte') - errCount++ + parser.execute(Buffer.from('*1\r\n*1\r\n$1\r\na\r\n')) + assert.strictEqual(replyCount, 1) + + parser.execute(Buffer.from('*1\r\n*1\r')) + parser.execute(Buffer.from('\n$1\r\na\r\n')) + assert.strictEqual(replyCount, 2) + + parser.execute(Buffer.from('*1\r\n*1\r\n')) + parser.execute(Buffer.from('$1\r\na\r\n')) + + assert.strictEqual(replyCount, 3, 'check reply should have been called three times') + }) + + it('parser error', function () { + function Abc () {} + Abc.prototype.checkReply = function (err) { + assert.strictEqual(typeof this.log, 'function') + assert.strictEqual(err.message, 'Protocol error, got "a" as reply type byte') + assert.strictEqual(err.name, 'ParserError') + assert(err instanceof RedisError) + assert(err instanceof ParserError) + assert(err instanceof Error) + assert(err.offset) + assert(err.buffer) + // assert(/\[97,42,49,13,42,49,13,36,49,96,122,97,115,100,13,10,97]/.test(err.buffer)) + // assert(/ParserError: Protocol error, got "a" as reply type byte/.test(util.inspect(err))) + replyCount++ + } + Abc.prototype.log = console.log + const test = new Abc() + const parser = newParser({ + returnFatalError: function (err) { + test.checkReply(err) } - const parser = new Parser({ - returnReply: checkReply, - returnError: checkError - }) + }) - parser.execute(Buffer.from('*1\r\n+OK\r\n\n+zasd\r\n')) - assert.strictEqual(replyCount, 1) - assert.strictEqual(errCount, 1) + parser.execute(Buffer.from('a*1\r*1\r$1`zasd\r\na')) + assert.strictEqual(replyCount, 1) + }) + + it('parser error resets the buffer', function () { + let errCount = 0 + function checkReply (reply) { + assert.strictEqual(reply.length, 1) + assert(Buffer.isBuffer(reply[0])) + assert.strictEqual(reply[0].toString(), 'CCC') + replyCount++ + } + function checkError (err) { + assert.strictEqual(err.message, 'Protocol error, got "b" as reply type byte') + errCount++ + } + const parser = new Parser({ + returnReply: checkReply, + returnError: checkError, + returnFatalError: checkError, + pushReply, + returnBuffers: true }) - it('should handle \\r and \\n characters properly', function () { - // If a string contains \r or \n characters it will always be send as a bulk string - const entries = ['foo\r', 'foo\r\nbar', '\r\nСанкт-Пет', 'foo\r\n', 'foo', 'foobar', 'foo\r', 'äfooöü', 'abc'] - function checkReply (reply) { - assert.strictEqual(reply, entries[replyCount]) - replyCount++ - } - const parser = newParser(checkReply) + // The chunk contains valid data after the protocol error + parser.execute(Buffer.from('*1\r\n+CCC\r\nb$1\r\nz\r\n+abc\r\n')) + assert.strictEqual(replyCount, 1) + assert.strictEqual(errCount, 1) + parser.execute(Buffer.from('*1\r\n+CCC\r\n')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('-Protocol error, got "b" as reply type byte\r\n')) + assert.strictEqual(errCount, 2) + }) - parser.execute(Buffer.from('$4\r\nfoo\r\r\n$8\r\nfoo\r\nbar\r\n$19\r\n\r\n')) - parser.execute(Buffer.from([208, 161, 208, 176, 208, 189, 208])) - parser.execute(Buffer.from([186, 209, 130, 45, 208, 159, 208, 181, 209, 130])) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('\r\n$5\r\nfoo\r\n\r\n')) - assert.strictEqual(replyCount, 4) - parser.execute(Buffer.from('+foo\r')) - assert.strictEqual(replyCount, 4) - parser.execute(Buffer.from('\n$6\r\nfoobar\r')) - assert.strictEqual(replyCount, 5) - parser.execute(Buffer.from('\n$4\r\nfoo\r\r\n')) - assert.strictEqual(replyCount, 7) - parser.execute(Buffer.from('$9\r\näfo')) - parser.execute(Buffer.from('oö')) - parser.execute(Buffer.from('ü\r')) - assert.strictEqual(replyCount, 7) - parser.execute(Buffer.from('\n+abc\r\n')) - assert.strictEqual(replyCount, 9) + it('parser error v3 without returnFatalError specified', function () { + let errCount = 0 + function checkReply (reply) { + assert.strictEqual(reply[0], 'OK') + replyCount++ + } + function checkError (err) { + assert.strictEqual(err.message, 'Protocol error, got "\\n" as reply type byte') + errCount++ + } + const parser = new Parser({ + returnReply: checkReply, + returnError: checkError, + pushReply: pushReply }) - it('line breaks in the beginning of the last chunk', function () { - function checkReply (reply) { - assert.deepEqual(reply, [['a']], 'Expecting multi-bulk reply of [["a"]]') - replyCount++ - } - const parser = newParser(checkReply) + parser.execute(Buffer.from('*1\r\n+OK\r\n\n+zasd\r\n')) + assert.strictEqual(replyCount, 1) + assert.strictEqual(errCount, 1) + }) - parser.execute(Buffer.from('*1\r\n*1\r\n$1\r\na')) - assert.strictEqual(replyCount, 0) + it('should handle \\r and \\n characters properly', function () { + // If a string contains \r or \n characters it will always be send as a bulk string + const entries = ['foo\r', 'foo\r\nbar', '\r\nСанкт-Пет', 'foo\r\n', 'foo', 'foobar', 'foo\r', 'äfooöü', 'abc'] + function checkReply (reply) { + assert.strictEqual(reply, entries[replyCount]) + replyCount++ + } + const parser = newParser(checkReply) - parser.execute(Buffer.from('\r\n*1\r\n*1\r')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\n$1\r\na\r\n*1\r\n*1\r\n$1\r\na\r\n')) + parser.execute(Buffer.from('$4\r\nfoo\r\r\n$8\r\nfoo\r\nbar\r\n$19\r\n\r\n')) + parser.execute(Buffer.from([208, 161, 208, 176, 208, 189, 208])) + parser.execute(Buffer.from([186, 209, 130, 45, 208, 159, 208, 181, 209, 130])) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('\r\n$5\r\nfoo\r\n\r\n')) + assert.strictEqual(replyCount, 4) + parser.execute(Buffer.from('+foo\r')) + assert.strictEqual(replyCount, 4) + parser.execute(Buffer.from('\n$6\r\nfoobar\r')) + assert.strictEqual(replyCount, 5) + parser.execute(Buffer.from('\n$4\r\nfoo\r\r\n')) + assert.strictEqual(replyCount, 7) + parser.execute(Buffer.from('$9\r\näfo')) + parser.execute(Buffer.from('oö')) + parser.execute(Buffer.from('ü\r')) + assert.strictEqual(replyCount, 7) + parser.execute(Buffer.from('\n+abc\r\n')) + assert.strictEqual(replyCount, 9) + }) - assert.strictEqual(replyCount, 3, 'check reply should have been called three times') - }) + it('line breaks in the beginning of the last chunk', function () { + function checkReply (reply) { + assert.deepStrictEqual(reply, [['a']], 'Expecting multi-bulk reply of [["a"]]') + replyCount++ + } + const parser = newParser(checkReply) - it('multiple chunks in a bulk string', function () { - function checkReply (reply) { - assert.strictEqual(reply, 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij') - replyCount++ - } - const parser = newParser(checkReply) + parser.execute(Buffer.from('*1\r\n*1\r\n$1\r\na')) + assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('$100\r\nabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('\r\n')) - assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r\n*1\r\n*1\r')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\n$1\r\na\r\n*1\r\n*1\r\n$1\r\na\r\n')) - parser.execute(Buffer.from('$100\r')) - parser.execute(Buffer.from('\nabcdefghijabcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghij')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from( - 'abcdefghij\r\n' + - '$100\r\nabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij\r\n' + - '$100\r\nabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij' - )) - assert.strictEqual(replyCount, 3) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij\r')) - assert.strictEqual(replyCount, 3) - parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 3, 'check reply should have been called three times') + }) - assert.strictEqual(replyCount, 4, 'check reply should have been called three times') - }) + it('multiple chunks in a bulk string', function () { + function checkReply (reply) { + assert.strictEqual(reply, 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij') + replyCount++ + } + const parser = newParser(checkReply) + + parser.execute(Buffer.from('$100\r\nabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('\r\n')) + assert.strictEqual(replyCount, 1) - it('multiple chunks with arrays different types', function () { - const predefinedData = [ - 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij', - 'test', - 100, - new ReplyError('Error message'), - ['The force awakens'], - new ReplyError() - ] - function checkReply (reply) { - for (let i = 0; i < reply.length; i++) { - if (Array.isArray(reply[i])) { - reply[i].forEach(function (reply, j) { - assert.strictEqual(reply, predefinedData[i][j]) - }) - } else if (reply[i] instanceof Error) { - if (Parser.name !== 'HiredisReplyParser') { // The hiredis always returns normal errors in case of nested ones - assert(reply[i] instanceof ReplyError) - assert.strictEqual(reply[i].name, predefinedData[i].name) - } - assert.strictEqual(reply[i].message, predefinedData[i].message) - } else { - assert.strictEqual(reply[i], predefinedData[i]) - } - } - replyCount++ - } - const parser = newParser({ - returnReply: checkReply, - returnBuffers: false - }) + parser.execute(Buffer.from('$100\r')) + parser.execute(Buffer.from('\nabcdefghijabcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghij')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from( + 'abcdefghij\r\n' + + '$100\r\nabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij\r\n' + + '$100\r\nabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij' + )) + assert.strictEqual(replyCount, 3) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij\r')) + assert.strictEqual(replyCount, 3) + parser.execute(Buffer.from('\n')) - parser.execute(Buffer.from('*6\r\n$100\r\nabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) - parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij\r\n')) - parser.execute(Buffer.from('+test\r')) - parser.execute(Buffer.from('\n:100')) - parser.execute(Buffer.from('\r\n-Error message')) - parser.execute(Buffer.from('\r\n*1\r\n$17\r\nThe force')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from(' awakens\r\n-\r\n$5')) - assert.strictEqual(replyCount, 1) - }) + assert.strictEqual(replyCount, 4, 'check reply should have been called three times') + }) - it('multiple chunks with nested partial arrays', function () { - const predefinedData = [ - 'abcdefghijabcdefghij', - 100, - '1234567890', - 100 - ] - function checkReply (reply) { - assert.strictEqual(reply.length, 1) - for (let i = 0; i < reply[0].length; i++) { - assert.strictEqual(reply[0][i], predefinedData[i]) + it('multiple chunks with arrays different types', function () { + const predefinedData = [ + 'abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij', + 'test', + 100, + new ReplyError('Error message'), + ['The force awakens'], + new ReplyError() + ] + function checkReply (reply) { + for (let i = 0; i < reply.length; i++) { + if (Array.isArray(reply[i])) { + reply[i].forEach(function (reply, j) { + assert.strictEqual(reply, predefinedData[i][j]) + }) + } else if (reply[i] instanceof Error) { + assert(reply[i] instanceof ReplyError) + assert.strictEqual(reply[i].name, predefinedData[i].name) + assert.strictEqual(reply[i].message, predefinedData[i].message) + } else { + assert.strictEqual(reply[i], predefinedData[i]) } - replyCount++ } - const parser = newParser({ - returnReply: checkReply - }) - parser.execute(Buffer.from('*1\r\n*4\r\n+abcdefghijabcdefghij\r\n:100')) - parser.execute(Buffer.from('\r\n$10\r\n1234567890\r\n:100')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('\r\n')) - assert.strictEqual(replyCount, 1) - }) + replyCount++ + } + const parser = newParser({ + returnReply: checkReply, + returnBuffers: false + }) + + parser.execute(Buffer.from('*6\r\n$100\r\nabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij')) + parser.execute(Buffer.from('abcdefghijabcdefghijabcdefghij\r\n')) + parser.execute(Buffer.from('+test\r')) + parser.execute(Buffer.from('\n:100')) + parser.execute(Buffer.from('\r\n-Error message')) + parser.execute(Buffer.from('\r\n*1\r\n$17\r\nThe force')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from(' awakens\r\n-\r\n$5')) + assert.strictEqual(replyCount, 1) + }) - it('return normal errors', function () { - function checkReply (reply) { - assert.strictEqual(reply.message, 'Error message') - replyCount++ + it('multiple chunks with nested partial arrays', function () { + const predefinedData = [ + 'abcdefghijabcdefghij', + 100, + '1234567890', + 100 + ] + function checkReply (reply) { + assert.strictEqual(reply.length, 1) + for (let i = 0; i < reply[0].length; i++) { + assert.strictEqual(reply[0][i], predefinedData[i]) } - const parser = newParser({ - returnError: checkReply - }) + replyCount++ + } + const parser = newParser({ + returnReply: checkReply + }) + parser.execute(Buffer.from('*1\r\n*4\r\n+abcdefghijabcdefghij\r\n:100')) + parser.execute(Buffer.from('\r\n$10\r\n1234567890\r\n:100')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('\r\n')) + assert.strictEqual(replyCount, 1) + }) - parser.execute(Buffer.from('-Error ')) - parser.execute(Buffer.from('message\r\n*3\r\n$17\r\nThe force')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from(' awakens\r\n$5')) - assert.strictEqual(replyCount, 1) + it('return normal errors', function () { + function checkReply (reply) { + assert.strictEqual(reply.message, 'Error message') + replyCount++ + } + const parser = newParser({ + returnError: checkReply }) - it('return null for empty arrays and empty bulk strings', function () { - function checkReply (reply) { - assert.strictEqual(reply, null) - replyCount++ - } - const parser = newParser(checkReply) + parser.execute(Buffer.from('-Error ')) + parser.execute(Buffer.from('message\r\n*3\r\n$17\r\nThe force')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from(' awakens\r\n$5')) + assert.strictEqual(replyCount, 1) + }) - parser.execute(Buffer.from('$-1\r\n*-')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('1')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\r\n$-')) - assert.strictEqual(replyCount, 2) - }) + it('return null', function () { + function checkReply (reply) { + assert.strictEqual(reply, null) + replyCount++ + } + const parser = newParser(checkReply) - it('return value even if all chunks are only 1 character long', function () { - function checkReply (reply) { - assert.strictEqual(reply, 1) - replyCount++ - } - const parser = newParser(checkReply) + parser.execute(Buffer.from('_\r\n_')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r\n$1')) + assert.strictEqual(replyCount, 2) + }) - parser.execute(Buffer.from(':')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('1')) - parser.execute(Buffer.from('\r')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('\n')) - assert.strictEqual(replyCount, 1) - }) + it('return value even if all chunks are only 1 character long', function () { + function checkReply (reply) { + assert.strictEqual(reply, 1) + replyCount++ + } + const parser = newParser(checkReply) + + parser.execute(Buffer.from(':')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('1')) + parser.execute(Buffer.from('\r')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 1) + }) - it('do not return before \\r\\n', function () { - function checkReply (reply) { - assert.strictEqual(reply, 1) - replyCount++ - } - const parser = newParser(checkReply) + it('do not return before \\r\\n', function () { + function checkReply (reply) { + assert.strictEqual(reply, 1) + replyCount++ + } + const parser = newParser(checkReply) - parser.execute(Buffer.from(':1\r\n:')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('1')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\r')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\n')) - assert.strictEqual(replyCount, 2) - }) + parser.execute(Buffer.from(':1\r\n:')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('1')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 2) + }) - it('return data as buffer if requested', function () { - function checkReply (reply) { - if (Array.isArray(reply)) { - reply = reply[0] - } - assert(Buffer.isBuffer(reply)) - assert.strictEqual(reply.inspect(), Buffer.from('test').inspect()) - replyCount++ + it('return data as buffer if requested', function () { + function checkReply (reply) { + if (Array.isArray(reply)) { + reply = reply[0] } - const parser = newParser(checkReply, 'buffer') + assert(Buffer.isBuffer(reply)) + assert.strictEqual(reply.inspect(), Buffer.from('test').inspect()) + replyCount++ + } + const parser = newParser(checkReply, 'buffer') - parser.execute(Buffer.from('+test\r\n')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('$4\r\ntest\r')) - parser.execute(Buffer.from('\n')) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('*1\r\n$4\r\nte')) - parser.execute(Buffer.from('st\r')) - parser.execute(Buffer.from('\n')) - assert.strictEqual(replyCount, 3) - }) + parser.execute(Buffer.from('+test\r\n')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('$4\r\ntest\r')) + parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('*1\r\n$4\r\nte')) + parser.execute(Buffer.from('st\r')) + parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 3) + }) - it('handle special case buffer sizes properly', function () { - const entries = ['test test ', 'test test test test ', 1234] - function checkReply (reply) { - assert.strictEqual(reply, entries[replyCount]) - replyCount++ - } - const parser = newParser(checkReply) - parser.execute(Buffer.from('$10\r\ntest ')) - assert.strictEqual(replyCount, 0) - parser.execute(Buffer.from('test \r\n$20\r\ntest test test test \r\n:1234\r')) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('\n')) - assert.strictEqual(replyCount, 3) - }) + it('handle special case buffer sizes properly', function () { + const entries = ['test test ', 'test test test test ', 1234] + function checkReply (reply) { + assert.strictEqual(reply, entries[replyCount]) + replyCount++ + } + const parser = newParser(checkReply) + parser.execute(Buffer.from('$10\r\ntest ')) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('test \r\n$20\r\ntest test test test \r\n:1234\r')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('\n')) + assert.strictEqual(replyCount, 3) + }) - it('return numbers as strings', function () { - if (Parser.name === 'HiredisReplyParser') { - return this.skip() - } - const entries = ['123', '590295810358705700002', '-99999999999999999', '4294967290', '90071992547409920', '10000040000000000000000000000000000000020'] - function checkReply (reply) { - assert.strictEqual(typeof reply, 'string') - assert.strictEqual(reply, entries[replyCount]) - replyCount++ - } - const parser = newParser({ - returnReply: checkReply, - stringNumbers: true - }) - parser.execute(Buffer.from(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:4294967290\r\n:90071992547409920\r\n:10000040000000000000000000000000000000020\r\n')) - assert.strictEqual(replyCount, 6) + it('return numbers as strings', function () { + const entries = ['123', '590295810358705700002', '-99999999999999999', '4294967290', '90071992547409920', '10000040000000000000000000000000000000020'] + function checkReply (reply) { + assert.strictEqual(typeof reply, 'string') + assert.strictEqual(reply, entries[replyCount]) + replyCount++ + } + const parser = newParser({ + returnReply: checkReply, + stringNumbers: true }) + parser.execute(Buffer.from(':123\r\n:590295810358705700002\r\n:-99999999999999999\r\n:4294967290\r\n:90071992547409920\r\n:10000040000000000000000000000000000000020\r\n')) + assert.strictEqual(replyCount, 6) + }) - it('handle big numbers', function () { - let number = 9007199254740991 // Number.MAX_SAFE_INTEGER - function checkReply (reply) { - assert.strictEqual(reply, number++) - replyCount++ - } - const parser = newParser(checkReply) - parser.execute(Buffer.from(':' + number + '\r\n')) - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from(':' + number + '\r\n')) - assert.strictEqual(replyCount, 2) - }) + it('handle big numbers', function () { + let number = 9007199254740991 // Number.MAX_SAFE_INTEGER + function checkReply (reply) { + assert.strictEqual(reply, number++) + replyCount++ + } + const parser = newParser(checkReply) + parser.execute(Buffer.from(':' + number + '\r\n')) + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from(':' + number + '\r\n')) + assert.strictEqual(replyCount, 2) + }) - it('handle big data with buffers', function (done) { - let chunks - const replies = [] - const jsParser = Parser.name === 'JavascriptRedisParser' - function checkReply (reply) { - replies.push(reply) - replyCount++ - } - const parser = newParser(checkReply, 'buffer') + it('handle big data with buffers', function (done) { + let chunks + const replies = [] + const jsParser = Parser.name === 'JavascriptRedisParser' + function checkReply (reply) { + replies.push(reply) + replyCount++ + } + const parser = newParser(checkReply, 'buffer') + parser.execute(Buffer.from('+test')) + assert.strictEqual(replyCount, 0) + createBufferOfSize(parser, 128 * 1024, '\r\n') + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r\n')) + assert.strictEqual(replyCount, 2) + setTimeout(function () { parser.execute(Buffer.from('+test')) - assert.strictEqual(replyCount, 0) - createBufferOfSize(parser, 128 * 1024, '\r\n') - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\r\n')) assert.strictEqual(replyCount, 2) - setTimeout(function () { - parser.execute(Buffer.from('+test')) - assert.strictEqual(replyCount, 2) - chunks = createBufferOfSize(parser, 256 * 1024, '\r\n') - assert.strictEqual(replyCount, 3) - parser.execute(Buffer.from('\r\n')) - assert.strictEqual(replyCount, 4) - }, 20) - // Delay done so the bufferPool is cleared and tested - // If the buffer is not cleared, the coverage is not going to be at 100 - setTimeout(function () { - const totalBuffer = Buffer.concat(chunks).toString() - assert.strictEqual(replies[3].toString(), totalBuffer) - done() - }, (jsParser ? 1400 : 40)) - }) - - it('handle big data', function () { - function checkReply (reply) { - assert.strictEqual(reply.length, 4 * 1024 * 1024) - replyCount++ - } - const parser = newParser(checkReply) - createBufferOfSize(parser, 4 * 1024 * 1024) - assert.strictEqual(replyCount, 0) + chunks = createBufferOfSize(parser, 256 * 1024, '\r\n') + assert.strictEqual(replyCount, 3) parser.execute(Buffer.from('\r\n')) - assert.strictEqual(replyCount, 1) - }) + assert.strictEqual(replyCount, 4) + }, 20) + // Delay done so the bufferPool is cleared and tested + // If the buffer is not cleared, the coverage is not going to be at 100 + setTimeout(function () { + const totalBuffer = Buffer.concat(chunks).toString() + assert.strictEqual(replies[3].toString(), totalBuffer) + done() + }, (jsParser ? 1400 : 40)) + }) - it('handle big data 2 with buffers', function (done) { - this.timeout(7500) - const size = 111.5 * 1024 * 1024 - const replyLen = [size, size * 2, 11, 11] - function checkReply (reply) { - assert.strictEqual(reply.length, replyLen[replyCount]) - replyCount++ - } - const parser = newParser(checkReply, 'buffer') - createBufferOfSize(parser, size) - assert.strictEqual(replyCount, 0) - createBufferOfSize(parser, size * 2, '\r\n') - assert.strictEqual(replyCount, 1) - parser.execute(Buffer.from('\r\n+hello world')) - assert.strictEqual(replyCount, 2) - parser.execute(Buffer.from('\r\n$11\r\nhuge')) - setTimeout(function () { - parser.execute(Buffer.from(' buffer\r\n')) - assert.strictEqual(replyCount, 4) - done() - }, 60) - }) + it('handle big data', function () { + function checkReply (reply) { + assert.strictEqual(reply.length, 4 * 1024 * 1024) + replyCount++ + } + const parser = newParser(checkReply) + createBufferOfSize(parser, 4 * 1024 * 1024) + assert.strictEqual(replyCount, 0) + parser.execute(Buffer.from('\r\n')) + assert.strictEqual(replyCount, 1) + }) + + it('handle big data 2 with buffers', function (done) { + this.timeout(7500) + const size = 111.5 * 1024 * 1024 + const replyLen = [size, size * 2, 11, 11] + function checkReply (reply) { + assert.strictEqual(reply.length, replyLen[replyCount]) + replyCount++ + } + const parser = newParser(checkReply, 'buffer') + createBufferOfSize(parser, size) + assert.strictEqual(replyCount, 0) + createBufferOfSize(parser, size * 2, '\r\n') + assert.strictEqual(replyCount, 1) + parser.execute(Buffer.from('\r\n+hello world')) + assert.strictEqual(replyCount, 2) + parser.execute(Buffer.from('\r\n$11\r\nhuge')) + setTimeout(function () { + parser.execute(Buffer.from(' buffer\r\n')) + assert.strictEqual(replyCount, 4) + done() + }, 60) }) }) })