feat: setInternalBufferSize

lib/index.js

@@ -75,6 +75,8 @@ module.exports.ObjectID = BSONJS.ObjectId;
 
 /** @type {import('bson').calculateObjectSize} */
 module.exports.calculateObjectSize = BSON.calculateObjectSize.bind(BSON);
+/** @type {import('bson').setInternalBufferSize} */
+module.exports.setInternalBufferSize = BSON.setInternalBufferSize.bind(BSON);
 /** @type {import('bson').serialize} */
 module.exports.serialize = BSON.serialize.bind(BSON);
 /** @type {import('bson').serializeWithBufferAndIndex} */

src/bson.cc

@@ -1277,6 +1277,7 @@ void BSON::Initialize(v8::Local<v8::Object> target) {
 
   // Instance methods
   Nan::SetPrototypeMethod(t, "calculateObjectSize", CalculateObjectSize);
+  Nan::SetPrototypeMethod(t, "setInternalBufferSize", SetInternalBufferSize);
   Nan::SetPrototypeMethod(t, "serialize", BSONSerialize);
   Nan::SetPrototypeMethod(t, "serializeWithBufferAndIndex",
                           SerializeWithBufferAndIndex);
@@ -1682,7 +1683,7 @@ NAN_METHOD(BSON::BSONSerialize) {
 
     // Check if we have a boolean value
     BSONSerializer<DataStream> data(bson, checkKeys, serializeFunctions,
-                                    ignoreUndefined, serialized_object);
+                                    ignoreUndefined, serialized_object, bson->maxBSONSize);
     data.SerializeDocument(object);
 
     // Get the object size
@@ -1762,6 +1763,19 @@ NAN_METHOD(BSON::CalculateObjectSize) {
   info.GetReturnValue().Set(
       Nan::New<Uint32>((uint32_t)countSerializer.GetSerializeSize()));
 }
+NAN_METHOD(BSON::SetInternalBufferSize) {
+  Nan::HandleScope scope;
+  if (info.Length() != 1 && !info[0]->IsNumber()) {
+    return Nan::ThrowError("First argument must be a number");
+  }
+  BSON *bson = ObjectWrap::Unwrap<BSON>(info.This());
+  double new_size = NanTo<double>(info[0]);
+
+  if (bson->maxBSONSize < new_size) {
+    bson->maxBSONSize = new_size;
+    bson->buffer.Reset(Unmaybe(Nan::NewBuffer(sizeof(char) * new_size)));
+  }
+}
 
 NAN_METHOD(BSON::SerializeWithBufferAndIndex) {
   Nan::HandleScope scope;
@@ -1833,7 +1847,7 @@ NAN_METHOD(BSON::SerializeWithBufferAndIndex) {
     size_t length = node::Buffer::Length(obj);
 
     BSONSerializer<DataStream> dataSerializer(
-        bson, checkKeys, serializeFunctions, ignoreUndefined, data + index);
+        bson, checkKeys, serializeFunctions, ignoreUndefined, data + index, bson->maxBSONSize);
     dataSerializer.SerializeDocument(bson->GetSerializeObject(info[0]));
     object_size = dataSerializer.GetSerializeSize();
 

src/bson.h

@@ -141,6 +141,7 @@ class BSON : public Nan::ObjectWrap {
 
   // Calculate size of function
   static NAN_METHOD(CalculateObjectSize);
+  static NAN_METHOD(SetInternalBufferSize);
   static NAN_METHOD(SerializeWithBufferAndIndex);
 
   // Constructor used for creating new BSON objects from C++
@@ -287,24 +288,27 @@ const size_t MAX_BSON_SIZE(1024 * 1024 * 17);
 class DataStream {
 public:
   DataStream(char *aDestinationBuffer)
-      : destinationBuffer(aDestinationBuffer), p(aDestinationBuffer), depth(0) {
+      : destinationBuffer(aDestinationBuffer), p(aDestinationBuffer), depth(0), maxBSONSize(MAX_BSON_SIZE) {
+  }
+  DataStream(char *aDestinationBuffer, size_t aMaxBSONSize)
+      : destinationBuffer(aDestinationBuffer), p(aDestinationBuffer), depth(0), maxBSONSize(aMaxBSONSize) {
   }
 
   void WriteByte(int value) {
-    if ((size_t)((p + 1) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 1) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     *p++ = value;
   }
 
   void WriteByte(const Local<Object> &object, const Local<String> &key) {
-    if ((size_t)((p + 1) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 1) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     *p++ = NanTo<int32_t>(NanGet(object, key));
   }
 
 #if USE_MISALIGNED_MEMORY_ACCESS
   void WriteInt32(int32_t value) {
-    if ((size_t)((p + 4) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 4) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
 #if defined(_MSC_VER)
     *reinterpret_cast<int32_t *>(p) =
@@ -317,7 +321,7 @@ class DataStream {
   }
 
   void WriteInt64(int64_t value) {
-    if ((size_t)((p + 8) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 8) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
 #if defined(_MSC_VER)
     *reinterpret_cast<int64_t *>(p) =
@@ -330,7 +334,7 @@ class DataStream {
   }
 
   void WriteDouble(double value) {
-    if ((size_t)((p + 8) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 8) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     *reinterpret_cast<double *>(p) = value;
     if (is_bigendian()) {
@@ -347,21 +351,21 @@ class DataStream {
   }
 #else
   void WriteInt32(int32_t value) {
-    if ((size_t)((p + 4) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 4) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     memcpy(p, &value, 4);
     p += 4;
   }
 
   void WriteInt64(int64_t value) {
-    if ((size_t)((p + 8) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 8) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     memcpy(p, &value, 8);
     p += 8;
   }
 
   void WriteDouble(double value) {
-    if ((size_t)((p + 8) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + 8) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     memcpy(p, &value, 8);
     p += 8;
@@ -395,7 +399,7 @@ class DataStream {
 
   void WriteLengthPrefixedString(const Local<String> &value) {
     int32_t length = NanUtf8Length(value) + 1;
-    if ((size_t)((p + length) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + length) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     WriteInt32(length);
     WriteString(value);
@@ -404,13 +408,13 @@ class DataStream {
   void WriteObjectId(const Local<Object> &object, const Local<String> &key);
 
   void WriteString(const Local<String> &value) {
-    if ((size_t)(p - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)(p - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     p += NanWriteUtf8(value, p);
   } // This returns the number of bytes inclusive of the NULL terminator.
 
   void WriteData(const char *data, size_t length) {
-    if ((size_t)((p + length) - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)((p + length) - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     memcpy(p, data, length);
     p += length;
@@ -427,7 +431,7 @@ class DataStream {
   }
 
   void *BeginWriteSize() {
-    if ((size_t)(p - destinationBuffer) > MAX_BSON_SIZE)
+    if ((size_t)(p - destinationBuffer) > maxBSONSize)
       throw "document is larger than max bson document size of 16MB";
     void *returnValue = p;
     p += 4;
@@ -469,6 +473,7 @@ class DataStream {
   char *const destinationBuffer; // base, never changes
   char *p;                       // cursor into buffer
   uint32_t depth;                // keeps track of recursive depth
+  uint32_t maxBSONSize;
 };
 
 template <typename T> class BSONSerializer : public T {
@@ -486,6 +491,11 @@ template <typename T> class BSONSerializer : public T {
       : Inherited(parentParam), checkKeys(aCheckKeys),
         serializeFunctions(aSerializeFunctions),
         ignoreUndefined(ignoreUndefined), bson(aBson) {}
+  BSONSerializer(BSON *aBson, bool aCheckKeys, bool aSerializeFunctions,
+                 bool ignoreUndefined, char *parentParam, size_t maxBSONSize)
+      : Inherited(parentParam, maxBSONSize), checkKeys(aCheckKeys),
+        serializeFunctions(aSerializeFunctions),
+        ignoreUndefined(ignoreUndefined), bson(aBson) {}
 
   void SerializeDocument(const Local<Value> &value);
   void SerializeArray(const Local<Value> &value);

test/node/bson_test_large.js

@@ -0,0 +1,117 @@
+'use strict';
+
+var Buffer = require('buffer').Buffer,
+  fs = require('fs'),
+  expect = require('chai').expect,
+  BSON = require('../..'),
+  Code = BSON.Code,
+  BSONRegExp = BSON.BSONRegExp,
+  Binary = BSON.Binary,
+  Timestamp = BSON.Timestamp,
+  Long = BSON.Long,
+  ObjectID = BSON.ObjectID,
+  ObjectId = BSON.ObjectID,
+  Symbol = BSON.Symbol,
+  DBRef = BSON.DBRef,
+  Decimal128 = BSON.Decimal128,
+  Int32 = BSON.Int32,
+  Double = BSON.Double,
+  MinKey = BSON.MinKey,
+  MaxKey = BSON.MaxKey,
+  BinaryParser = require('../binary_parser').BinaryParser,
+  vm = require('vm');
+
+var createBSON = require('../utils');
+
+var assertBuffersEqual = function(done, buffer1, buffer2) {
+  if (buffer1.length !== buffer2.length) {
+    done('Buffers do not have the same length', buffer1, buffer2);
+  }
+
+  for (var i = 0; i < buffer1.length; i++) {
+    expect(buffer1[i]).to.equal(buffer2[i]);
+  }
+};
+
+/**
+ * Module for parsing an ISO 8601 formatted string into a Date object.
+ */
+var ISODate = function(string) {
+  var match;
+
+  if (typeof string.getTime === 'function') return string;
+  else if (
+    (match = string.match(
+      /^(\d{4})(-(\d{2})(-(\d{2})(T(\d{2}):(\d{2})(:(\d{2})(\.(\d+))?)?(Z|((\+|-)(\d{2}):(\d{2}))))?)?)?$/
+    ))
+  ) {
+    var date = new Date();
+    date.setUTCFullYear(Number(match[1]));
+    date.setUTCMonth(Number(match[3]) - 1 || 0);
+    date.setUTCDate(Number(match[5]) || 0);
+    date.setUTCHours(Number(match[7]) || 0);
+    date.setUTCMinutes(Number(match[8]) || 0);
+    date.setUTCSeconds(Number(match[10]) || 0);
+    date.setUTCMilliseconds(Number('.' + match[12]) * 1000 || 0);
+
+    if (match[13] && match[13] !== 'Z') {
+      var h = Number(match[16]) || 0,
+        m = Number(match[17]) || 0;
+
+      h *= 3600000;
+      m *= 60000;
+
+      var offset = h + m;
+      if (match[15] === '+') offset = -offset;
+
+      date = new Date(date.valueOf() + offset);
+    }
+
+    return date;
+  } else throw new Error('Invalid ISO 8601 date given.', __filename);
+};
+
+describe('BSON', function() {
+  /**
+   * @ignore
+   */
+  it('Should correctly serialize a given javascript object using a BSON instance with more than 16MB', function(done) {
+    var expectedSize = 20*1024*1024;
+    // Create a simple object
+    var doc = { large: Buffer.alloc(expectedSize) };
+    // Create a BSON parser instance
+    var bson = createBSON();
+    bson.setInternalBufferSize(expectedSize+17);
+    // Serialize the object to the buffer, checking keys and not serializing functions
+    var buffer = bson.serialize(doc);
+    // check
+    expect(buffer.length).to.equal(expectedSize+17);
+
+    done();
+  });
+
+  /**
+   * @ignore
+   */
+  it('Should correctly serializeWithBufferAndIndex a given javascript object using a BSON instance with more than 16MB', function(
+    done
+  ) {
+    var expectedSize = 20*1024*1024;
+    // Create a simple object
+    var doc = { large: Buffer.alloc(expectedSize) };
+    // Create a BSON parser instance
+    var bson = createBSON();
+    // Calculate the size of the document, no function serialization
+    var size = bson.calculateObjectSize(doc);
+    // Allocate a buffer
+    var buffer = new Buffer(size);
+    bson.setInternalBufferSize(size);
+    // Serialize the object to the buffer, checking keys and not serializing functions
+    var index = bson.serializeWithBufferAndIndex(doc, buffer);
+    // check
+    expect(size).to.equal(expectedSize+17);
+    expect(index).to.equal(expectedSize+16);
+
+    done();
+  });
+});