From 775f6f24bbd475aa9a9d95a475cb26d7391c5576 Mon Sep 17 00:00:00 2001 From: avaer Date: Sat, 10 Nov 2018 18:39:05 -0500 Subject: [PATCH 01/31] Clean up HTMLAudioElement native audio binding guards --- src/DOM.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/DOM.js b/src/DOM.js index f6b692f257..8e1bc33c9d 100644 --- a/src/DOM.js +++ b/src/DOM.js @@ -2396,25 +2396,21 @@ class HTMLAudioElement extends HTMLMediaElement { } get paused() { - return this.audio ? this.audio.paused : true; + return this.audio.paused; } get currentTime() { - return this.audio && this.audio.currentTime; + return this.audio.currentTime; } set currentTime(currentTime) { - if (this.audio) { - this.audio.currentTime = currentTime; - } + this.audio.currentTime = currentTime; } get duration() { return this.audio && this.audio.duration; } set duration(duration) { - if (this.audio) { - this.audio.duration = duration; - } + this.audio.duration = duration; } get buffered() { From 2433fd776313fbad99f095ab52334d4509080f38 Mon Sep 17 00:00:00 2001 From: avaer Date: Sat, 10 Nov 2018 18:39:57 -0500 Subject: [PATCH 02/31] Merge DOM.js HTMLVideoElement video and webcam implementations --- src/DOM.js | 248 ++++++++++++++++++++--------------------------------- 1 file changed, 95 insertions(+), 153 deletions(-) diff --git a/src/DOM.js b/src/DOM.js index 8e1bc33c9d..bcdcbd43fb 100644 --- a/src/DOM.js +++ b/src/DOM.js @@ -2432,119 +2432,6 @@ module.exports.HTMLAudioElement = HTMLAudioElement; class HTMLVideoElement extends HTMLMediaElement { constructor(attrs = [], value = '', location = null) { - super('VIDEO', attrs, value, location); - - this.readyState = HTMLMediaElement.HAVE_NOTHING; - this.data = new Uint8Array(0); - - this.on('attribute', (name, value) => { - if (name === 'src' && value) { - this.readyState = 'loading'; - - const src = value; - - this.readyState = HTMLMediaElement.HAVE_ENOUGH_DATA; - - if (urls.has(value)) { - const blob = urls.get(value); - if (blob instanceof Bindings.bindings.nativeVideo.VideoDevice) { - this.video = blob; - } - } - - this.ownerDocument.resources.addResource((onprogress, cb) => { - const progressEvent = new Event('progress', {target: this}); - progressEvent.loaded = 1; - progressEvent.total = 1; - progressEvent.lengthComputable = true; - this._emit(progressEvent); - - this.readyState = 'complete'; - - this._dispatchEventOnDocumentReady(new Event('canplay', {target: this})); - this._dispatchEventOnDocumentReady(new Event('canplaythrough', {target: this})); - - cb(); - }); - } - }); - } - - get width() { - return this.video ? this.video.width : 0; - } - set width(width) {} - get height() { - return this.video ? this.video.height : 0; - } - set height(height) {} - - get autoplay() { - return this.getAttribute('autoplay'); - } - set autoplay(autoplay) { - this.setAttribute('autoplay', autoplay); - } - - getBoundingClientRect() { - return new DOMRect(0, 0, this.width, this.height); - } - - get data() { - return this.video ? this.video.data : null; - } - set data(data) {} - - play() { - const _getDevice = facingMode => { - switch (facingMode) { - case 'user': return devices[0]; - case 'environment': return devices[1]; - case 'left': return devices[2]; - case 'right': return devices[3]; - default: return devices[0]; - } - } - const _getName = facingMode => (process.platform === 'darwin' ? '' : 'video=') + _getDevice(facingMode).name; - const _getOptions = facingMode => { - if (process.platform === 'darwin') { - return 'framerate='+_getDevice(facingMode).modes[0].fps; - } else { - return null; - } - } - if (this.video) { - this.video.close(); - this.video.open( - _getName(this.video.constraints.facingMode), - _getOptions(this.video.constraints.facingMode) - ); - } - - return Promise.resolve(); - } - pause() { - if (this.video) { - this.video.close(); - } - } - - get buffered() { - return new TimeRanges([0, this.duration]); - } - set buffered(buffered) {} - - update() { - if (this.video) { - this.video.update(); - } - } -} -module.exports.HTMLVideoElement = HTMLVideoElement; - -/* -class HTMLVideoElement extends HTMLMediaElement { - constructor(attrs = [], value = '') { super('VIDEO', attrs, value); this.readyState = HTMLMediaElement.HAVE_NOTHING; @@ -2552,43 +2439,64 @@ class HTMLVideoElement extends HTMLMediaElement { this.on('attribute', (name, value) => { if (name === 'src' && value) { - console.log('video downloading...'); const src = value; - this.ownerDocument.defaultView.fetch(src) - .then(res => { - console.log('video download res'); - if (res.status >= 200 && res.status < 300) { - return res.arrayBuffer(); + this.ownerDocument.resources.addResource((onprogress, cb) => { + (() => { + const b = urls.get(src); + if (b instanceof bindings.nativeVideo.VideoDevice) { + this.video = b; + + return Promise.resolve(); } else { - return Promise.reject(new Error(`video src got invalid status code (url: ${JSON.stringify(src)}, code: ${res.status})`)); - } - }) - .then(arrayBuffer => { - console.log('video download arraybuffer'); - try { - this.video.load(arrayBuffer); - } catch(err) { - throw new Error(`failed to decode video: ${err.message} (url: ${JSON.stringify(src)}, size: ${arrayBuffer.byteLength})`); + return this.ownerDocument.defaultView.fetch(src) + .then(res => { + if (res.status >= 200 && res.status < 300) { + return res.arrayBuffer(); + } else { + return Promise.reject(new Error(`video src got invalid status code (url: ${JSON.stringify(src)}, code: ${res.status})`)); + } + }) + .then(arrayBuffer => { + try { + this.video.load(arrayBuffer); + } catch(err) { + throw new Error(`failed to decode video: ${err.message} (url: ${JSON.stringify(src)}, size: ${arrayBuffer.byteLength})`); + } + }) } - }) - .then(() => { - console.log('video download done'); - this.readyState = HTMLMediaElement.HAVE_ENOUGH_DATA; - this._emit('canplay'); - this._emit('canplaythrough'); - }) - .catch(err => { - this._emit('error', err); - }); + })() + .then(() => { + this.readyState = HTMLMediaElement.HAVE_ENOUGH_DATA; + + const progressEvent = new Event('progress', {target: this}); + progressEvent.loaded = 1; + progressEvent.total = 1; + progressEvent.lengthComputable = true; + this._emit(progressEvent); + + this._dispatchEventOnDocumentReady(new Event('canplay', {target: this})); + this._dispatchEventOnDocumentReady(new Event('canplaythrough', {target: this})); + + cb(); + }) + .catch(err => { + console.warn('failed to load audio:', src); + + const e = new ErrorEvent('error', {target: this}); + e.message = err.message; + e.stack = err.stack; + this._dispatchEventOnDocumentReady(e); + + cb(err); + }); + }); } else if (name === 'loop') { this.video.loop = !!value || value === ''; } else if (name === 'autoplay') { const autoplay = !!value || value === ''; if (autoplay) { - console.log('video set autoplay'); const canplay = () => { - console.log('video autoplay play'); this.video.play(); _cleanup(); }; @@ -2605,7 +2513,7 @@ class HTMLVideoElement extends HTMLMediaElement { } }); } - + get width() { return this.video.width; } @@ -2639,31 +2547,61 @@ class HTMLVideoElement extends HTMLMediaElement { set data(data) {} play() { - this.video.play(); + if (this.video instanceof bindings.nativeVideo.VideoDevice) { + const _getDevice = facingMode => { + switch (facingMode) { + case 'user': return devices[0]; + case 'environment': return devices[1]; + case 'left': return devices[2]; + case 'right': return devices[3]; + default: return devices[0]; + } + } + const _getName = facingMode => (process.platform === 'darwin' ? '' : 'video=') + _getDevice(facingMode).name; + const _getOptions = facingMode => { + if (process.platform === 'darwin') { + return 'framerate='+_getDevice(facingMode).modes[0].fps; + } else { + return null; + } + } + if (this.video) { + this.video.close(); + this.video.open( + _getName(this.video.constraints.facingMode), + _getOptions(this.video.constraints.facingMode) + ); + } + } else { + + } return Promise.resolve(); } pause() { - this.video.pause(); + if (this.video instanceof bindings.nativeVideo.VideoDevice) { // XXX + this.video.close(); + } else { + this.video.pause(); + } } get currentTime() { - return this.video && this.video.currentTime; + return this.video.currentTime; } set currentTime(currentTime) { - if (this.video) { - this.video.currentTime = currentTime; - } + this.video.currentTime = currentTime; } get duration() { - return this.video && this.video.duration; + return this.video.duration; } - set duration(duration) { - if (this.video) { - this.video.duration = duration; - } + set duration(duration) {} + + get buffered() { + return new TimeRanges([0, this.duration]); } + set buffered(buffered) {} run() { let running = false; @@ -2697,8 +2635,12 @@ class HTMLVideoElement extends HTMLMediaElement { return running; } + + update() { + this.video.update(); + } } -*/ +module.exports.HTMLVideoElement = HTMLVideoElement; function _hash(s) { let result = 0; From 6bbafabe47745a109069fe69bcf04c14c60a91bf Mon Sep 17 00:00:00 2001 From: avaer Date: Sat, 10 Nov 2018 19:28:27 -0500 Subject: [PATCH 03/31] Hook in missinf Video play in HTMLVideoElement --- src/DOM.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DOM.js b/src/DOM.js index bcdcbd43fb..3f1b2fa865 100644 --- a/src/DOM.js +++ b/src/DOM.js @@ -2547,7 +2547,7 @@ class HTMLVideoElement extends HTMLMediaElement { set data(data) {} play() { - if (this.video instanceof bindings.nativeVideo.VideoDevice) { + if (this.video instanceof bindings.nativeVideo.VideoDevice) { // XXX const _getDevice = facingMode => { switch (facingMode) { case 'user': return devices[0]; @@ -2573,7 +2573,7 @@ class HTMLVideoElement extends HTMLMediaElement { ); } } else { - + this.video.play(); } return Promise.resolve(); From 6eae16d41cf89771a8f9e2973d10c39d6975db61 Mon Sep 17 00:00:00 2001 From: avaer Date: Sat, 10 Nov 2018 19:29:30 -0500 Subject: [PATCH 04/31] Bugfix Video.cpp bits per pixel --- deps/exokit-bindings/videocontext/src/Video.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index e6f3955931..5c6e6e5591 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -11,7 +11,8 @@ extern "C" { namespace ffmpeg { const int kBufferSize = 4 * 1024; -const AVPixelFormat kPixelFormat = AV_PIX_FMT_RGB24; +const int bpp = 4; +const AVPixelFormat kPixelFormat = AV_PIX_FMT_RGBA; AppData::AppData() : dataPos(0), @@ -436,7 +437,7 @@ NAN_GETTER(Video::DataGetter) { unsigned int width = video->GetWidth(); unsigned int height = video->GetHeight(); - unsigned int dataSize = width * height * 3; + unsigned int dataSize = width * height * bpp; if (video->dataArray.IsEmpty()) { Local arrayBuffer = ArrayBuffer::New(Isolate::GetCurrent(), dataSize); Local uint8ClampedArray = Uint8ClampedArray::New(arrayBuffer, 0, arrayBuffer->ByteLength()); From b70ac372b3ba40ebc5fb6b73c44cbb25d4f8ffc7 Mon Sep 17 00:00:00 2001 From: avaer Date: Sat, 10 Nov 2018 19:40:39 -0500 Subject: [PATCH 05/31] Bugfix video load error message --- src/DOM.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DOM.js b/src/DOM.js index 3f1b2fa865..c5d3c3e9f4 100644 --- a/src/DOM.js +++ b/src/DOM.js @@ -2481,7 +2481,7 @@ class HTMLVideoElement extends HTMLMediaElement { cb(); }) .catch(err => { - console.warn('failed to load audio:', src); + console.warn('failed to load video:', src); const e = new ErrorEvent('error', {target: this}); e.message = err.message; From fd02e53f4631a45de1db56a79396714e45add5e8 Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 00:01:47 -0500 Subject: [PATCH 06/31] Add Video.cpp error message output --- .../videocontext/src/Video.cpp | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 5c6e6e5591..4b26bc6432 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -57,7 +57,8 @@ bool AppData::set(vector &memory, string *error) { fmt_ctx = avformat_alloc_context(); io_ctx = avio_alloc_context((unsigned char *)av_malloc(kBufferSize), kBufferSize, 0, this, bufferRead, nullptr, bufferSeek); fmt_ctx->pb = io_ctx; - if (avformat_open_input(&fmt_ctx, "memory input", nullptr, nullptr) < 0) { + int ret = avformat_open_input(&fmt_ctx, "memory input", nullptr, nullptr); + if (ret < 0) { if (error) { *error = "failed to open input"; } @@ -65,9 +66,13 @@ bool AppData::set(vector &memory, string *error) { } // find stream info - if (avformat_find_stream_info(fmt_ctx, nullptr) < 0) { + ret = avformat_find_stream_info(fmt_ctx, nullptr); + if (ret < 0) { if (error) { - *error = "failed to get stream info"; + *error = "failed to get stream info: "; + char errbuf[1024]; + av_strerror(ret, errbuf, sizeof(errbuf)); + *error += errbuf; } return false; } @@ -99,15 +104,20 @@ bool AppData::set(vector &memory, string *error) { decoder = avcodec_find_decoder(codec_ctx->codec_id); if (decoder == nullptr) { if (error) { - *error = "failed to find decoder"; + *error = "failed to find decoder: "; + *error += avcodec_get_name(codec_ctx->codec_id); } return false; } // open the decoder - if (avcodec_open2(codec_ctx, decoder, nullptr) < 0) { + ret = avcodec_open2(codec_ctx, decoder, nullptr); + if (ret < 0) { if (error) { - *error = "failed to open codec"; + *error = "failed to open codec: "; + char errbuf[1024]; + av_strerror(ret, errbuf, sizeof(errbuf)); + *error += errbuf; } return false; } From d7322653c7881ce7f9acee56109efc758291d9eb Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 00:02:17 -0500 Subject: [PATCH 07/31] Clean up Video.cpp error result capture --- deps/exokit-bindings/videocontext/src/Video.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 4b26bc6432..38c26f1f60 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -181,13 +181,14 @@ FrameStatus AppData::advanceToFrameAt(double timestamp) { } bool packetValid = false; + int ret; for (;;) { if (packetValid) { av_free_packet(packet); packetValid = false; } - int ret = av_read_frame(fmt_ctx, packet); + ret = av_read_frame(fmt_ctx, packet); packetValid = true; if (ret == AVERROR_EOF) { av_free_packet(packet); @@ -204,7 +205,8 @@ FrameStatus AppData::advanceToFrameAt(double timestamp) { } // we have a valid packet at this point int frame_finished = 0; - if (avcodec_decode_video2(codec_ctx, av_frame, &frame_finished, packet) < 0) { + ret = avcodec_decode_video2(codec_ctx, av_frame, &frame_finished, packet); + if (ret < 0) { av_free_packet(packet); return FRAME_STATUS_ERROR; } From 528654c68a6d881aa2c9dc5c4d0c461a4bb730eb Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 02:16:02 -0500 Subject: [PATCH 08/31] Bugfix sws_scale called on incomplete video frames --- deps/exokit-bindings/videocontext/src/Video.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 38c26f1f60..38f126e744 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -211,9 +211,8 @@ FrameStatus AppData::advanceToFrameAt(double timestamp) { return FRAME_STATUS_ERROR; } - sws_scale(conv_ctx, av_frame->data, av_frame->linesize, 0, codec_ctx->height, gl_frame->data, gl_frame->linesize); - if (frame_finished) { + sws_scale(conv_ctx, av_frame->data, av_frame->linesize, 0, codec_ctx->height, gl_frame->data, gl_frame->linesize); lastTimestamp = (double)packet->pts * timeBase; } From 01a649db8eca3012cb14c509d26befb030e0a6e6 Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:25:57 -0500 Subject: [PATCH 09/31] Bump video buffer size --- deps/exokit-bindings/videocontext/src/Video.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 38f126e744..f7a8c7a8ac 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -10,7 +10,7 @@ extern "C" { namespace ffmpeg { -const int kBufferSize = 4 * 1024; +const int kBufferSize = 8 * 1024; const int bpp = 4; const AVPixelFormat kPixelFormat = AV_PIX_FMT_RGBA; From 9169223b2d029df4cb5a1b5bae130847f08942da Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:26:25 -0500 Subject: [PATCH 10/31] Flag format context as AVFMT_FLAG_CUSTOM_IO --- deps/exokit-bindings/videocontext/src/Video.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index f7a8c7a8ac..bd86cf43b6 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -57,7 +57,8 @@ bool AppData::set(vector &memory, string *error) { fmt_ctx = avformat_alloc_context(); io_ctx = avio_alloc_context((unsigned char *)av_malloc(kBufferSize), kBufferSize, 0, this, bufferRead, nullptr, bufferSeek); fmt_ctx->pb = io_ctx; - int ret = avformat_open_input(&fmt_ctx, "memory input", nullptr, nullptr); + fmt_ctx->flags |= AVFMT_FLAG_CUSTOM_IO; + int ret = avformat_open_input(&fmt_ctx, "memory", nullptr, nullptr); if (ret < 0) { if (error) { *error = "failed to open input"; From 36f5de2815b4d4e96fb3c6ecd3b024ee3c1582b9 Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:26:47 -0500 Subject: [PATCH 11/31] Clean up video set data stream index --- deps/exokit-bindings/videocontext/src/Video.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index bd86cf43b6..363ad50d3c 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -81,7 +81,8 @@ bool AppData::set(vector &memory, string *error) { // dump debug info // av_dump_format(fmt_ctx, 0, argv[1], 0); - // find the video stream + // find the video stream + stream_idx = -1; for (unsigned int i = 0; i < fmt_ctx->nb_streams; ++i) { if (fmt_ctx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) From 33c534977296321cb55b9ea48d7dd9fa2ef70e6d Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:26:56 -0500 Subject: [PATCH 12/31] Small video.cpp cleanup --- deps/exokit-bindings/videocontext/src/Video.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 363ad50d3c..60690f6922 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -91,7 +91,6 @@ bool AppData::set(vector &memory, string *error) { break; } } - if (stream_idx == -1) { if (error) { *error = "failed to find video stream"; From c5037f1f5504cdeca0a7c8a557893be03a6d324f Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:27:24 -0500 Subject: [PATCH 13/31] Clean up video.cpp connnntext initialization --- .../exokit-bindings/videocontext/src/Video.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 60690f6922..4e4a6c22a9 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -99,19 +99,29 @@ bool AppData::set(vector &memory, string *error) { } video_stream = fmt_ctx->streams[stream_idx]; - codec_ctx = video_stream->codec; - + // find the decoder - decoder = avcodec_find_decoder(codec_ctx->codec_id); + decoder = avcodec_find_decoder(video_stream->codec->codec_id); if (decoder == nullptr) { if (error) { *error = "failed to find decoder: "; - *error += avcodec_get_name(codec_ctx->codec_id); + *error += avcodec_get_name(video_stream->codec->codec_id); } return false; } // open the decoder + codec_ctx = avcodec_alloc_context3(decoder); + ret = avcodec_copy_context(codec_ctx, video_stream->codec); + if (ret < 0) { + if (error) { + *error = "failed to copy codec context: "; + char errbuf[1024]; + av_strerror(ret, errbuf, sizeof(errbuf)); + *error += errbuf; + } + return false; + } ret = avcodec_open2(codec_ctx, decoder, nullptr); if (ret < 0) { if (error) { From ced25faa3fe372eef5c60cf4b40d8fe1e4bc9cfe Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 04:27:49 -0500 Subject: [PATCH 14/31] Clean up video.cpp sws context initialization --- deps/exokit-bindings/videocontext/src/Video.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/exokit-bindings/videocontext/src/Video.cpp b/deps/exokit-bindings/videocontext/src/Video.cpp index 4e4a6c22a9..b0c8a34ca2 100644 --- a/deps/exokit-bindings/videocontext/src/Video.cpp +++ b/deps/exokit-bindings/videocontext/src/Video.cpp @@ -143,7 +143,7 @@ bool AppData::set(vector &memory, string *error) { // allocate the converter conv_ctx = sws_getContext( - codec_ctx->width, codec_ctx->height, codec_ctx->pix_fmt, + codec_ctx->width, codec_ctx->height, codec_ctx->pix_fmt != -1 ? codec_ctx->pix_fmt : AV_PIX_FMT_YUV420P, codec_ctx->width, codec_ctx->height, kPixelFormat, SWS_BICUBIC, nullptr, nullptr, nullptr ); From c77a2b7000a46598e8f8cb7a915191adceefd5c7 Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 08:19:38 -0500 Subject: [PATCH 15/31] Bump native-video-deps version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4595ca61ea..181fdb326b 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "native-canvas-deps": "0.0.50", "native-graphics-deps": "0.0.20", "native-openvr-deps": "0.0.17", - "native-video-deps": "0.0.30", + "native-video-deps": "0.0.34", "node-ipc": "^9.1.1", "node-localstorage": "^1.3.1", "parse-int": "^1.0.2", From 6c49df325a839ab149da97efbd6108ee3b9e03bb Mon Sep 17 00:00:00 2001 From: avaer Date: Sun, 11 Nov 2018 18:47:52 -0500 Subject: [PATCH 16/31] Bugfix video devices query in DOM.js --- src/DOM.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DOM.js b/src/DOM.js index c5d3c3e9f4..fe78f418be 100644 --- a/src/DOM.js +++ b/src/DOM.js @@ -2549,6 +2549,7 @@ class HTMLVideoElement extends HTMLMediaElement { play() { if (this.video instanceof bindings.nativeVideo.VideoDevice) { // XXX const _getDevice = facingMode => { + const devices = bindings.nativeVideo.Video.getDevices(); switch (facingMode) { case 'user': return devices[0]; case 'environment': return devices[1]; From 454b15ffab6d85d5985b2cde012cdb4fdc68c921 Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Sun, 11 Nov 2018 21:38:21 -0500 Subject: [PATCH 17/31] Add missing linux libx264 linkage --- binding.gyp | 1 + 1 file changed, 1 insertion(+) diff --git a/binding.gyp b/binding.gyp index 0e932600a6..a7257e1a4b 100644 --- a/binding.gyp +++ b/binding.gyp @@ -177,6 +177,7 @@ '-lXxf86vm', '-lXrandr', '-lXi', + '-lx264', '-lasound', '-lexpat', ], From 8164bcdc1c6d08aa643174ed885d1721441cbfa8 Mon Sep 17 00:00:00 2001 From: Avaer Kazmer Date: Mon, 12 Nov 2018 04:32:51 -0500 Subject: [PATCH 18/31] Initial video threading --- .../videocontext/include/Video.h | 35 +++- .../videocontext/src/Video.cpp | 167 +++++++++++++----- 2 files changed, 156 insertions(+), 46 deletions(-) diff --git a/deps/exokit-bindings/videocontext/include/Video.h b/deps/exokit-bindings/videocontext/include/Video.h index 1f80ef4f37..6e0bdc2f1a 100644 --- a/deps/exokit-bindings/videocontext/include/Video.h +++ b/deps/exokit-bindings/videocontext/include/Video.h @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include extern "C" { #include @@ -27,6 +30,12 @@ enum FrameStatus { FRAME_STATUS_EOF, }; +class AppData; +class Video; +class VideoCamera; +class VideoRequest; +class VideoResponse; + class AppData { public: AppData(); @@ -87,12 +96,14 @@ class Video : public ObjectWrap { double getRequiredCurrentTimeS(); double getFrameCurrentTimeS(); FrameStatus advanceToFrameAt(double timestamp); + static void queueInVideoThread(std::function fn, std::function cb); + static void runInMainThread(); Video(); ~Video(); private: - AppData data; + AppData appData; bool loaded; bool playing; bool loop; @@ -100,9 +111,13 @@ class Video : public ObjectWrap { double startFrameTime; Nan::Persistent dataArray; bool dataDirty; -}; -class VideoCamera; + std::thread thread; + uv_sem_t requestSem; + // uv_sem_t responseSem; + std::mutex requestMutex; + std::deque requestQueue; +}; class VideoDevice : public ObjectWrap { public: @@ -126,6 +141,20 @@ class VideoDevice : public ObjectWrap { Nan::Persistent imageData; }; +class VideoRequest { +public: + std::function)> fn; + std::function cb; +}; + +class VideoResponse { +public: + std::function fn; +}; + +extern std::mutex responseMutex; +extern uv_async_t requestAsync; +extern std::deque responseQueue; extern std::vector