From e1e2634fdb105a03d741ee976db7abde63cc9fb4 Mon Sep 17 00:00:00 2001 From: Alexandru Prisacariu Date: Tue, 17 Mar 2020 13:04:22 +0200 Subject: [PATCH 1/4] Loading images using workers --- src/loaders/HtmlImage.js | 154 +++++++++++++----- src/workers/fetchImage.js | 42 +++++ src/workers/fetchImageUsingImageBitmap.js | 47 ++++++ src/workers/fetchImageUsingOffscreenCanvas.js | 64 ++++++++ 4 files changed, 263 insertions(+), 44 deletions(-) create mode 100644 src/workers/fetchImage.js create mode 100644 src/workers/fetchImageUsingImageBitmap.js create mode 100644 src/workers/fetchImageUsingOffscreenCanvas.js diff --git a/src/loaders/HtmlImage.js b/src/loaders/HtmlImage.js index d19939dc3..b7cedd2f5 100644 --- a/src/loaders/HtmlImage.js +++ b/src/loaders/HtmlImage.js @@ -42,6 +42,40 @@ function HtmlImageLoader(stage) { throw new Error('Stage type incompatible with loader'); } this._stage = stage; + + // This variable will have the response callbacks where the keys will be + // the image URL and the value will be a function + this._imageFetchersCallbacks = {}; + + this._isSimpleImageFetcherWorker = true; + + function imageFetcherWorkerOnMessage(event) { + this._imageFetchersCallbacks[event.data.imageURL](event); + delete this._imageFetchersCallbacks[event.data.imageURL]; + } + + // Check what method can use for loading the images + // Check if the browser supports `OffscreenCanvas` and `createImageBitmap` + // else using only fetch + if ( + typeof window.OffscreenCanvas === "function" && + typeof window.createImageBitmap === "function" + ) { + this._imageFetcherNoResizeWorker = + new require("../workers/fetchImageUsingImageBitmap")(); + + this._imageFetcherResizeWorker = + new require("../workers/fetchImageUsingOffscreenCanvas")(); + + this._imageFetcherNoResizeWorker.onmessage = imageFetcherWorkerOnMessage; + this._imageFetcherResizeWorker.onmessage = imageFetcherWorkerOnMessage; + + this._isSimpleImageFetcherWorker = false; + } else { + this._imageFetcherWorker = new require("../workers/fetchImage")(); + + this._imageFetcherWorker.onmessage = imageFetcherWorkerOnMessage; + } } /** @@ -53,20 +87,6 @@ function HtmlImageLoader(stage) { * @return {function()} A function to cancel loading. */ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { - var img = new Image(); - - // Allow cross-domain image loading. - // This is required to be able to create WebGL textures from images fetched - // from a different domain. Note that setting the crossorigin attribute to - // 'anonymous' will trigger a CORS preflight for cross-domain requests, but no - // credentials (cookies or HTTP auth) will be sent; to do so, the attribute - // would have to be set to 'use-credentials' instead. Unfortunately, this is - // not a safe choice, as it causes requests to fail when the response contains - // an Access-Control-Allow-Origin header with a wildcard. See the section - // "Credentialed requests and wildcards" on: - // https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS - img.crossOrigin = 'anonymous'; - var x = rect && rect.x || 0; var y = rect && rect.y || 0; var width = rect && rect.width || 1; @@ -74,43 +94,89 @@ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { done = once(done); - img.onload = function() { - if (x === 0 && y === 0 && width === 1 && height === 1) { - done(null, new StaticAsset(img)); + var cancelFunction; + var shouldCancel = false; + + if (this._isSimpleImageFetcherWorker) { + var internalCancelFunction; + + this._imageFetchersCallbacks[url] = function(event) { + if (shouldCancel) return; + + var img = new Image(); + + const objectURL = URL.createObjectURL(event.data.blob); + + img.onload = () => { + URL.revokeObjectURL(objectURL); + + if (x === 0 && y === 0 && width === 1 && height === 1) { + done(null, new StaticAsset(img)); + } + else { + x *= img.naturalWidth; + y *= img.naturalHeight; + width *= img.naturalWidth; + height *= img.naturalHeight; + + var canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + var context = canvas.getContext('2d'); + + context.drawImage(img, x, y, width, height, 0, 0, width, height); + + done(null, new StaticAsset(canvas)); + } + }; + + img.url = objectURL; + + internalCancelFunction = function() { + img.onload = img.onerror = null; + img.src = ''; + } + }; + + cancelFunction = function() { + shouldCancel = true; + if (internalCancelFunction) internalCancelFunction(); + done.apply(null, arguments); } - else { - x *= img.naturalWidth; - y *= img.naturalHeight; - width *= img.naturalWidth; - height *= img.naturalHeight; - - var canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - var context = canvas.getContext('2d'); - context.drawImage(img, x, y, width, height, 0, 0, width, height); + this._imageFetcherWorker.postMessage({ imageURL: url }); + } else { + this._imageFetchersCallbacks[url] = function(event) { + if (shouldCancel) return; + done(null, new StaticAsset(event.data.imageBitmap)); + }; - done(null, new StaticAsset(canvas)); + cancelFunction = function() { + shouldCancel = true; + done.apply(null, arguments); } - }; - img.onerror = function() { - // TODO: is there any way to distinguish a network error from other - // kinds of errors? For now we always return NetworkError since this - // prevents images to be retried continuously while we are offline. - done(new NetworkError('Network error: ' + url)); - }; - - img.src = url; - - function cancel() { - img.onload = img.onerror = null; - img.src = ''; - done.apply(null, arguments); + if (x === 0 && y === 0 && width === 1 && height === 1) { + this._imageFetcherNoResizeWorker.postMessage({ imageURL: url }); + } else { + const mainCanvas = document.createElement("canvas"); + const mainCanvasOffscreen = mainCanvas.transferControlToOffscreen(); + + this._imageFetcherResizeWorker.postMessage( + { + imageURL: url, + canvas: mainCanvasOffscreen, + x, + y, + width, + height + }, + [mainCanvasOffscreen] + ); + } } - return cancel; + return cancelFunction; }; module.exports = HtmlImageLoader; diff --git a/src/workers/fetchImage.js b/src/workers/fetchImage.js new file mode 100644 index 000000000..cd5888caa --- /dev/null +++ b/src/workers/fetchImage.js @@ -0,0 +1,42 @@ +/* + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @class FetchImage + * @classdesc + * + * Worker that load an image using fetch and return the blob of it + */ + +/** + * @typedef {Object} WorkerResult + * @property {String} imageURL the URL of the image for identification + * @property {Blob} imageBlob the image data as a blob + */ + +/** + * The listener of the worker + * + * @param {String} imageURL the URL of the image + * + * @returns {WorkerResult} + */ +onmessage = async event => { + postMessage({ + imageURL: event.data.imageURL, + imageBlob: await (await fetch(event.data.imageURL)).blob() + }); +}; \ No newline at end of file diff --git a/src/workers/fetchImageUsingImageBitmap.js b/src/workers/fetchImageUsingImageBitmap.js new file mode 100644 index 000000000..436b3193f --- /dev/null +++ b/src/workers/fetchImageUsingImageBitmap.js @@ -0,0 +1,47 @@ +/* + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @class FetchImageUsingImageBitmap + * @classdesc + * + * Worker that load an image using fetch and createImageBitmap the return it + */ + +/** + * @typedef {Object} WorkerResult + * @property {String} imageURL the URL of the image for identification + * @property {ImageBitmap} imageBitmap the image as an image bitmap + */ + +/** + * The listener of the worker + * + * @param {String} imageURL the URL of the image + * + * @returns {WorkerResult} + */ +onmessage = async event => { + const blob = await (await fetch(event.data.imageURL)).blob(); + + postMessage({ + imageURL: event.data.imageURL, + imageBitmap: await createImageBitmap( + blob, + { imageOrientation: "flipY" } + ) + }); +}; \ No newline at end of file diff --git a/src/workers/fetchImageUsingOffscreenCanvas.js b/src/workers/fetchImageUsingOffscreenCanvas.js new file mode 100644 index 000000000..b9ee026e1 --- /dev/null +++ b/src/workers/fetchImageUsingOffscreenCanvas.js @@ -0,0 +1,64 @@ +/* + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @class FetchImageUsingOffscreenCanvasWorker + * @classdesc + * + * Worker that load an image using fetch and createImageBitmap then resize it + * using an offscreenCanvas and return an imageBitmap of the image + */ + +/** + * @typedef {Object} WorkerResult + * @property {String} imageURL the URL of the image for identification + * @property {ImageBitmap} imageBitmap the image as an image bitmap + */ + +/** + * The listener of the worker + * + * @param {String} imageURL the URL of the image + * @param {OffscreenCanvas} canvas the canvas to draw the loaded image + * @param {Number} x The x coordinate + * @param {Number} y The y coordinate + * @param {Number} width The width + * @param {Number} height The width + * + * @returns {WorkerResult} + */ +onmessage = async event => { + const blob = await (await fetch(event.data.imageURL)).blob(); + let imageBitmap = await createImageBitmap( + blob, + { imageOrientation: "flipY" } + ); + + const x = event.data.x; + const y = event.data.y; + const width = event.data.width; + const height = event.data.height; + + const canvas = event.data.canvas; + const ctx = canvas.getContext("2d"); + ctx.drawImage(b, x, y, width, height, 0, 0, width, height); + imageBitmap = await createImageBitmap(await canvas.convertToBlob()); + + postMessage({ + imageURL: event.data.imageURL, + imageBitmap: imageBitmap + }); +}; \ No newline at end of file From 3edd69ef6586cb01a3aac96e96599b12f7e0d07f Mon Sep 17 00:00:00 2001 From: Alexandru Prisacariu Date: Tue, 17 Mar 2020 13:21:42 +0200 Subject: [PATCH 2/4] Fixed loading workers --- src/loaders/HtmlImage.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/loaders/HtmlImage.js b/src/loaders/HtmlImage.js index b7cedd2f5..89a0a65ac 100644 --- a/src/loaders/HtmlImage.js +++ b/src/loaders/HtmlImage.js @@ -43,6 +43,8 @@ function HtmlImageLoader(stage) { } this._stage = stage; + const self = this; + // This variable will have the response callbacks where the keys will be // the image URL and the value will be a function this._imageFetchersCallbacks = {}; @@ -50,8 +52,8 @@ function HtmlImageLoader(stage) { this._isSimpleImageFetcherWorker = true; function imageFetcherWorkerOnMessage(event) { - this._imageFetchersCallbacks[event.data.imageURL](event); - delete this._imageFetchersCallbacks[event.data.imageURL]; + self._imageFetchersCallbacks[event.data.imageURL](event); + delete self._imageFetchersCallbacks[event.data.imageURL]; } // Check what method can use for loading the images @@ -62,17 +64,17 @@ function HtmlImageLoader(stage) { typeof window.createImageBitmap === "function" ) { this._imageFetcherNoResizeWorker = - new require("../workers/fetchImageUsingImageBitmap")(); + new Worker("../workers/fetchImageUsingImageBitmap.js"); this._imageFetcherResizeWorker = - new require("../workers/fetchImageUsingOffscreenCanvas")(); + new Worker("../workers/fetchImageUsingOffscreenCanvas.js"); this._imageFetcherNoResizeWorker.onmessage = imageFetcherWorkerOnMessage; this._imageFetcherResizeWorker.onmessage = imageFetcherWorkerOnMessage; this._isSimpleImageFetcherWorker = false; } else { - this._imageFetcherWorker = new require("../workers/fetchImage")(); + this._imageFetcherWorker = new Worker("../workers/fetchImage.js"); this._imageFetcherWorker.onmessage = imageFetcherWorkerOnMessage; } @@ -179,4 +181,4 @@ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { return cancelFunction; }; -module.exports = HtmlImageLoader; +module.exports = HtmlImageLoader; \ No newline at end of file From 8378c1ebbc88d34466761ce4fb1742ec867f17d1 Mon Sep 17 00:00:00 2001 From: Alexandru Prisacariu Date: Tue, 17 Mar 2020 13:51:36 +0200 Subject: [PATCH 3/4] Now using normal img load when no available OffscreenCanvas --- src/loaders/HtmlImage.js | 92 ++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/src/loaders/HtmlImage.js b/src/loaders/HtmlImage.js index 89a0a65ac..50adb523d 100644 --- a/src/loaders/HtmlImage.js +++ b/src/loaders/HtmlImage.js @@ -45,12 +45,12 @@ function HtmlImageLoader(stage) { const self = this; + this._useWorkers = false; + // This variable will have the response callbacks where the keys will be // the image URL and the value will be a function this._imageFetchersCallbacks = {}; - this._isSimpleImageFetcherWorker = true; - function imageFetcherWorkerOnMessage(event) { self._imageFetchersCallbacks[event.data.imageURL](event); delete self._imageFetchersCallbacks[event.data.imageURL]; @@ -72,11 +72,7 @@ function HtmlImageLoader(stage) { this._imageFetcherNoResizeWorker.onmessage = imageFetcherWorkerOnMessage; this._imageFetcherResizeWorker.onmessage = imageFetcherWorkerOnMessage; - this._isSimpleImageFetcherWorker = false; - } else { - this._imageFetcherWorker = new Worker("../workers/fetchImage.js"); - - this._imageFetcherWorker.onmessage = imageFetcherWorkerOnMessage; + this._useWorkers = true; } } @@ -99,54 +95,56 @@ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { var cancelFunction; var shouldCancel = false; - if (this._isSimpleImageFetcherWorker) { - var internalCancelFunction; + if (!this._useWorkers) { + var img = new Image(); + + // Allow cross-domain image loading. + // This is required to be able to create WebGL textures from images fetched + // from a different domain. Note that setting the crossorigin attribute to + // 'anonymous' will trigger a CORS preflight for cross-domain requests, but no + // credentials (cookies or HTTP auth) will be sent; to do so, the attribute + // would have to be set to 'use-credentials' instead. Unfortunately, this is + // not a safe choice, as it causes requests to fail when the response contains + // an Access-Control-Allow-Origin header with a wildcard. See the section + // "Credentialed requests and wildcards" on: + // https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS + img.crossOrigin = 'anonymous'; + + img.onload = () => { + if (x === 0 && y === 0 && width === 1 && height === 1) { + done(null, new StaticAsset(img)); + } + else { + x *= img.naturalWidth; + y *= img.naturalHeight; + width *= img.naturalWidth; + height *= img.naturalHeight; - this._imageFetchersCallbacks[url] = function(event) { - if (shouldCancel) return; + var canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + var context = canvas.getContext('2d'); - var img = new Image(); - - const objectURL = URL.createObjectURL(event.data.blob); - - img.onload = () => { - URL.revokeObjectURL(objectURL); - - if (x === 0 && y === 0 && width === 1 && height === 1) { - done(null, new StaticAsset(img)); - } - else { - x *= img.naturalWidth; - y *= img.naturalHeight; - width *= img.naturalWidth; - height *= img.naturalHeight; - - var canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - var context = canvas.getContext('2d'); - - context.drawImage(img, x, y, width, height, 0, 0, width, height); - - done(null, new StaticAsset(canvas)); - } - }; - - img.url = objectURL; - - internalCancelFunction = function() { - img.onload = img.onerror = null; - img.src = ''; + context.drawImage(img, x, y, width, height, 0, 0, width, height); + + done(null, new StaticAsset(canvas)); } }; + img.onerror = function() { + // TODO: is there any way to distinguish a network error from other + // kinds of errors? For now we always return NetworkError since this + // prevents images to be retried continuously while we are offline. + done(new NetworkError('Network error: ' + url)); + }; + + img.src = url; + cancelFunction = function() { - shouldCancel = true; - if (internalCancelFunction) internalCancelFunction(); + img.onload = img.onerror = null; + img.src = ''; done.apply(null, arguments); } - - this._imageFetcherWorker.postMessage({ imageURL: url }); } else { this._imageFetchersCallbacks[url] = function(event) { if (shouldCancel) return; From 182aee293ee42ae6bfdf96b06a8a4d202d7065e1 Mon Sep 17 00:00:00 2001 From: Aykelith Date: Thu, 9 Apr 2020 22:00:15 +0300 Subject: [PATCH 4/4] Resolved the release build to compile --- package.json | 1 + src/loaders/HtmlImage.js | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 645444dd8..a84771aec 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ }, "dependencies": { "bowser": "2.7.0", + "code": "^5.2.4", "gl-matrix": "3.1.0", "hammerjs": "2.0.4", "minimal-event-emitter": "1.0.0" diff --git a/src/loaders/HtmlImage.js b/src/loaders/HtmlImage.js index 50adb523d..f76500a0f 100644 --- a/src/loaders/HtmlImage.js +++ b/src/loaders/HtmlImage.js @@ -110,7 +110,7 @@ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { // https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS img.crossOrigin = 'anonymous'; - img.onload = () => { + img.onload = function() { if (x === 0 && y === 0 && width === 1 && height === 1) { done(null, new StaticAsset(img)); } @@ -166,10 +166,10 @@ HtmlImageLoader.prototype.loadImage = function(url, rect, done) { { imageURL: url, canvas: mainCanvasOffscreen, - x, - y, - width, - height + x: x, + y: y, + width: width, + height: height }, [mainCanvasOffscreen] );