Thumbnails for new video attachments and video quotes (#2293)

Two major changes here:

1. When you attach a video to your message, you see a thumbnail of its first frame in the composition area, instead of a generic file icon
2. When you reply to a message with a video in it, your message will include a thumbnail of that video (we'll also generate thumbnails for video quotes if not provided already)

I also made a change to our quote-loading algorithm, since I noticed that our previous approach had some performance issues now that we support video. Where before, we privileged our ability to make local thumbnails at a higher quality, now we defer to any thumbnail we have saved on the quote. So the algorithm for preparing a quote for display is now:

1. Load thumbnail from disk
2. Check to see if quoted message is in memory already. If it is, and we weren't able to load a thumbnail already, generate it from quoted message.
3. If there is an attachment in quote, and we couldn't find message in memory, and the attachment contentType is either image or video, we will go to the database to find the referenced message and create a thumbnail from it.

Bonus bug-fix: The scroll down button in the bottom right of the conversation no longer has an outline when you click it.
pull/1/head
Scott Nonnenberg 7 years ago committed by GitHub
commit fdd6985a79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -111,6 +111,7 @@ module.exports = function(grunt) {
'!js/views/conversation_search_view.js',
'!js/views/conversation_view.js',
'!js/views/debug_log_view.js',
'!js/views/file_input_view.js',
'!js/views/message_view.js',
'!js/models/conversations.js',
'!js/models/messages.js',
@ -170,6 +171,7 @@ module.exports = function(grunt) {
'!js/views/conversation_search_view.js',
'!js/views/conversation_view.js',
'!js/views/debug_log_view.js',
'!js/views/file_input_view.js',
'!js/views/message_view.js',
'!js/Mp3LameEncoder.min.js',
'!js/WebAudioRecorderMp3.js',

@ -237,7 +237,12 @@
</div>
</script>
<script type='text/x-tmpl-mustache' id='attachment-preview'>
<div class='image-container'>
<img src='{{ source }}' class='preview' />
<div class='outer'>
<div class='play icon'></div>
</div>
</div>
<a class='x close' alt='remove attachment' href='#'></a>
</script>
<script type='text/x-tmpl-mustache' id='file-view'>

@ -628,13 +628,23 @@
async makeThumbnailAttachment(attachment) {
const attachmentWithData = await loadAttachmentData(attachment);
const { data, contentType } = attachmentWithData;
const objectUrl = this.makeObjectUrl(data, contentType);
const thumbnail = await Whisper.FileInputView.makeThumbnail(128, objectUrl);
const objectUrl = Signal.Util.arrayBufferToObjectURL({
data,
type: contentType,
});
const thumbnail = Signal.Util.GoogleChrome.isImageTypeSupported(contentType)
? await Whisper.FileInputView.makeImageThumbnail(128, objectUrl)
: await Whisper.FileInputView.makeVideoThumbnail(128, objectUrl);
URL.revokeObjectURL(objectUrl);
const arrayBuffer = await this.blobToArrayBuffer(thumbnail);
const finalContentType = 'image/png';
const finalObjectUrl = this.makeObjectUrl(arrayBuffer, finalContentType);
const finalObjectUrl = Signal.Util.arrayBufferToObjectURL({
data: arrayBuffer,
type: finalContentType,
});
return {
data: arrayBuffer,
@ -654,7 +664,8 @@
attachments: await Promise.all((attachments || []).map(async (attachment) => {
const { contentType } = attachment;
const willMakeThumbnail =
Signal.Util.GoogleChrome.isImageTypeSupported(contentType);
Signal.Util.GoogleChrome.isImageTypeSupported(contentType) ||
Signal.Util.GoogleChrome.isVideoTypeSupported(contentType);
return {
contentType,
@ -1121,12 +1132,6 @@
forceRender(message) {
message.trigger('change', message);
},
makeObjectUrl(data, contentType) {
const blob = new Blob([data], {
type: contentType,
});
return URL.createObjectURL(blob);
},
makeMessagesLookup(messages) {
return messages.reduce((acc, message) => {
const { source, sent_at: sentAt } = message.attributes;
@ -1154,9 +1159,12 @@
const { attachments, id, author } = quote;
const first = attachments[0];
// Maybe in the future we could try to pull the thumbnail from a video ourselves,
// but for now we will rely on incoming thumbnails only.
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
if (!first || message.quoteThumbnail) {
return false;
}
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
!Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
return false;
}
@ -1181,7 +1189,7 @@
} catch (error) {
console.log(
'Problem loading attachment data for quoted message from database',
error && error.stack ? error.stack : error
Signal.Types.Errors.toLogFormat(error)
);
return false;
}
@ -1194,9 +1202,12 @@
const { attachments } = quote;
const first = attachments[0];
// Maybe in the future we could try to pull thumbnails video ourselves,
// but for now we will rely on incoming thumbnails only.
if (!first || !Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
if (!first || message.quoteThumbnail) {
return;
}
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
!Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
return;
}
@ -1221,7 +1232,8 @@
const { quote } = message.attributes;
const { attachments } = quote;
const first = attachments[0];
if (!first) {
if (!first || message.quoteThumbnail) {
return false;
}
@ -1230,11 +1242,13 @@
if (!thumbnail) {
return false;
}
try {
const thumbnailWithData = await loadAttachmentData(thumbnail);
thumbnailWithData.objectUrl = this.makeObjectUrl(
thumbnailWithData.data,
thumbnailWithData.contentType
);
const { data, contentType } = thumbnailWithData;
thumbnailWithData.objectUrl = Signal.Util.arrayBufferToObjectURL({
data,
type: contentType,
});
// If we update this data in place, there's the risk that this data could be
// saved back to the database
@ -1242,6 +1256,13 @@
message.quoteThumbnail = thumbnailWithData;
return true;
} catch (error) {
console.log(
'loadQuoteThumbnail: had trouble loading thumbnail data from disk',
error && error.stack ? error.stack : error
);
return false;
}
},
async processQuotes(messages) {
const lookup = this.makeMessagesLookup(messages);
@ -1259,21 +1280,22 @@
return;
}
// 1. Check to see if we've already loaded the target message into memory
// 1. Load provided thumbnail
const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
// 2. Check to see if we've already loaded the target message into memory
const { author, id } = quote;
const key = this.makeKey(author, id);
const quotedMessage = lookup[key];
if (quotedMessage) {
// eslint-disable-next-line no-param-reassign
await this.loadQuotedMessage(message, quotedMessage);
// Note: in the future when we generate our own thumbnail we won't need to rely
// on incoming thumbnail if we have our local message in hand.
if (!message.quotedMessage.imageUrl) {
await this.loadQuoteThumbnail(message, quote);
this.forceRender(message);
return;
}
// No need to go further if we already have a thumbnail
if (gotThumbnail) {
this.forceRender(message);
return;
}
@ -1292,22 +1314,9 @@
// eslint-disable-next-line no-param-reassign
message.quoteIsProcessed = true;
// 2. Go to the database for the real referenced attachment
// 3. As a last resort, go to the database to generate a thumbnail on-demand
const loaded = await this.loadQuotedMessageFromDatabase(message, id);
if (loaded) {
// Note: in the future when we generate our own thumbnail we won't need to rely
// on incoming thumbnail if we have our local message in hand.
if (!message.quotedMessageFromDatabase.imageUrl) {
await this.loadQuoteThumbnail(message, quote);
}
this.forceRender(message);
return;
}
// 3. Finally, use the provided thumbnail
const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
if (gotThumbnail) {
this.forceRender(message);
}
});

@ -181,10 +181,6 @@
URL.revokeObjectURL(this.quoteThumbnail.objectUrl);
this.quoteThumbnail = null;
}
if (this.quotedMessageFromDatabase) {
this.quotedMessageFromDatabase.unload();
this.quotedMessageFromDatabase = null;
}
if (this.quotedMessage) {
this.quotedMessage = null;
}

@ -1,36 +1,42 @@
/* eslint-disable */
/* global textsecure: false */
/* global Whisper: false */
/* global i18n: false */
/* global loadImage: false */
/* global Backbone: false */
/* global _: false */
/* global Signal: false */
// eslint-disable-next-line func-names
(function () {
'use strict';
window.Whisper = window.Whisper || {};
const { MIME } = window.Signal.Types;
Whisper.FileSizeToast = Whisper.ToastView.extend({
templateName: 'file-size-modal',
render_attributes: function() {
render_attributes() {
return {
'file-size-warning': i18n('fileSizeWarning'),
limit: this.model.limit,
units: this.model.units
units: this.model.units,
};
}
},
});
Whisper.UnsupportedFileTypeToast = Whisper.ToastView.extend({
template: i18n('unsupportedFileType')
template: i18n('unsupportedFileType'),
});
function makeThumbnail(size, objectUrl) {
return new Promise(function(resolve, reject) {
var img = document.createElement('img');
function makeImageThumbnail(size, objectUrl) {
return new Promise(((resolve, reject) => {
const img = document.createElement('img');
img.onerror = reject;
img.onload = function () {
img.onload = () => {
// using components/blueimp-load-image
// first, make the correct size
var canvas = loadImage.scale(img, {
let canvas = loadImage.scale(img, {
canvas: true,
cover: true,
maxWidth: size,
@ -49,20 +55,75 @@
minHeight: size,
});
var blob = window.dataURLToBlobSync(canvas.toDataURL('image/png'));
const blob = window.dataURLToBlobSync(canvas.toDataURL('image/png'));
resolve(blob);
};
img.src = objectUrl;
}));
}
function makeVideoScreenshot(objectUrl) {
return new Promise(((resolve, reject) => {
const video = document.createElement('video');
function capture() {
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
const image = window.dataURLToBlobSync(canvas.toDataURL('image/png'));
video.removeEventListener('canplay', capture);
resolve(image);
}
video.addEventListener('canplay', capture);
video.addEventListener('error', (error) => {
console.log(
'makeVideoThumbnail error',
Signal.Types.Errors.toLogFormat(error)
);
reject(error);
});
video.src = objectUrl;
}));
}
function blobToArrayBuffer(blob) {
return new Promise((resolve, reject) => {
const fileReader = new FileReader();
fileReader.onload = e => resolve(e.target.result);
fileReader.onerror = reject;
fileReader.onabort = reject;
fileReader.readAsArrayBuffer(blob);
});
}
async function makeVideoThumbnail(size, videoObjectUrl) {
const blob = await makeVideoScreenshot(videoObjectUrl);
const data = await blobToArrayBuffer(blob);
const screenshotObjectUrl = Signal.Util.arrayBufferToObjectURL({
data,
type: 'image/png',
});
const thumbnail = await makeImageThumbnail(size, screenshotObjectUrl);
URL.revokeObjectURL(screenshotObjectUrl);
return thumbnail;
}
Whisper.FileInputView = Backbone.View.extend({
tagName: 'span',
className: 'file-input',
initialize: function(options) {
initialize(options) {
this.$input = this.$('input[type=file]');
this.$input.click(function(e) {
this.$input.click((e) => {
e.stopPropagation();
});
this.thumb = new Whisper.AttachmentPreviewView();
@ -75,76 +136,81 @@
'change .choose-file': 'previewImages',
'click .close': 'deleteFiles',
'click .choose-file': 'open',
'drop': 'openDropped',
'dragover': 'showArea',
'dragleave': 'hideArea',
'paste': 'onPaste'
drop: 'openDropped',
dragover: 'showArea',
dragleave: 'hideArea',
paste: 'onPaste',
},
open: function(e) {
open(e) {
e.preventDefault();
// hack
if (this.window && this.window.chrome && this.window.chrome.fileSystem) {
this.window.chrome.fileSystem.chooseEntry({type: 'openFile'}, function(entry) {
this.window.chrome.fileSystem.chooseEntry({ type: 'openFile' }, (entry) => {
if (!entry) {
return;
}
entry.file(function(file) {
entry.file((file) => {
this.file = file;
this.previewImages();
}.bind(this));
}.bind(this));
});
});
} else {
this.$input.click();
}
},
addThumb: function(src) {
addThumb(src, options = {}) {
_.defaults(options, { addPlayIcon: false });
this.$('.avatar').hide();
this.thumb.src = src;
this.$('.attachment-previews').append(this.thumb.render().el);
this.thumb.$('img')[0].onload = function() {
if (options.addPlayIcon) {
this.$el.addClass('video-attachment');
} else {
this.$el.removeClass('video-attachment');
}
this.thumb.$('img')[0].onload = () => {
this.$el.trigger('force-resize');
}.bind(this);
};
},
autoScale: function(file) {
if (file.type.split('/')[0] !== 'image'
|| file.type === 'image/gif'
|| file.type === 'image/tiff') {
autoScale(file) {
if (file.type.split('/')[0] !== 'image' ||
file.type === 'image/gif' ||
file.type === 'image/tiff') {
// nothing to do
return Promise.resolve(file);
}
return new Promise(function(resolve, reject) {
var url = URL.createObjectURL(file);
var img = document.createElement('img');
return new Promise(((resolve, reject) => {
const url = URL.createObjectURL(file);
const img = document.createElement('img');
img.onerror = reject;
img.onload = function () {
img.onload = () => {
URL.revokeObjectURL(url);
var maxSize = 6000 * 1024;
var maxHeight = 4096;
var maxWidth = 4096;
if (img.width <= maxWidth && img.height <= maxHeight &&
file.size <= maxSize) {
const maxSize = 6000 * 1024;
const maxHeight = 4096;
const maxWidth = 4096;
if (img.width <= maxWidth && img.height <= maxHeight && file.size <= maxSize) {
resolve(file);
return;
}
var canvas = loadImage.scale(img, {
canvas: true, maxWidth: maxWidth, maxHeight: maxHeight
const canvas = loadImage.scale(img, {
canvas: true, maxWidth, maxHeight,
});
var quality = 0.95;
var i = 4;
var blob;
let quality = 0.95;
let i = 4;
let blob;
do {
i = i - 1;
blob = window.dataURLToBlobSync(
canvas.toDataURL('image/jpeg', quality)
);
quality = quality * maxSize / blob.size;
i -= 1;
blob = window.dataURLToBlobSync(canvas.toDataURL('image/jpeg', quality));
quality = (quality * maxSize) / blob.size;
// NOTE: During testing with a large image, we observed the
// `quality` value being > 1. Should we clamp it to [0.5, 1.0]?
// See: https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toBlob#Syntax
@ -156,44 +222,59 @@
resolve(blob);
};
img.src = url;
});
}));
},
previewImages: function() {
async previewImages() {
this.clearForm();
var file = this.file || this.$input.prop('files')[0];
if (!file) { return; }
var type = file.type.split('/')[0];
if (file.type === 'image/tiff') {
type = 'file';
const file = this.file || this.$input.prop('files')[0];
if (!file) {
return;
}
switch (type) {
case 'audio': this.addThumb('images/audio.svg'); break;
case 'video': this.addThumb('images/video.svg'); break;
case 'image':
const contentType = file.type;
const renderVideoPreview = async () => {
// we use the variable on this here to ensure cleanup if we're interrupted
this.previewObjectUrl = URL.createObjectURL(file);
const thumbnail = await makeVideoScreenshot(this.previewObjectUrl);
URL.revokeObjectURL(this.previewObjectUrl);
const data = await blobToArrayBuffer(thumbnail);
this.previewObjectUrl = Signal.Util.arrayBufferToObjectURL({
data,
type: 'image/png',
});
this.addThumb(this.previewObjectUrl, { addPlayIcon: true });
};
const renderImagePreview = async () => {
if (!MIME.isJPEG(file.type)) {
this.previewObjectUrl = URL.createObjectURL(file);
this.addThumb(this.previewObjectUrl);
break;
return;
}
// NOTE: Temporarily allow `then` until we convert the entire file
// to `async` / `await`:
// eslint-disable-next-line more/no-then
window.autoOrientImage(file)
.then(dataURL => this.addThumb(dataURL));
break;
default:
this.addThumb('images/file.svg'); break;
const dataUrl = await window.autoOrientImage(file);
this.addThumb(dataUrl);
};
if (Signal.Util.GoogleChrome.isImageTypeSupported(contentType)) {
renderImagePreview();
} else if (Signal.Util.GoogleChrome.isVideoTypeSupported(contentType)) {
renderVideoPreview();
} else if (MIME.isAudio(contentType)) {
this.addThumb('images/audio.svg');
} else {
this.addThumb('images/file.svg');
}
// NOTE: Temporarily allow `then` until we convert the entire file
// to `async` / `await`:
// eslint-disable-next-line more/no-then
this.autoScale(file).then(function(blob) {
var limitKb = 1000000;
var blobType = file.type === 'image/gif' ? 'gif' : type;
const blob = await this.autoScale(file);
let limitKb = 1000000;
const blobType = file.type === 'image/gif'
? 'gif'
: contentType.split('/')[0];
switch (blobType) {
case 'image':
limitKb = 6000; break;
@ -206,31 +287,28 @@
default:
limitKb = 100000; break;
}
if ((blob.size/1024).toFixed(4) >= limitKb) {
var units = ['kB','MB','GB'];
var u = -1;
var limit = limitKb * 1000;
if ((blob.size / 1024).toFixed(4) >= limitKb) {
const units = ['kB', 'MB', 'GB'];
let u = -1;
let limit = limitKb * 1000;
do {
limit /= 1000;
++u;
u += 1;
} while (limit >= 1000 && u < units.length - 1);
var toast = new Whisper.FileSizeToast({
model: {limit: limit, units: units[u]}
const toast = new Whisper.FileSizeToast({
model: { limit, units: units[u] },
});
toast.$el.insertAfter(this.$el);
toast.render();
this.deleteFiles();
}
}.bind(this));
},
hasFiles: function() {
var files = this.file ? [this.file] : this.$input.prop('files');
hasFiles() {
const files = this.file ? [this.file] : this.$input.prop('files');
return files && files.length && files.length > 0;
},
/* eslint-enable */
/* jshint ignore:start */
getFiles() {
const files = this.file ? [this.file] : Array.from(this.$input.prop('files'));
const promise = Promise.all(files.map(file => this.getFile(file)));
@ -262,44 +340,45 @@
.then(this.readFile)
.then(setFlags(attachmentFlags));
},
/* jshint ignore:end */
/* eslint-disable */
getThumbnail: function() {
async getThumbnail() {
// Scale and crop an image to 256px square
var size = 256;
var file = this.file || this.$input.prop('files')[0];
if (file === undefined || file.type.split('/')[0] !== 'image' || file.type === 'image/gif') {
const size = 256;
const file = this.file || this.$input.prop('files')[0];
if (file === undefined ||
file.type.split('/')[0] !== 'image' ||
file.type === 'image/gif') {
// nothing to do
return Promise.resolve();
}
const objectUrl = URL.createObjectURL(file);
return makeThumbnail(256, file).then(function(arrayBuffer) {
URL.revokeObjectURL(url);
const arrayBuffer = await makeImageThumbnail(size, objectUrl);
URL.revokeObjectURL(objectUrl);
return this.readFile(arrayBuffer);
});
},
// File -> Promise Attachment
readFile: function(file) {
return new Promise(function(resolve, reject) {
var FR = new FileReader();
FR.onload = function(e) {
readFile(file) {
return new Promise(((resolve, reject) => {
const FR = new FileReader();
FR.onload = (e) => {
resolve({
data: e.target.result,
contentType: file.type,
fileName: file.name,
size: file.size
size: file.size,
});
};
FR.onerror = reject;
FR.onabort = reject;
FR.readAsArrayBuffer(file);
});
}));
},
clearForm: function() {
clearForm() {
if (this.previewObjectUrl) {
URL.revokeObjectURL(this.previewObjectUrl);
this.previewObjectUrl = null;
@ -310,7 +389,7 @@
this.$el.trigger('force-resize');
},
deleteFiles: function(e) {
deleteFiles(e) {
if (e) { e.stopPropagation(); }
this.clearForm();
this.$input.wrap('<form>').parent('form').trigger('reset');
@ -320,20 +399,22 @@
this.isVoiceNote = false;
},
openDropped: function(e) {
if (e.originalEvent.dataTransfer.types[0] != 'Files') {
openDropped(e) {
if (e.originalEvent.dataTransfer.types[0] !== 'Files') {
return;
}
e.stopPropagation();
e.preventDefault();
// eslint-disable-next-line prefer-destructuring
this.file = e.originalEvent.dataTransfer.files[0];
this.previewImages();
this.$el.removeClass('dropoff');
},
showArea: function(e) {
if (e.originalEvent.dataTransfer.types[0] != 'Files') {
showArea(e) {
if (e.originalEvent.dataTransfer.types[0] !== 'Files') {
return;
}
@ -342,8 +423,8 @@
this.$el.addClass('dropoff');
},
hideArea: function(e) {
if (e.originalEvent.dataTransfer.types[0] != 'Files') {
hideArea(e) {
if (e.originalEvent.dataTransfer.types[0] !== 'Files') {
return;
}
@ -351,10 +432,10 @@
e.preventDefault();
this.$el.removeClass('dropoff');
},
onPaste: function(e) {
var items = e.originalEvent.clipboardData.items;
var imgBlob = null;
for (var i = 0; i < items.length; i++) {
onPaste(e) {
const { items } = e.originalEvent.clipboardData;
let imgBlob = null;
for (let i = 0; i < items.length; i += 1) {
if (items[i].type.split('/')[0] === 'image') {
imgBlob = items[i].getAsFile();
}
@ -363,8 +444,10 @@
this.file = imgBlob;
this.previewImages();
}
}
},
});
Whisper.FileInputView.makeThumbnail = makeThumbnail;
})();
Whisper.FileInputView.makeImageThumbnail = makeImageThumbnail;
Whisper.FileInputView.makeVideoThumbnail = makeVideoThumbnail;
Whisper.FileInputView.makeVideoScreenshot = makeVideoScreenshot;
}());

@ -927,6 +927,32 @@ span.status {
form.send {
background: #ffffff;
&.video-attachment {
.image-container {
position: relative;
}
.outer {
position: absolute;
top: 0;
right: 0;
bottom: 0;
left: 0;
text-align: center;
display: flex;
align-items: center;
justify-content: center;
.play.icon {
height: 30px;
width: 30px;
@include color-svg('../images/play.svg', white);
}
}
}
}
input, textarea {
@ -935,8 +961,10 @@ span.status {
.attachment-previews {
padding: 0 36px;
.attachment-preview {
padding: 13px 10px 0;
}
img {
border: 2px solid #ddd;
@ -1122,6 +1150,7 @@ span.status {
background-color: white;
border: none;
box-shadow: 0px 3px 5px 0px rgba(0,0,0,0.2);
outline: none;
.icon {
@include color-svg('../images/down.svg', $grey_l3);

Loading…
Cancel
Save