fix up types for link preview generations

ts/components/conversation/SessionStagedLinkPreview.tsx

@@ -6,7 +6,7 @@ import { arrayBufferFromFile } from '../../types/Attachment';
 import { AttachmentUtil, LinkPreviewUtil } from '../../util';
 import { fetchLinkPreviewImage } from '../../util/linkPreviewFetch';
 import { StagedLinkPreview } from './StagedLinkPreview';
-import { getImageDimensions, THUMBNAIL_SIDE } from '../../types/attachments/VisualAttachment';
+import { getImageDimensions } from '../../types/attachments/VisualAttachment';
 
 export interface StagedLinkPreviewProps extends StagedLinkPreviewData {
   onClose: (url: string) => void;
@@ -62,15 +62,12 @@ export const getPreview = async (
 
       // Ensure that this file is either small enough or is resized to meet our
       //   requirements for attachments
-      const withBlob = await AttachmentUtil.autoScale(
-        {
-          contentType: fullSizeImage.contentType,
-          blob: new Blob([fullSizeImage.data], {
-            type: fullSizeImage.contentType,
-          }),
-        },
-        { maxSide: THUMBNAIL_SIDE, maxSize: 100 * 1000 } // this is a preview image. No need for it to be crazy big. 100k is big enough
-      );
+      const withBlob = await AttachmentUtil.autoScaleForThumbnail({
+        contentType: fullSizeImage.contentType,
+        blob: new Blob([fullSizeImage.data], {
+          type: fullSizeImage.contentType,
+        }),
+      });
 
       const data = await arrayBufferFromFile(withBlob.blob);
       objectUrl = URL.createObjectURL(withBlob.blob);
@@ -115,7 +112,7 @@ export const SessionStagedLinkPreview = (props: StagedLinkPreviewProps) => {
       title={props.title}
       domain={props.domain}
       url={props.url}
-      image={props.image as any}
+      image={props.image}
     />
   );
 };

ts/components/conversation/StagedLinkPreview.tsx

@@ -3,15 +3,17 @@ import classNames from 'classnames';
 
 import { Image } from './Image';
 
-import { AttachmentType, isImageAttachment } from '../../types/Attachment';
 import { SessionSpinner } from '../basic/SessionSpinner';
+import { StagedLinkPreviewImage } from './composition/CompositionBox';
+import { isImage } from '../../types/MIME';
+import { fromArrayBufferToBase64 } from '../../session/utils/String';
 
 type Props = {
   isLoaded: boolean;
   title: null | string;
   url: null | string;
   domain: null | string;
-  image?: AttachmentType;
+  image?: StagedLinkPreviewImage;
 
   onClose: (url: string) => void;
 };
@@ -19,13 +21,17 @@ type Props = {
 export const StagedLinkPreview = (props: Props) => {
   const { isLoaded, onClose, title, image, domain, url } = props;
 
-  const isImage = image && isImageAttachment(image);
+  const isContentTypeImage = image && isImage(image.contentType);
   if (isLoaded && !(title && domain)) {
     return null;
   }
 
   const isLoading = !isLoaded;
 
+  const dataToRender = image?.data
+    ? `data:image/jpeg;base64, ${fromArrayBufferToBase64(image?.data)}`
+    : '';
+
   return (
     <div
       className={classNames(
@@ -34,15 +40,15 @@ export const StagedLinkPreview = (props: Props) => {
       )}
     >
       {isLoading ? <SessionSpinner loading={isLoading} /> : null}
-      {isLoaded && image && isImage ? (
+      {isLoaded && image && isContentTypeImage ? (
         <div className="module-staged-link-preview__icon-container">
           <Image
             alt={window.i18n('stagedPreviewThumbnail', [domain || ''])}
             softCorners={true}
             height={72}
             width={72}
-            url={image.url}
-            attachment={image}
+            url={dataToRender}
+            attachment={image as any}
           />
         </div>
       ) : null}

ts/components/conversation/composition/CompositionBox.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import _, { debounce } from 'lodash';
+import _, { debounce, isEmpty } from 'lodash';
 
 import * as MIME from '../../../types/MIME';
 
@@ -46,7 +46,10 @@ import { Flex } from '../../basic/Flex';
 import { CaptionEditor } from '../../CaptionEditor';
 import { StagedAttachmentList } from '../StagedAttachmentList';
 import { processNewAttachment } from '../../../types/MessageAttachment';
-import { StagedAttachmentImportedType } from '../../../util/attachmentsUtil';
+import {
+  StagedAttachmentImportedType,
+  StagedPreviewImportedType,
+} from '../../../util/attachmentsUtil';
 
 export interface ReplyingToMessageProps {
   convoId: string;
@@ -57,12 +60,20 @@ export interface ReplyingToMessageProps {
   attachments?: Array<any>;
 }
 
+export type StagedLinkPreviewImage = {
+  data: ArrayBuffer;
+  size: number;
+  width: number;
+  height: number;
+  contentType: string;
+};
+
 export interface StagedLinkPreviewData {
   isLoaded: boolean;
   title: string | null;
   url: string | null;
   domain: string | null;
-  image?: AttachmentType;
+  image?: StagedLinkPreviewImage;
 }
 
 export interface StagedAttachmentType extends AttachmentType {
@@ -592,25 +603,6 @@ class CompositionBoxInner extends React.Component<Props, State> {
 
     getPreview(firstLink, abortController.signal)
       .then(ret => {
-        let image: AttachmentType | undefined;
-        if (ret) {
-          if (ret.image?.width) {
-            if (ret.image) {
-              const blob = new Blob([ret.image.data], {
-                type: ret.image.contentType,
-              });
-              const imageAttachment = {
-                ...ret.image,
-                url: URL.createObjectURL(blob),
-                fileName: 'preview',
-                fileSize: null,
-                screenshot: null,
-                thumbnail: null,
-              };
-              image = imageAttachment;
-            }
-          }
-        }
         // we finished loading the preview, and checking the abortConrtoller, we are still not aborted.
         // => update the staged preview
         if (this.linkPreviewAbortController && !this.linkPreviewAbortController.signal.aborted) {
@@ -620,7 +612,7 @@ class CompositionBoxInner extends React.Component<Props, State> {
               title: ret?.title || null,
               url: ret?.url || null,
               domain: (ret?.url && window.Signal.LinkPreviews.getDomain(ret.url)) || '',
-              image,
+              image: ret?.image,
             },
           });
         } else if (this.linkPreviewAbortController) {
@@ -827,20 +819,19 @@ class CompositionBoxInner extends React.Component<Props, State> {
       'attachments'
     );
 
-    // we consider that a link previews without a title at least is not a preview
-    const linkPreviews =
-      (stagedLinkPreview &&
-        stagedLinkPreview.isLoaded &&
-        stagedLinkPreview.title?.length && [_.pick(stagedLinkPreview, 'url', 'image', 'title')]) ||
-      [];
+    // we consider that a link preview without a title at least is not a preview
+    const linkPreview =
+      stagedLinkPreview?.isLoaded && stagedLinkPreview.title?.length
+        ? _.pick(stagedLinkPreview, 'url', 'image', 'title')
+        : undefined;
 
     try {
-      const attachments = await this.getFiles();
+      const { attachments, previews } = await this.getFiles(linkPreview);
       this.props.sendMessage({
         body: messagePlaintext,
         attachments: attachments || [],
         quote: extractedQuotedMessageProps,
-        preview: linkPreviews,
+        preview: previews,
         groupInvitation: undefined,
       });
 
@@ -867,20 +858,54 @@ class CompositionBoxInner extends React.Component<Props, State> {
   }
 
   // this function is called right before sending a message, to gather really the files behind attachments.
-  private async getFiles(): Promise<Array<StagedAttachmentImportedType>> {
+  private async getFiles(
+    linkPreview?: Pick<StagedLinkPreviewData, 'url' | 'title' | 'image'>
+  ): Promise<{
+    attachments: Array<StagedAttachmentImportedType>;
+    previews: Array<StagedPreviewImportedType>;
+  }> {
     const { stagedAttachments } = this.props;
 
+    let attachments: Array<StagedAttachmentImportedType> = [];
+    let previews: Array<StagedPreviewImportedType> = [];
+
     if (_.isEmpty(stagedAttachments)) {
-      return [];
+      attachments = [];
+    } else {
+      // scale them down
+      const files = await Promise.all(stagedAttachments.map(AttachmentUtil.getFileAndStoreLocally));
+
+      attachments = _.compact(files);
     }
-    // scale them down
-    const files = await Promise.all(stagedAttachments.map(AttachmentUtil.getFileAndStoreLocally));
+
+    if (!linkPreview || _.isEmpty(linkPreview) || !linkPreview.url || !linkPreview.title) {
+      previews = [];
+    } else {
+      const sharedDetails = { url: linkPreview.url, title: linkPreview.title };
+      // store the first image preview locally and get the path and details back to include them in the message
+      const firstLinkPreviewImage = linkPreview.image;
+      if (firstLinkPreviewImage && !isEmpty(firstLinkPreviewImage)) {
+        const storedLinkPreviewAttachment = await AttachmentUtil.getFileAndStoreLocallyImageBuffer(
+          firstLinkPreviewImage.data
+        );
+        if (storedLinkPreviewAttachment) {
+          previews = [{ ...sharedDetails, image: storedLinkPreviewAttachment }];
+        } else {
+          // we couldn't save the image or whatever error happened, just return the url + title
+          previews = [sharedDetails];
+        }
+      } else {
+        // we did not fetch an image from the server
+        previews = [sharedDetails];
+      }
+    }
+
     window.inboxStore?.dispatch(
       removeAllStagedAttachmentsInConversation({
         conversationKey: this.props.selectedConversationKey,
       })
     );
-    return _.compact(files);
+    return { attachments, previews };
   }
 
   private async sendVoiceMessage(audioBlob: Blob) {

ts/types/MessageAttachment.ts

@@ -1,5 +1,5 @@
 import { remote } from 'electron';
-import { isArrayBuffer, isUndefined, omit } from 'lodash';
+import { isArrayBuffer, isUndefined, omit, isEmpty } from 'lodash';
 import {
   createAbsolutePathGetter,
   createDeleter,
@@ -112,46 +112,52 @@ export const getAttachmentPath = () => {
 
 export const loadAttachmentData = loadData;
 
-export const loadPreviewData = async (preview: any) => {
-  if (!preview || !preview.length) {
+export const loadPreviewData = async (preview: any): Promise<Array<any>> => {
+  if (!preview || !preview.length || isEmpty(preview[0])) {
     return [];
   }
 
-  return Promise.all(
-    preview.map(async (item: any) => {
-      if (!item.image) {
-        return item;
-      }
+  const firstPreview = preview[0];
+  if (!firstPreview.image) {
+    return [firstPreview];
+  }
 
-      return {
-        ...item,
-        image: await loadAttachmentData(item.image),
-      };
-    })
-  );
+  return [
+    {
+      ...firstPreview,
+      image: await loadAttachmentData(firstPreview.image),
+    },
+  ];
 };
 
 export const loadQuoteData = async (quote: any) => {
   if (!quote) {
     return null;
   }
+  console.warn('loadQuoteData', quote);
 
-  return {
-    ...quote,
-    attachments: await Promise.all(
-      (quote.attachments || []).map(async (attachment: any) => {
-        const { thumbnail } = attachment;
+  if (!quote.attachments?.length || isEmpty(quote.attachments[0])) {
+    return quote;
+  }
 
-        if (!thumbnail || !thumbnail.path) {
-          return attachment;
-        }
+  const quotedFirstAttachment = await quote.attachments[0];
 
-        return {
-          ...attachment,
-          thumbnail: await loadAttachmentData(thumbnail),
-        };
-      })
-    ),
+  const { thumbnail } = quotedFirstAttachment;
+
+  if (!thumbnail || !thumbnail.path) {
+    return {
+      ...quote,
+      attachments: [quotedFirstAttachment],
+    };
+  }
+  const quotedAttachmentWithThumbnail = {
+    ...quotedFirstAttachment,
+    thumbnail: await loadAttachmentData(thumbnail),
+  };
+
+  return {
+    ...quote,
+    attachments: [quotedAttachmentWithThumbnail],
   };
 };
 

ts/util/attachmentsUtil.ts

@@ -95,6 +95,10 @@ export async function autoScaleForIncomingAvatar(incomingAvatar: ArrayBuffer) {
   );
 }
 
+/**
+ * Auto scale an attachment to get a thumbnail from it. We consider that a thumbnail is currently at most 200 ko, is a square and has a maxSize of THUMBNAIL_SIDE
+ * @param attachment the attachment to auto scale
+ */
 export async function autoScaleForThumbnail<T extends { contentType: string; blob: Blob }>(
   attachment: T
 ) {
@@ -218,6 +222,24 @@ export type StagedAttachmentImportedType = Omit<
   'file' | 'url' | 'fileSize'
 > & { flags?: number };
 
+/**
+ * This is the type of the image of a link preview once it was saved in the attachment folder
+ */
+export type StagedImagePreviewImportedType = Pick<
+  StagedAttachmentType,
+  'contentType' | 'path' | 'size' | 'width' | 'height'
+>;
+
+/**
+ * This is the type of a complete preview imported in the app, hence with the image being a StagedImagePreviewImportedType.
+ * This is the one to be used in uploadData and which should be saved in the database message models
+ */
+export type StagedPreviewImportedType = {
+  url: string;
+  title: string;
+  image?: StagedImagePreviewImportedType;
+};
+
 export async function getFileAndStoreLocally(
   attachment: StagedAttachmentType
 ): Promise<StagedAttachmentImportedType | null> {
@@ -265,6 +287,35 @@ export async function getFileAndStoreLocally(
   };
 }
 
+export async function getFileAndStoreLocallyImageBuffer(imageBuffer: ArrayBuffer) {
+  if (!imageBuffer || !imageBuffer.byteLength) {
+    return null;
+  }
+
+  const contentType = imageType(new Uint8Array(imageBuffer))?.mime || IMAGE_UNKNOWN;
+
+  const blob = new Blob([imageBuffer], { type: contentType });
+
+  const scaled = await autoScaleForThumbnail({
+    contentType,
+    blob,
+  });
+
+  // this operation might change the file size, so be sure to rely on it on return here.
+  const attachmentSavedLocally = await processNewAttachment({
+    data: await scaled.blob.arrayBuffer(),
+    contentType: scaled.contentType,
+  });
+
+  return {
+    contentType: scaled.contentType,
+    path: attachmentSavedLocally.path,
+    width: scaled.width,
+    height: scaled.height,
+    size: attachmentSavedLocally.size,
+  };
+}
+
 export type AttachmentFileType = {
   attachment: any;
   data: ArrayBuffer;