add download of attachments for opengroupv2

ts/receiver/attachments.ts

@@ -5,6 +5,8 @@ import { saveMessage } from '../../ts/data/data';
 import { fromBase64ToArrayBuffer } from '../session/utils/String';
 import { AttachmentDownloads, AttachmentUtils } from '../session/utils';
 import { ConversationModel } from '../models/conversation';
+import { downloadFileOpenGroupV2 } from '../opengroup/opengroupV2/OpenGroupAPIV2';
+import { OpenGroupRequestCommonType } from '../opengroup/opengroupV2/ApiUtil';
 
 export async function downloadAttachment(attachment: any) {
   const serverUrl = new URL(attachment.url).origin;
@@ -83,75 +85,32 @@ export async function downloadAttachment(attachment: any) {
   };
 }
 
-export async function downloadAttachmentOpenGrouPV2(attachment: any) {
+export async function downloadAttachmentOpenGroupV2(
-  const serverUrl = new URL(attachment.url).origin;
+  attachment: any,
-
+  roomInfos: OpenGroupRequestCommonType
-  // The fileserver adds the `-static` part for some reason
+) {
-  const defaultFileserver = _.includes(
+  const dataUint = await downloadFileOpenGroupV2(attachment.id, roomInfos);
-    ['https://file-static.lokinet.org', 'https://file.getsession.org'],
-    serverUrl
-  );
-
-  let res: ArrayBuffer | null = null;
-
-  // TODO: we need attachments to remember which API should be used to retrieve them
-  if (!defaultFileserver) {
-    const serverAPI = await window.lokiPublicChatAPI.findOrCreateServer(serverUrl);
-
-    if (serverAPI) {
-      res = await serverAPI.downloadAttachment(attachment.url);
-    }
-  }
-
-  // Fallback to using the default fileserver
-  if (defaultFileserver || !res || res.byteLength === 0) {
-    res = await window.lokiFileServerAPI.downloadAttachment(attachment.url);
-  }
 
-  if (res.byteLength === 0) {
+  if (!dataUint?.length) {
     window.log.error('Failed to download attachment. Length is 0');
     throw new Error(`Failed to download attachment. Length is 0 for ${attachment.url}`);
   }
 
-  // FIXME "178" test to remove once this is fixed server side.
+  let data = dataUint;
-  if (!window.lokiFeatureFlags.useFileOnionRequestsV2) {
+  if (attachment.size !== dataUint.length) {
-    if (res.byteLength === 178) {
+    // we might have padding, check that all the remaining bytes are padding bytes
-      window.log.error(
+    // otherwise we have an error.
-        'Data of 178 length corresponds of a 404 returned as 200 by file.getsession.org.'
+    if (AttachmentUtils.isLeftOfBufferPaddingOnly(dataUint.buffer, attachment.size)) {
+      // we can safely remove the padding
+      data = data.slice(0, attachment.size);
+    } else {
+      throw new Error(
+        `downloadAttachment: Size ${attachment.size} did not match downloaded attachment size ${data.byteLength}`
       );
-      throw new Error(`downloadAttachment: invalid response for ${attachment.url}`);
     }
   } else {
-    // if useFileOnionRequestsV2 is true, we expect an ArrayBuffer not empty
+    // nothing to do, the attachment has already the correct size. There is just no padding included, which is bas
-  }
+    window.log.warn('Received opengroupv2 unpadded attachment');
-
-  // The attachment id is actually just the absolute url of the attachment
-  let data = res;
-  if (!attachment.isRaw) {
-    const { key, digest, size } = attachment;
-
-    if (!key || !digest) {
-      throw new Error('Attachment is not raw but we do not have a key to decode it');
-    }
-
-    data = await window.textsecure.crypto.decryptAttachment(
-      data,
-      fromBase64ToArrayBuffer(key),
-      fromBase64ToArrayBuffer(digest)
-    );
-
-    if (!size || size !== data.byteLength) {
-      // we might have padding, check that all the remaining bytes are padding bytes
-      // otherwise we have an error.
-      if (AttachmentUtils.isLeftOfBufferPaddingOnly(data, size)) {
-        // we can safely remove the padding
-        data = data.slice(0, size);
-      } else {
-        throw new Error(
-          `downloadAttachment: Size ${size} did not match downloaded attachment size ${data.byteLength}`
-        );
-      }
-    }
   }
 
   return {
@@ -166,13 +125,15 @@ async function processNormalAttachments(
   convo: ConversationModel
 ): Promise<number> {
   const isOpenGroupV2 = convo.isOpenGroupV2();
+  const openGroupV2Details = (isOpenGroupV2 && convo.toOpenGroupV2()) || undefined;
   const attachments = await Promise.all(
-    normalAttachments.map((attachment: any, index: any) => {
+    normalAttachments.map(async (attachment: any, index: any) => {
       return AttachmentDownloads.addJob(attachment, {
         messageId: message.id,
         type: 'attachment',
         index,
         isOpenGroupV2,
+        openGroupV2Details,
       });
     })
   );

ts/receiver/dataMessage.ts

@@ -359,7 +359,7 @@ export async function isMessageDuplicate({
     }
     const filteredResult = [result].filter((m: any) => m.attributes.body === message.body);
     if (serverId) {
-      return filteredResult.some(m => isDuplicate(m, { ...message, serverId }, source));
+      return filteredResult.some(m => isDuplicateServerId(m, { ...message, serverId }, source));
     }
     return filteredResult.some(m => isDuplicate(m, message, source));
   } catch (error) {
@@ -368,6 +368,25 @@ export async function isMessageDuplicate({
   }
 }
 
+/**
+ * This function is to be used to check for duplicates for open group v2 messages.
+ * It just check that the sender and the serverId of a received and an already saved messages are the same
+ */
+export const isDuplicateServerId = (
+  m: MessageModel,
+  testedMessage: MessageDuplicateSearchType,
+  source: string
+) => {
+  // The username in this case is the users pubKey
+  const sameUsername = m.attributes.source === source;
+  // testedMessage.id is needed as long as we support opengroupv1
+  const sameServerId =
+    m.attributes.serverId !== undefined &&
+    (testedMessage.serverId || testedMessage.id) === m.attributes.serverId;
+
+  return sameUsername && sameServerId;
+};
+
 export const isDuplicate = (
   m: MessageModel,
   testedMessage: MessageDuplicateSearchType,

ts/session/snode_api/onions.ts

@@ -5,6 +5,7 @@ import { Snode } from './snodePool';
 import ByteBuffer from 'bytebuffer';
 import { StringUtils } from '../utils';
 import { OnionPaths } from '../onions';
+import Long from 'long';
 
 export enum RequestError {
   BAD_PATH = 'BAD_PATH',
@@ -330,7 +331,13 @@ const processOnionResponse = async (
   }
 
   try {
-    const jsonRes: SnodeResponse = JSON.parse(plaintext);
+    const jsonRes: SnodeResponse = JSON.parse(plaintext, (key, value) => {
+      if (typeof value === 'number' && value > Number.MAX_SAFE_INTEGER) {
+        window.log.warn('Received an out of bounds js number');
+      }
+      return value;
+    });
+
     return jsonRes;
   } catch (e) {
     log.error(

ts/session/utils/AttachmentsDownload.ts

@@ -11,7 +11,7 @@ import {
   setAttachmentDownloadJobPending,
 } from '../../../ts/data/data';
 import { MessageModel } from '../../models/message';
-import { downloadAttachment } from '../../receiver/attachments';
+import { downloadAttachment, downloadAttachmentOpenGroupV2 } from '../../receiver/attachments';
 import { MessageController } from '../messages';
 
 const MAX_ATTACHMENT_JOB_PARALLELISM = 3;
@@ -128,8 +128,10 @@ async function _maybeStartJob() {
   }
 }
 
+// tslint:disable-next-line: cyclomatic-complexity
 async function _runJob(job: any) {
-  const { id, messageId, attachment, type, index, attempts, isOpenGroupV2 } = job || {};
+  const { id, messageId, attachment, type, index, attempts, isOpenGroupV2, openGroupV2Details } =
+    job || {};
   let message;
 
   try {
@@ -143,6 +145,16 @@ async function _runJob(job: any) {
       await _finishJob(null, id);
       return;
     }
+
+    if (isOpenGroupV2 && (!openGroupV2Details?.serverUrl || !openGroupV2Details.roomId)) {
+      window.log.warn(
+        'isOpenGroupV2 download attachment, but no valid openGroupV2Details given:',
+        openGroupV2Details
+      );
+      await _finishJob(null, id);
+      return;
+    }
+
     message = MessageController.getInstance().register(found.id, found);
 
     const pending = true;
@@ -151,7 +163,11 @@ async function _runJob(job: any) {
     let downloaded;
 
     try {
-      downloaded = await downloadAttachment(attachment);
+      if (isOpenGroupV2) {
+        downloaded = await downloadAttachmentOpenGroupV2(attachment, openGroupV2Details);
+      } else {
+        downloaded = await downloadAttachment(attachment);
+      }
     } catch (error) {
       // Attachments on the server expire after 60 days, then start returning 404
       if (error && error.code === 404) {

ts/session/utils/AttachmentsV2.ts

@@ -57,10 +57,9 @@ export async function uploadV2(params: UploadParamsV2): Promise<AttachmentPointe
     caption: attachment.caption,
   };
 
-  const paddedAttachment: ArrayBuffer =
+  const paddedAttachment: ArrayBuffer = window.lokiFeatureFlags.padOutgoingAttachments
-    (window.lokiFeatureFlags.padOutgoingAttachments &&
+    ? AttachmentUtils.addAttachmentPadding(attachment.data)
-      AttachmentUtils.addAttachmentPadding(attachment.data)) ||
+    : attachment.data;
-    attachment.data;
 
   const fileDetails = await uploadFileOpenGroupV2(new Uint8Array(paddedAttachment), openGroup);