fix: add handling of deleteMsgs and deleteAttachmentsBefore for groups

.github/workflows/pull-request.yml

@@ -6,7 +6,6 @@ on:
     branches:
       - clearnet
       - unstable
-      - unstable1
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

ts/data/data.ts

@@ -1,5 +1,6 @@
 // eslint:disable: no-require-imports no-var-requires one-variable-per-declaration no-void-expression function-name
 
+import { GroupPubkeyType } from 'libsession_util_nodejs';
 import _, { isEmpty } from 'lodash';
 import { MessageResultProps } from '../components/search/MessageSearchResults';
 import { ConversationModel } from '../models/conversation';
@@ -284,6 +285,20 @@ async function removeMessagesByIds(ids: Array<string>): Promise<void> {
   await channels.removeMessagesByIds(ids);
 }
 
+async function removeAllMessagesInConversationSentBefore(args: {
+  deleteBeforeSeconds: number;
+  conversationId: GroupPubkeyType;
+}): Promise<Array<string>> {
+  return channels.removeAllMessagesInConversationSentBefore(args);
+}
+
+async function removeAllAttachmentsInConversationSentBefore(args: {
+  deleteAttachBeforeSeconds: number;
+  conversationId: GroupPubkeyType;
+}): Promise<Array<string>> {
+  return channels.removeAllAttachmentsInConversationSentBefore(args);
+}
+
 async function getMessageIdsFromServerIds(
   serverIds: Array<string> | Array<number>,
   conversationId: string
@@ -823,6 +838,8 @@ export const Data = {
   saveMessages,
   removeMessage,
   removeMessagesByIds,
+  removeAllMessagesInConversationSentBefore,
+  removeAllAttachmentsInConversationSentBefore,
   cleanUpExpirationTimerUpdateHistory,
   getMessageIdsFromServerIds,
   getMessageById,

ts/data/dataInit.ts

@@ -41,6 +41,8 @@ const channelsToMake = new Set([
   'saveMessages',
   'removeMessage',
   'removeMessagesByIds',
+  'removeAllAttachmentsInConversationSentBefore',
+  'removeAllMessagesInConversationSentBefore',
   'cleanUpExpirationTimerUpdateHistory',
   'getUnreadByConversation',
   'getUnreadDisappearingByConversation',

ts/node/attachment_channel.ts

@@ -5,7 +5,7 @@ import { isString, map } from 'lodash';
 import path from 'path';
 import rimraf from 'rimraf';
 
-import { createDeleter, getAttachmentsPath } from '../shared/attachments/shared_attachments';
+import { getAttachmentsPath } from '../shared/attachments/shared_attachments';
 import { sqlNode } from './sql'; // checked - only node
 
 let initialized = false;
@@ -21,24 +21,6 @@ const ensureDirectory = async (userDataPath: string) => {
   await fse.ensureDir(getAttachmentsPath(userDataPath));
 };
 
-const deleteAll = async ({
-  userDataPath,
-  attachments,
-}: {
-  userDataPath: string;
-  attachments: any;
-}) => {
-  const deleteFromDisk = createDeleter(getAttachmentsPath(userDataPath));
-
-  for (let index = 0, max = attachments.length; index < max; index += 1) {
-    const file = attachments[index];
-    // eslint-disable-next-line no-await-in-loop
-    await deleteFromDisk(file);
-  }
-
-  console.log(`deleteAll: deleted ${attachments.length} files`);
-};
-
 const getAllAttachments = async (userDataPath: string) => {
   const dir = getAttachmentsPath(userDataPath);
   const pattern = path.join(dir, '**', '*');
@@ -50,7 +32,7 @@ const getAllAttachments = async (userDataPath: string) => {
 async function cleanupOrphanedAttachments(userDataPath: string) {
   const allAttachments = await getAllAttachments(userDataPath);
   const orphanedAttachments = sqlNode.removeKnownAttachments(allAttachments);
-  await deleteAll({
+  await sqlNode.deleteAll({
     userDataPath,
     attachments: orphanedAttachments,
   });

ts/node/sql.ts

@@ -24,6 +24,7 @@ import {
   uniq,
 } from 'lodash';
 
+import { GroupPubkeyType } from 'libsession_util_nodejs';
 import { ConversationAttributes } from '../models/conversationAttributes';
 import { PubKey } from '../session/types/PubKey'; // checked - only node
 import { redactAll } from '../util/privacy'; // checked - only node
@@ -66,6 +67,7 @@ import { MessageAttributes } from '../models/messageType';
 import { SignalService } from '../protobuf';
 import { Quote } from '../receiver/types';
 import { DURATION } from '../session/constants';
+import { createDeleter, getAttachmentsPath } from '../shared/attachments/shared_attachments';
 import {
   getSQLCipherIntegrityCheck,
   openAndMigrateDatabase,
@@ -990,6 +992,62 @@ function removeMessagesByIds(ids: Array<string>, instance?: BetterSqlite3.Databa
   console.log(`removeMessagesByIds of length ${ids.length} took ${Date.now() - start}ms`);
 }
 
+function removeAllMessagesInConversationSentBefore(
+  {
+    deleteBeforeSeconds,
+    conversationId,
+  }: { deleteBeforeSeconds: number; conversationId: GroupPubkeyType },
+  instance?: BetterSqlite3.Database
+) {
+  const msgIds = assertGlobalInstanceOrInstance(instance)
+    .prepare(
+      `SELECT id FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId AND sent_at <= $beforeMs;`
+    )
+    .all({ conversationId, beforeMs: deleteBeforeSeconds * 1000 }) as Array<string>;
+
+  assertGlobalInstanceOrInstance(instance)
+    .prepare(
+      `DELETE FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId AND sent_at <= $beforeMs;`
+    )
+    .run({ conversationId, beforeMs: deleteBeforeSeconds * 1000 });
+  console.info('removeAllMessagesInConversationSentBefore deleted msgIds:', JSON.stringify(msgIds));
+  return msgIds;
+}
+
+async function removeAllAttachmentsInConversationSentBefore(
+  {
+    deleteAttachBeforeSeconds,
+    conversationId,
+  }: { deleteAttachBeforeSeconds: number; conversationId: GroupPubkeyType },
+  instance?: BetterSqlite3.Database
+) {
+  const rows = assertGlobalInstanceOrInstance(instance)
+    .prepare(
+      `SELECT json FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId AND sent_at <= $beforeMs`
+    )
+    .all({ conversationId, beforeMs: deleteAttachBeforeSeconds * 1000 });
+  const messages = map(rows, row => jsonToObject(row.json));
+
+  const externalFiles: Array<string> = [];
+  const msgIdsWithChanges: Array<string> = [];
+  forEach(messages, message => {
+    const externalFilesMsg = getExternalFilesForMessage(message);
+    if (externalFilesMsg.length) {
+      externalFiles.push(...externalFilesMsg);
+      msgIdsWithChanges.push();
+    }
+  });
+  const uniqPathsToRemove = uniq(externalFiles);
+  console.info('removeAllAttachmentsInConversationSentBefore removing attachments:', externalFiles);
+  console.info(
+    'removeAllAttachmentsInConversationSentBefore impacted msgIds:',
+    JSON.stringify(msgIdsWithChanges)
+  );
+
+  const userDataPath = app.getPath('userData');
+  await deleteAll({ userDataPath, attachments: uniqPathsToRemove });
+}
+
 function removeAllMessagesInConversation(
   conversationId: string,
   instance?: BetterSqlite3.Database
@@ -1890,7 +1948,7 @@ function getMessagesWithFileAttachments(conversationId: string, limit: number) {
 
 function getExternalFilesForMessage(message: any) {
   const { attachments, quote, preview } = message;
-  const files: Array<any> = [];
+  const files: Array<string> = [];
 
   forEach(attachments, attachment => {
     const { path: file, thumbnail, screenshot } = attachment;
@@ -1954,6 +2012,24 @@ function getExternalFilesForConversation(
   return files;
 }
 
+async function deleteAll({
+  userDataPath,
+  attachments,
+}: {
+  userDataPath: string;
+  attachments: Array<string>;
+}) {
+  const deleteFromDisk = createDeleter(getAttachmentsPath(userDataPath));
+
+  for (let index = 0, max = attachments.length; index < max; index += 1) {
+    const file = attachments[index];
+    // eslint-disable-next-line no-await-in-loop
+    await deleteFromDisk(file);
+  }
+
+  console.log(`deleteAll: deleted ${attachments.length} files`);
+}
+
 function removeKnownAttachments(allAttachments: Array<string>) {
   const lookup = fromPairs(map(allAttachments, file => [file, true]));
   const chunkSize = 50;
@@ -2476,6 +2552,8 @@ export const sqlNode = {
   saveMessages,
   removeMessage,
   removeMessagesByIds,
+  removeAllMessagesInConversationSentBefore,
+  removeAllAttachmentsInConversationSentBefore,
   cleanUpExpirationTimerUpdateHistory,
   removeAllMessagesInConversation,
   getUnreadByConversation,
@@ -2513,7 +2591,7 @@ export const sqlNode = {
   removeAttachmentDownloadJob,
   removeAllAttachmentDownloadJobs,
   removeKnownAttachments,
-
+  deleteAll,
   removeAll,
 
   getMessagesWithVisualMediaAttachments,

ts/node/sql_channel.ts

@@ -1,7 +1,7 @@
 import { app, ipcMain } from 'electron';
-import { sqlNode } from './sql'; // checked - only node
-import { userConfig } from './config/user_config'; // checked - only node
 import { ephemeralConfig } from './config/ephemeral_config'; // checked - only node
+import { userConfig } from './config/user_config'; // checked - only node
+import { sqlNode } from './sql'; // checked - only node
 
 let initialized = false;
 
@@ -13,14 +13,15 @@ export function initializeSqlChannel() {
     throw new Error('sqlChannels: already initialized!');
   }
 
-  ipcMain.on(SQL_CHANNEL_KEY, (event, jobId, callName, ...args) => {
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => {
     try {
       const fn = (sqlNode as any)[callName];
       if (!fn) {
         throw new Error(`sql channel: ${callName} is not an available function`);
       }
 
-      const result = fn(...args);
+      const result = await fn(...args);
 
       event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result);
     } catch (error) {

ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts

@@ -1,4 +1,6 @@
 import { GroupPubkeyType } from 'libsession_util_nodejs';
+import { isFinite, isNumber } from 'lodash';
+import { Data } from '../../../../data/data';
 import { groupInfoActions } from '../../../../state/ducks/metaGroups';
 import { MetaGroupWrapperActions } from '../../../../webworker/workers/browser/libsession_worker_interface';
 import { ed25519Str } from '../../../onions/onionPath';
@@ -7,6 +9,37 @@ import { LibSessionUtil } from '../../../utils/libsession/libsession_utils';
 import { SnodeNamespaces } from '../namespaces';
 import { RetrieveMessageItemWithNamespace } from '../types';
 
+async function handleMetaMergeResults(groupPk: GroupPubkeyType) {
+  const infos = await MetaGroupWrapperActions.infoGet(groupPk);
+  if (
+    infos &&
+    isNumber(infos.deleteBeforeSeconds) &&
+    isFinite(infos.deleteBeforeSeconds) &&
+    infos.deleteBeforeSeconds > 0
+  ) {
+    // delete any messages in this conversation sent before that timestamp (in seconds)
+    const deletedMsgIds = await Data.removeAllMessagesInConversationSentBefore({
+      deleteBeforeSeconds: infos.deleteBeforeSeconds,
+      conversationId: groupPk,
+    });
+    console.warn('deletedMsgIds', deletedMsgIds);
+  }
+
+  if (
+    infos &&
+    isNumber(infos.deleteAttachBeforeSeconds) &&
+    isFinite(infos.deleteAttachBeforeSeconds) &&
+    infos.deleteAttachBeforeSeconds > 0
+  ) {
+    // delete any attachments in this conversation sent before that timestamp (in seconds)
+    const impactedMsgids = await Data.removeAllAttachmentsInConversationSentBefore({
+      deleteAttachBeforeSeconds: infos.deleteAttachBeforeSeconds,
+      conversationId: groupPk,
+    });
+    console.warn('impactedMsgids', impactedMsgids);
+  }
+}
+
 async function handleGroupSharedConfigMessages(
   groupConfigMessages: Array<RetrieveMessageItemWithNamespace>,
   groupPk: GroupPubkeyType
@@ -54,6 +87,8 @@ async function handleGroupSharedConfigMessages(
     // do the merge with our current state
     await MetaGroupWrapperActions.metaMerge(groupPk, toMerge);
 
+    await handleMetaMergeResults(groupPk);
+
     // save updated dumps to the DB right away
     await LibSessionUtil.saveDumpsToDb(groupPk);