From 315b4b4e4c570bf53932d8d85697dd54b4f4e68d Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Wed, 4 Dec 2024 15:47:27 +1100 Subject: [PATCH] fix: add a way to empty all seenHashes & lastHashes for a pukbey --- ts/data/data.ts | 17 +++--- ts/data/dataInit.ts | 1 + ts/interactions/conversationInteractions.ts | 2 + ts/node/database_utility.ts | 1 + ts/node/migration/sessionMigrations.ts | 22 +++++++ ts/node/migration/signalMigrations.ts | 3 +- ts/node/sql.ts | 58 +++++++++++++------ ts/receiver/configMessage.ts | 1 + .../libsession/handleLibSessionMessage.ts | 1 + ts/session/apis/snode_api/swarmPolling.ts | 29 +++++++--- .../SwarmPollingGroupConfig.ts | 1 + .../conversations/ConversationController.ts | 12 +++- ts/session/sending/MessageSender.ts | 7 ++- ts/state/ducks/metaGroups.ts | 1 + ts/types/sqlSharedTypes.ts | 2 + 15 files changed, 119 insertions(+), 39 deletions(-) diff --git a/ts/data/data.ts b/ts/data/data.ts index fab909d6f..99ce43b28 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -18,6 +18,7 @@ import { AsyncWrapper, MsgDuplicateSearchOpenGroup, SaveConversationReturn, + SaveSeenMessageHash, UnprocessedDataNode, UpdateLastHashType, } from '../types/sqlSharedTypes'; @@ -215,17 +216,16 @@ async function cleanLastHashes(): Promise { await channels.cleanLastHashes(); } -export type SeenMessageHashes = { - expiresAt: number; - hash: string; -}; - -async function saveSeenMessageHashes(data: Array): Promise { +async function saveSeenMessageHashes(data: Array): Promise { await channels.saveSeenMessageHashes(cleanData(data)); } -async function clearLastHashesForConvoId(data: { convoId: string }): Promise { - await channels.clearLastHashesForConvoId(cleanData(data)); +async function clearLastHashesForConvoId(conversationId: string): Promise { + await channels.clearLastHashesForConvoId(conversationId); +} + +async function emptySeenMessageHashesForConversation(conversationId: string): Promise { + await channels.emptySeenMessageHashesForConversation(conversationId); } async function updateLastHash(data: UpdateLastHashType): Promise { @@ -867,6 +867,7 @@ export const Data = { cleanLastHashes, clearLastHashesForConvoId, saveSeenMessageHashes, + emptySeenMessageHashesForConversation, updateLastHash, saveMessage, saveMessages, diff --git a/ts/data/dataInit.ts b/ts/data/dataInit.ts index 473a78b43..2696fe343 100644 --- a/ts/data/dataInit.ts +++ b/ts/data/dataInit.ts @@ -40,6 +40,7 @@ const channelsToMake = new Set([ 'updateLastHash', 'clearLastHashesForConvoId', 'saveSeenMessageHashes', + 'emptySeenMessageHashesForConversation', 'saveMessages', 'removeMessage', 'removeMessagesByIds', diff --git a/ts/interactions/conversationInteractions.ts b/ts/interactions/conversationInteractions.ts index 3efec954e..26d5b20fb 100644 --- a/ts/interactions/conversationInteractions.ts +++ b/ts/interactions/conversationInteractions.ts @@ -242,6 +242,7 @@ export async function declineConversationWithoutConfirm({ forceDestroyForAllMembers: false, fromSyncMessage: false, sendLeaveMessage: false, + clearFetchedHashes: false, }); } @@ -435,6 +436,7 @@ async function leaveGroupOrCommunityByConvoId({ deleteAllMessagesOnSwarm: false, deletionType: 'doNotKeep', forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } await clearConversationInteractionState({ conversationId }); diff --git a/ts/node/database_utility.ts b/ts/node/database_utility.ts index 157f6d745..329394094 100644 --- a/ts/node/database_utility.ts +++ b/ts/node/database_utility.ts @@ -17,6 +17,7 @@ export const ITEMS_TABLE = 'items'; export const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; export const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; export const LAST_HASHES_TABLE = 'lastHashes'; +export const SEEN_MESSAGE_TABLE = 'seenMessages'; export const HEX_KEY = /[^0-9A-Fa-f]/; diff --git a/ts/node/migration/sessionMigrations.ts b/ts/node/migration/sessionMigrations.ts index 158a5f38d..54dc29800 100644 --- a/ts/node/migration/sessionMigrations.ts +++ b/ts/node/migration/sessionMigrations.ts @@ -19,6 +19,7 @@ import { MESSAGES_TABLE, NODES_FOR_PUBKEY_TABLE, OPEN_GROUP_ROOMS_V2_TABLE, + SEEN_MESSAGE_TABLE, dropFtsAndTriggers, objectToJSON, rebuildFtsTable, @@ -106,6 +107,7 @@ const LOKI_SCHEMA_VERSIONS = [ updateToSessionSchemaVersion36, updateToSessionSchemaVersion37, updateToSessionSchemaVersion38, + updateToSessionSchemaVersion39, ]; function updateToSessionSchemaVersion1(currentVersion: number, db: BetterSqlite3.Database) { @@ -2003,6 +2005,26 @@ function updateToSessionSchemaVersion38(currentVersion: number, db: BetterSqlite console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); } +function updateToSessionSchemaVersion39(currentVersion: number, db: BetterSqlite3.Database) { + const targetVersion = 39; + if (currentVersion >= targetVersion) { + return; + } + + console.log(`updateToSessionSchemaVersion${targetVersion}: starting...`); + + db.transaction(() => { + db.exec(`ALTER TABLE ${SEEN_MESSAGE_TABLE} ADD COLUMN conversationId TEXT;`); + + db.exec(`CREATE INDEX seen_hashes_per_pubkey ON ${SEEN_MESSAGE_TABLE} ( + conversationId + );`); + writeSessionSchemaVersion(targetVersion, db); + })(); + + console.log(`updateToSessionSchemaVersion${targetVersion}: success!`); +} + export function printTableColumns(table: string, db: BetterSqlite3.Database) { console.info(db.pragma(`table_info('${table}');`)); } diff --git a/ts/node/migration/signalMigrations.ts b/ts/node/migration/signalMigrations.ts index 6d7bddcd7..fb01760b9 100644 --- a/ts/node/migration/signalMigrations.ts +++ b/ts/node/migration/signalMigrations.ts @@ -11,6 +11,7 @@ import { LAST_HASHES_TABLE, MESSAGES_FTS_TABLE, MESSAGES_TABLE, + SEEN_MESSAGE_TABLE, } from '../database_utility'; import { getAppRootPath } from '../getRootPath'; import { updateSessionSchema } from './sessionMigrations'; @@ -245,7 +246,7 @@ function updateToSchemaVersion6(currentVersion: number, db: BetterSqlite3.Databa expiresAt INTEGER ); - CREATE TABLE seenMessages( + CREATE TABLE ${SEEN_MESSAGE_TABLE}( hash TEXT PRIMARY KEY, expiresAt INTEGER ); diff --git a/ts/node/sql.ts b/ts/node/sql.ts index 2616dfbb9..f70f1f986 100644 --- a/ts/node/sql.ts +++ b/ts/node/sql.ts @@ -48,6 +48,7 @@ import { NODES_FOR_PUBKEY_TABLE, objectToJSON, OPEN_GROUP_ROOMS_V2_TABLE, + SEEN_MESSAGE_TABLE, toSqliteBoolean, } from './database_utility'; import type { SetupI18nReturnType } from '../types/localizer'; // checked - only node @@ -58,6 +59,7 @@ import { MsgDuplicateSearchOpenGroup, roomHasBlindEnabled, SaveConversationReturn, + SaveSeenMessageHash, UnprocessedDataNode, UnprocessedParameter, UpdateLastHashType, @@ -937,12 +939,21 @@ function saveMessage(data: MessageAttributes) { return id; } -function saveSeenMessageHashes(arrayOfHashes: Array) { +function saveSeenMessageHashes(arrayOfHashes: Array) { assertGlobalInstance().transaction(() => { map(arrayOfHashes, saveSeenMessageHash); })(); } +function emptySeenMessageHashesForConversation(conversationId: string) { + if (!isString(conversationId) || isEmpty(conversationId)) { + throw new Error('emptySeenMessageHashesForConversation: conversationId is not a string'); + } + assertGlobalInstance() + .prepare(`DELETE FROM ${SEEN_MESSAGE_TABLE} WHERE conversationId=$conversationId`) + .run({ conversationId }); +} + function updateLastHash(data: UpdateLastHashType) { const { convoId, snode, hash, expiresAt, namespace } = data; if (!isNumber(namespace)) { @@ -973,32 +984,43 @@ function updateLastHash(data: UpdateLastHashType) { }); } -function clearLastHashesForConvoId(data: { convoId: string }) { - const { convoId } = data; - if (!isString(convoId)) { - throw new Error('clearLastHashesForPubkey: convoId not a string'); +function clearLastHashesForConvoId(conversationId: string) { + if (!isString(conversationId) || isEmpty(conversationId)) { + throw new Error('clearLastHashesForConvoId: conversationId is not a string'); } - assertGlobalInstance().prepare(`DELETE FROM ${LAST_HASHES_TABLE} WHERE pubkey=$convoId;`).run({ - convoId, - }); + assertGlobalInstance() + .prepare(`DELETE FROM ${LAST_HASHES_TABLE} WHERE id=$conversationId`) + .run({ conversationId }); } -function saveSeenMessageHash(data: any) { - const { expiresAt, hash } = data; +function saveSeenMessageHash(data: SaveSeenMessageHash) { + const { expiresAt, hash, conversationId } = data; + if (!isString(conversationId)) { + throw new Error('saveSeenMessageHash conversationId must be a string'); + } + if (!isString(hash)) { + throw new Error('saveSeenMessageHash hash must be a string'); + } + if (!isNumber(expiresAt)) { + throw new Error('saveSeenMessageHash expiresAt must be a number'); + } try { assertGlobalInstance() .prepare( - `INSERT OR REPLACE INTO seenMessages ( + `INSERT OR REPLACE INTO ${SEEN_MESSAGE_TABLE} ( expiresAt, - hash + hash, + conversationId ) values ( $expiresAt, - $hash + $hash, + $conversationId );` ) .run({ expiresAt, hash, + conversationId, }); } catch (e) { console.error('saveSeenMessageHash failed:', e.message); @@ -1012,7 +1034,7 @@ function cleanLastHashes() { } function cleanSeenMessages() { - assertGlobalInstance().prepare('DELETE FROM seenMessages WHERE expiresAt <= $now;').run({ + assertGlobalInstance().prepare(`DELETE FROM ${SEEN_MESSAGE_TABLE} WHERE expiresAt <= $now;`).run({ now: Date.now(), }); } @@ -1748,7 +1770,9 @@ function getLastHashBySnode(convoId: string, snode: string, namespace: number) { function getSeenMessagesByHashList(hashes: Array) { const rows = assertGlobalInstance() - .prepare(`SELECT * FROM seenMessages WHERE hash IN ( ${hashes.map(() => '?').join(', ')} );`) + .prepare( + `SELECT * FROM ${SEEN_MESSAGE_TABLE} WHERE hash IN ( ${hashes.map(() => '?').join(', ')} );` + ) .all(hashes); return map(rows, row => row.hash); @@ -2008,7 +2032,7 @@ function removeAll() { DELETE FROM ${LAST_HASHES_TABLE}; DELETE FROM ${NODES_FOR_PUBKEY_TABLE}; DELETE FROM ${CLOSED_GROUP_V2_KEY_PAIRS_TABLE}; - DELETE FROM seenMessages; + DELETE FROM ${SEEN_MESSAGE_TABLE}; DELETE FROM ${CONVERSATIONS_TABLE}; DELETE FROM ${MESSAGES_TABLE}; DELETE FROM ${ATTACHMENT_DOWNLOADS_TABLE}; @@ -2657,7 +2681,7 @@ export const sqlNode = { cleanLastHashes, clearLastHashesForConvoId, saveSeenMessageHashes, - saveSeenMessageHash, + emptySeenMessageHashesForConversation, updateLastHash, saveMessages, removeMessage, diff --git a/ts/receiver/configMessage.ts b/ts/receiver/configMessage.ts index e70c0378d..6e300ff29 100644 --- a/ts/receiver/configMessage.ts +++ b/ts/receiver/configMessage.ts @@ -759,6 +759,7 @@ async function handleSingleGroupUpdateToLeave(toLeave: GroupPubkeyType) { deletionType: 'doNotKeep', deleteAllMessagesOnSwarm: false, forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } catch (e) { window.log.info('Failed to deleteClosedGroup with: ', e.message); diff --git a/ts/receiver/libsession/handleLibSessionMessage.ts b/ts/receiver/libsession/handleLibSessionMessage.ts index e7975c5f3..d019bde01 100644 --- a/ts/receiver/libsession/handleLibSessionMessage.ts +++ b/ts/receiver/libsession/handleLibSessionMessage.ts @@ -57,6 +57,7 @@ async function handleLibSessionKickedMessage({ deletionType: inviteWasPending ? 'doNotKeep' : 'keepAsKicked', deleteAllMessagesOnSwarm: false, forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } diff --git a/ts/session/apis/snode_api/swarmPolling.ts b/ts/session/apis/snode_api/swarmPolling.ts index f38c61152..a5ea2ba03 100644 --- a/ts/session/apis/snode_api/swarmPolling.ts +++ b/ts/session/apis/snode_api/swarmPolling.ts @@ -489,7 +489,7 @@ export class SwarmPolling { const newMessages = await this.handleSeenMessages(uniqOtherMsgs); window.log.info( - `SwarmPolling: handleSeenMessages: ${newMessages.length} out of ${uniqOtherMsgs.length} are not seen yet. snode: ${toPollFrom ? ed25519Str(toPollFrom.pubkey_ed25519) : 'undefined'}` + `SwarmPolling: handleSeenMessages: ${newMessages.length} out of ${uniqOtherMsgs.length} are not seen yet about pk:${ed25519Str(pubkey)} snode: ${toPollFrom ? ed25519Str(toPollFrom.pubkey_ed25519) : 'undefined'}` ); if (type === ConversationTypeEnum.GROUPV2) { if (!PubKey.is03Pubkey(pubkey)) { @@ -510,7 +510,7 @@ export class SwarmPolling { // private and legacy groups are cached, so we can mark them as seen right away, they are still in the cache until processed correctly. // at some point we should get rid of the cache completely, and do the same logic as for groupv2 above - await this.updateSeenMessages(newMessages); + await this.updateSeenMessages(newMessages, pubkey); // trigger the handling of all the other messages, not shared config related and not groupv2 encrypted newMessages.forEach(m => { const extracted = extractWebSocketContent(m.data, m.hash); @@ -696,6 +696,7 @@ export class SwarmPolling { deletionType: 'doNotKeep', deleteAllMessagesOnSwarm: false, forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } } @@ -722,19 +723,22 @@ export class SwarmPolling { } const incomingHashes = messages.map((m: RetrieveMessageItem) => m.hash); - const dupHashes = await Data.getSeenMessagesByHashList(incomingHashes); const newMessages = messages.filter((m: RetrieveMessageItem) => !dupHashes.includes(m.hash)); return newMessages; } - private async updateSeenMessages(processedMessages: Array) { + private async updateSeenMessages( + processedMessages: Array, + conversationId: string + ) { if (processedMessages.length) { const newHashes = processedMessages.map((m: RetrieveMessageItem) => ({ // NOTE setting expiresAt will trigger the global function destroyExpiredMessages() on it's next interval expiresAt: m.expiration, hash: m.hash, + conversationId, })); await Data.saveSeenMessageHashes(newHashes); } @@ -822,6 +826,17 @@ export class SwarmPolling { return this.lastHashes[nodeEdKey][pubkey][namespace]; } + public async resetLastHashesForConversation(conversationId: string) { + await Data.clearLastHashesForConvoId(conversationId); + const snodeKeys = Object.keys(this.lastHashes); + for (let index = 0; index < snodeKeys.length; index++) { + const snodeKey = snodeKeys[index]; + if (!isEmpty(this.lastHashes[snodeKey][conversationId])) { + this.lastHashes[snodeKey][conversationId] = {}; + } + } + } + public async pollOnceForOurDisplayName(abortSignal?: AbortSignal): Promise { if (abortSignal?.aborted) { throw new NotFoundError('[pollOnceForOurDisplayName] aborted right away'); @@ -1089,6 +1104,7 @@ async function handleMessagesForGroupV2( { hash: msg.hash, expiresAt: msg.expiration, + conversationId: groupPk, }, ]); } catch (e) { @@ -1096,9 +1112,4 @@ async function handleMessagesForGroupV2( } } } - - // make sure that all the message above are indeed seen (extra check as everything should already be marked as seen in the loop above) - await Data.saveSeenMessageHashes( - newMessages.map(m => ({ hash: m.hash, expiresAt: m.expiration })) - ); } diff --git a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts index bfed36920..4c1ef39f3 100644 --- a/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts +++ b/ts/session/apis/snode_api/swarm_polling_config/SwarmPollingGroupConfig.ts @@ -48,6 +48,7 @@ async function handleMetaMergeResults(groupPk: GroupPubkeyType) { deletionType: 'keepAsDestroyed', // we just got something from the group's swarm, so it is not pendingInvite deleteAllMessagesOnSwarm: false, forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } else { if ( diff --git a/ts/session/conversations/ConversationController.ts b/ts/session/conversations/ConversationController.ts index 945d99f66..efb126115 100644 --- a/ts/session/conversations/ConversationController.ts +++ b/ts/session/conversations/ConversationController.ts @@ -259,11 +259,13 @@ class ConvoController { deletionType, deleteAllMessagesOnSwarm, forceDestroyForAllMembers, + clearFetchedHashes, }: DeleteOptions & { sendLeaveMessage: boolean; deletionType: 'doNotKeep' | 'keepAsKicked' | 'keepAsDestroyed'; deleteAllMessagesOnSwarm: boolean; forceDestroyForAllMembers: boolean; + clearFetchedHashes: boolean; } ) { if (!PubKey.is03Pubkey(groupPk)) { @@ -271,7 +273,7 @@ class ConvoController { } window.log.info( - `deleteGroup: ${ed25519Str(groupPk)}, sendLeaveMessage:${sendLeaveMessage}, fromSyncMessage:${fromSyncMessage}, deletionType:${deletionType}, deleteAllMessagesOnSwarm:${deleteAllMessagesOnSwarm}, forceDestroyForAllMembers:${forceDestroyForAllMembers}` + `deleteGroup: ${ed25519Str(groupPk)}, sendLeaveMessage:${sendLeaveMessage}, fromSyncMessage:${fromSyncMessage}, deletionType:${deletionType}, deleteAllMessagesOnSwarm:${deleteAllMessagesOnSwarm}, forceDestroyForAllMembers:${forceDestroyForAllMembers}, clearFetchedHashes:${clearFetchedHashes}` ); // this deletes all messages in the conversation @@ -373,6 +375,14 @@ class ConvoController { await this.removeGroupOrCommunityFromDBAndRedux(groupPk); } + // We want to clear the lastHash and the seenHashes of the corresponding group. + // We do this so that if we get reinvited to the group, we will + // fetch and display all the messages from the group's swarm again. + if (clearFetchedHashes) { + await getSwarmPollingInstance().resetLastHashesForConversation(groupPk); + await Data.emptySeenMessageHashesForConversation(groupPk); + } + await SessionUtilConvoInfoVolatile.removeGroupFromWrapper(groupPk); // release the memory (and the current meta-dumps in memory for that group) window.log.info(`freeing meta group wrapper: ${ed25519Str(groupPk)}`); diff --git a/ts/session/sending/MessageSender.ts b/ts/session/sending/MessageSender.ts index c90566fb4..197449b6d 100644 --- a/ts/session/sending/MessageSender.ts +++ b/ts/session/sending/MessageSender.ts @@ -4,7 +4,7 @@ import { AbortController } from 'abort-controller'; import { GroupPubkeyType, PubkeyType } from 'libsession_util_nodejs'; import { isArray, isEmpty, isNumber, isString } from 'lodash'; import pRetry from 'p-retry'; -import { Data, SeenMessageHashes } from '../../data/data'; +import { Data } from '../../data/data'; import { UserGroupsWrapperActions } from '../../webworker/workers/browser/libsession_worker_interface'; import { OpenGroupMessageV2 } from '../apis/open_group_api/opengroupV2/OpenGroupMessageV2'; import { @@ -53,7 +53,7 @@ import { UserUtils } from '../utils'; import { ed25519Str, fromUInt8ArrayToBase64 } from '../utils/String'; import { MessageSentHandler } from './MessageSentHandler'; import { EncryptAndWrapMessageResults, MessageWrapper } from './MessageWrapper'; -import { stringify } from '../../types/sqlSharedTypes'; +import { SaveSeenMessageHash, stringify } from '../../types/sqlSharedTypes'; import { OpenGroupRequestCommonType } from '../../data/types'; import { NetworkTime } from '../../util/NetworkTime'; import { MergedAbortSignal } from '../apis/snode_api/requestWith'; @@ -632,7 +632,7 @@ async function handleBatchResultWithSubRequests({ return; } - const seenHashes: Array = []; + const seenHashes: Array = []; for (let index = 0; index < subRequests.length; index++) { const subRequest = subRequests[index]; @@ -658,6 +658,7 @@ async function handleBatchResultWithSubRequests({ seenHashes.push({ expiresAt: NetworkTime.now() + TTL_DEFAULT.CONTENT_MESSAGE, // non config msg expire at CONTENT_MESSAGE at most hash: storedHash, + conversationId: destination, }); // We need to store the hash of our synced message for a 1o1. (as this is the one stored on our swarm) diff --git a/ts/state/ducks/metaGroups.ts b/ts/state/ducks/metaGroups.ts index db7799fd0..7df015dda 100644 --- a/ts/state/ducks/metaGroups.ts +++ b/ts/state/ducks/metaGroups.ts @@ -268,6 +268,7 @@ const initNewGroupInWrapper = createAsyncThunk( deletionType: 'doNotKeep', deleteAllMessagesOnSwarm: false, forceDestroyForAllMembers: false, + clearFetchedHashes: true, }); } throw e; diff --git a/ts/types/sqlSharedTypes.ts b/ts/types/sqlSharedTypes.ts index 5965b5e60..fb4d29726 100644 --- a/ts/types/sqlSharedTypes.ts +++ b/ts/types/sqlSharedTypes.ts @@ -319,3 +319,5 @@ export function stringify(obj: unknown) { 2 ); } + +export type SaveSeenMessageHash = { expiresAt: number; hash: string; conversationId: string };