From 9cff53a2565a2689a905c7ae2e8a4d11ab8b98ff Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 11 Jun 2021 12:04:07 +1000 Subject: [PATCH 1/4] call table names on sql when possible --- app/sql.js | 211 +++++++++--------- .../opengroupV2/OpenGroupAPIV2CompactPoll.ts | 3 +- 2 files changed, 113 insertions(+), 101 deletions(-) diff --git a/app/sql.js b/app/sql.js index e5d9d2f07..27d802427 100644 --- a/app/sql.js +++ b/app/sql.js @@ -123,6 +123,23 @@ module.exports = { removeV2OpenGroupRoom, }; +const CONVERSATIONS_TABLE = 'conversations'; +const MESSAGES_TABLE = 'messages'; +const SENDER_KEYS_TABLE = 'senderKeys'; +const SERVERS_TOKEN_TABLE = 'servers'; +const NODES_FOR_PUBKEY_TABLE = 'nodesForPubkey'; +const OPEN_GROUP_ROOMS_V2_TABLE = 'openGroupRoomsV2'; +const IDENTITY_KEYS_TABLE = 'identityKeys'; +const PRE_KEYS_TABLE = 'preKeys'; +const CONTACT_PRE_KEYS_TABLE = 'contactPreKeys'; +const CONTACT_SIGNED_PRE_KEYS_TABLE = 'contactSignedPreKeys'; +const SIGNED_PRE_KEYS_TABLE = 'signedPreKeys'; +const SESSIONS_TABLE = 'sessions'; +const GUARD_NODE_TABLE = 'guardNodes'; +const ITEMS_TABLE = 'items'; +const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; +const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; + function objectToJSON(data) { return JSON.stringify(data); } @@ -227,7 +244,7 @@ async function updateToSchemaVersion1(currentVersion, instance) { await instance.run('BEGIN TRANSACTION;'); await instance.run( - `CREATE TABLE messages( + `CREATE TABLE ${MESSAGES_TABLE}( id STRING PRIMARY KEY ASC, json TEXT, @@ -246,40 +263,40 @@ async function updateToSchemaVersion1(currentVersion, instance) { );` ); - await instance.run(`CREATE INDEX messages_unread ON messages ( + await instance.run(`CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( unread );`); - await instance.run(`CREATE INDEX messages_expires_at ON messages ( + await instance.run(`CREATE INDEX messages_expires_at ON ${MESSAGES_TABLE} ( expires_at );`); - await instance.run(`CREATE INDEX messages_receipt ON messages ( + await instance.run(`CREATE INDEX messages_receipt ON ${MESSAGES_TABLE} ( sent_at );`); - await instance.run(`CREATE INDEX messages_schemaVersion ON messages ( + await instance.run(`CREATE INDEX messages_schemaVersion ON ${MESSAGES_TABLE} ( schemaVersion );`); - await instance.run(`CREATE INDEX messages_conversation ON messages ( + await instance.run(`CREATE INDEX messages_conversation ON ${MESSAGES_TABLE} ( conversationId, received_at );`); - await instance.run(`CREATE INDEX messages_duplicate_check ON messages ( + await instance.run(`CREATE INDEX messages_duplicate_check ON ${MESSAGES_TABLE} ( source, sourceDevice, sent_at );`); - await instance.run(`CREATE INDEX messages_hasAttachments ON messages ( + await instance.run(`CREATE INDEX messages_hasAttachments ON ${MESSAGES_TABLE} ( conversationId, hasAttachments, received_at );`); - await instance.run(`CREATE INDEX messages_hasFileAttachments ON messages ( + await instance.run(`CREATE INDEX messages_hasFileAttachments ON ${MESSAGES_TABLE} ( conversationId, hasFileAttachments, received_at );`); - await instance.run(`CREATE INDEX messages_hasVisualMediaAttachments ON messages ( + await instance.run(`CREATE INDEX messages_hasVisualMediaAttachments ON ${MESSAGES_TABLE} ( conversationId, hasVisualMediaAttachments, received_at @@ -313,28 +330,28 @@ async function updateToSchemaVersion2(currentVersion, instance) { await instance.run('BEGIN TRANSACTION;'); await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN expireTimer INTEGER;` ); await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN expirationStartTimestamp INTEGER;` ); await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN type STRING;` ); - await instance.run(`CREATE INDEX messages_expiring ON messages ( + await instance.run(`CREATE INDEX messages_expiring ON ${MESSAGES_TABLE} ( expireTimer, expirationStartTimestamp, expires_at );`); await instance.run( - `UPDATE messages SET + `UPDATE ${MESSAGES_TABLE} SET expirationStartTimestamp = json_extract(json, '$.expirationStartTimestamp'), expireTimer = json_extract(json, '$.expireTimer'), type = json_extract(json, '$.type');` @@ -358,13 +375,13 @@ async function updateToSchemaVersion3(currentVersion, instance) { await instance.run('DROP INDEX messages_expiring;'); await instance.run('DROP INDEX messages_unread;'); - await instance.run(`CREATE INDEX messages_without_timer ON messages ( + await instance.run(`CREATE INDEX messages_without_timer ON ${MESSAGES_TABLE} ( expireTimer, expires_at, type ) WHERE expires_at IS NULL AND expireTimer IS NOT NULL;`); - await instance.run(`CREATE INDEX messages_unread ON messages ( + await instance.run(`CREATE INDEX messages_unread ON ${MESSAGES_TABLE} ( conversationId, unread ) WHERE unread IS NOT NULL;`); @@ -566,10 +583,10 @@ async function updateToSchemaVersion8(currentVersion, instance) { // First, we pull a new body field out of the message table's json blob await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN body TEXT;` ); - await instance.run("UPDATE messages SET body = json_extract(json, '$.body')"); + await instance.run(`UPDATE ${MESSAGES_TABLE} SET body = json_extract(json, '$.body')`); // Then we create our full-text search table and populate it await instance.run(` @@ -583,7 +600,7 @@ async function updateToSchemaVersion8(currentVersion, instance) { // Then we set up triggers to keep the full-text search table up to date await instance.run(` - CREATE TRIGGER messages_on_insert AFTER INSERT ON messages BEGIN + CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN INSERT INTO messages_fts ( id, body @@ -594,12 +611,12 @@ async function updateToSchemaVersion8(currentVersion, instance) { END; `); await instance.run(` - CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN + CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN DELETE FROM messages_fts WHERE id = old.id; END; `); await instance.run(` - CREATE TRIGGER messages_on_update AFTER UPDATE ON messages BEGIN + CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN DELETE FROM messages_fts WHERE id = old.id; INSERT INTO messages_fts( id, @@ -774,17 +791,16 @@ const LOKI_SCHEMA_VERSIONS = [ updateToLokiSchemaVersion13, ]; -const SERVERS_TOKEN_TABLE = 'servers'; - async function updateToLokiSchemaVersion1(currentVersion, instance) { - if (currentVersion >= 1) { + const targetVersion = 1; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion1: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN serverId INTEGER;` ); await instance.run( @@ -798,18 +814,20 @@ async function updateToLokiSchemaVersion1(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 1 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion1: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion2(currentVersion, instance) { - if (currentVersion >= 2) { + const targetVersion = 2; + + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion2: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await instance.run( @@ -827,15 +845,17 @@ async function updateToLokiSchemaVersion2(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 2 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion2: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion3(currentVersion, instance) { - if (currentVersion >= 3) { + const targetVersion = 3; + + if (currentVersion >= targetVersion) { return; } @@ -846,29 +866,28 @@ async function updateToLokiSchemaVersion3(currentVersion, instance) { );` ); - console.log('updateToLokiSchemaVersion3: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await instance.run( `INSERT INTO loki_schema ( version ) values ( - 3 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion3: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } -const SENDER_KEYS_TABLE = 'senderKeys'; - async function updateToLokiSchemaVersion4(currentVersion, instance) { - if (currentVersion >= 4) { + const targetVersion = 4; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion4: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); // We don't bother migrating values, any old messages that @@ -903,22 +922,21 @@ async function updateToLokiSchemaVersion4(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 4 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion4: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } -const NODES_FOR_PUBKEY_TABLE = 'nodesForPubkey'; - async function updateToLokiSchemaVersion5(currentVersion, instance) { - if (currentVersion >= 5) { + const targetVersion = 5; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion5: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); @@ -933,20 +951,21 @@ async function updateToLokiSchemaVersion5(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 5 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion5: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion6(currentVersion, instance) { - if (currentVersion >= 6) { + const targetVersion = 6; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion6: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); @@ -961,20 +980,21 @@ async function updateToLokiSchemaVersion6(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 6 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion6: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion7(currentVersion, instance) { - if (currentVersion >= 7) { + const targetVersion = 7; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion7: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); @@ -985,23 +1005,24 @@ async function updateToLokiSchemaVersion7(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 7 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion7: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion8(currentVersion, instance) { - if (currentVersion >= 8) { + const targetVersion = 8; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion8: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await instance.run( - `ALTER TABLE messages + `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN serverTimestamp INTEGER;` ); @@ -1009,18 +1030,19 @@ async function updateToLokiSchemaVersion8(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 8 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion8: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion9(currentVersion, instance) { - if (currentVersion >= 9) { + const targetVersion = 9; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion9: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await removePrefixFromGroupConversations(instance); @@ -1029,18 +1051,19 @@ async function updateToLokiSchemaVersion9(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 9 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion9: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion10(currentVersion, instance) { - if (currentVersion >= 10) { + const targetVersion = 10; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion10: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await createEncryptionKeyPairsForClosedGroup(instance); @@ -1049,18 +1072,19 @@ async function updateToLokiSchemaVersion10(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 10 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion10: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion11(currentVersion, instance) { - if (currentVersion >= 11) { + const targetVersion = 11; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion11: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await updateExistingClosedGroupToClosedGroup(instance); @@ -1069,19 +1093,19 @@ async function updateToLokiSchemaVersion11(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 11 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion11: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } -const OPEN_GROUP_ROOMS_V2_TABLE = 'openGroupRoomsV2'; async function updateToLokiSchemaVersion12(currentVersion, instance) { - if (currentVersion >= 12) { + const targetVersion = 12; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion12: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); await instance.run( @@ -1098,18 +1122,19 @@ async function updateToLokiSchemaVersion12(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 12 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion12: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateToLokiSchemaVersion13(currentVersion, instance) { - if (currentVersion >= 13) { + const targetVersion = 13; + if (currentVersion >= targetVersion) { return; } - console.log('updateToLokiSchemaVersion13: starting...'); + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); // Clear any already deleted db entries. @@ -1119,12 +1144,12 @@ async function updateToLokiSchemaVersion13(currentVersion, instance) { `INSERT INTO loki_schema ( version ) values ( - 13 + ${targetVersion} );` ); await instance.run('COMMIT TRANSACTION;'); - console.log('updateToLokiSchemaVersion13: success!'); + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } async function updateLokiSchema(instance) { @@ -1305,37 +1330,29 @@ async function removePasswordHash() { return removeItemById(PASS_HASH_ID); } -const IDENTITY_KEYS_TABLE = 'identityKeys'; async function getIdentityKeyById(id, instance) { return getById(IDENTITY_KEYS_TABLE, id, instance); } // those removeAll calls are currently only used to cleanup the db from old data // TODO remove those and move those removeAll in a migration -const PRE_KEYS_TABLE = 'preKeys'; async function removeAllPreKeys() { return removeAllFromTable(PRE_KEYS_TABLE); } -const CONTACT_PRE_KEYS_TABLE = 'contactPreKeys'; async function removeAllContactPreKeys() { return removeAllFromTable(CONTACT_PRE_KEYS_TABLE); } -const CONTACT_SIGNED_PRE_KEYS_TABLE = 'contactSignedPreKeys'; async function removeAllContactSignedPreKeys() { return removeAllFromTable(CONTACT_SIGNED_PRE_KEYS_TABLE); } -const SIGNED_PRE_KEYS_TABLE = 'signedPreKeys'; async function removeAllSignedPreKeys() { return removeAllFromTable(SIGNED_PRE_KEYS_TABLE); } -const SESSIONS_TABLE = 'sessions'; async function removeAllSessions() { return removeAllFromTable(SESSIONS_TABLE); } -const GUARD_NODE_TABLE = 'guardNodes'; - async function getGuardNodes() { const nodes = await db.all(`SELECT ed25519PubKey FROM ${GUARD_NODE_TABLE};`); @@ -1370,7 +1387,6 @@ async function updateGuardNodes(nodes) { // Return all the paired pubkeys for a specific pubkey (excluded), // irrespective of their Primary or Secondary status. -const ITEMS_TABLE = 'items'; async function createOrUpdateItem(data, instance) { return createOrUpdate(ITEMS_TABLE, data, instance); } @@ -1466,8 +1482,6 @@ async function updateSwarmNodesForPubkey(pubkey, snodeEdKeys) { ); } -const CONVERSATIONS_TABLE = 'conversations'; -const MESSAGES_TABLE = 'messages'; async function getConversationCount() { const row = await db.get(`SELECT count(*) from ${CONVERSATIONS_TABLE};`); @@ -1707,7 +1721,7 @@ async function searchMessages(query, { limit } = {}) { messages.json, snippet(messages_fts, -1, '<>', '<>', '...', 15) as snippet FROM messages_fts - INNER JOIN messages on messages_fts.id = messages.id + INNER JOIN ${MESSAGES_TABLE} on messages_fts.id = messages.id WHERE messages_fts match $query ORDER BY messages.received_at DESC @@ -1730,7 +1744,7 @@ async function searchMessagesInConversation(query, conversationId, { limit } = { messages.json, snippet(messages_fts, -1, '<>', '<>', '...', 15) as snippet FROM messages_fts - INNER JOIN messages on messages_fts.id = messages.id + INNER JOIN ${MESSAGES_TABLE} on messages_fts.id = messages.id WHERE messages_fts match $query AND messages.conversationId = $conversationId @@ -2305,7 +2319,6 @@ async function removeAllUnprocessed() { await db.run('DELETE FROM unprocessed;'); } -const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; async function getNextAttachmentDownloadJobs(limit, options = {}) { const timestamp = options.timestamp || Date.now(); @@ -2539,7 +2552,7 @@ async function removeKnownAttachments(allAttachments) { count += messages.length; } - console.log(`removeKnownAttachments: Done processing ${count} messages`); + console.log(`removeKnownAttachments: Done processing ${count} ${MESSAGES_TABLE}`); complete = false; count = 0; @@ -2656,8 +2669,6 @@ async function removePrefixFromGroupConversations(instance) { ); } -const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; - async function createEncryptionKeyPairsForClosedGroup(instance) { await instance.run( `CREATE TABLE ${CLOSED_GROUP_V2_KEY_PAIRS_TABLE} ( diff --git a/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts b/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts index d1a747eef..6bf7ebe3e 100644 --- a/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts +++ b/ts/opengroup/opengroupV2/OpenGroupAPIV2CompactPoll.ts @@ -143,8 +143,9 @@ const getAllValidRoomInfos = async ( return null; } allServerPubKeys.push(fetchedInfo.serverPublicKey); + const tokenInProgress = await getAuthToken({ serverUrl, roomId }); - return fetchedInfo; + return { ...fetchedInfo, token: tokenInProgress || undefined }; } catch (e) { window?.log?.warn('failed to fetch roominfos for room', roomId); return null; From 27fb732f50024efeafbdec268723b1dd71d2bd3f Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 11 Jun 2021 12:09:57 +1000 Subject: [PATCH 2/4] cleanup unused prekeys table --- app/sql.js | 172 +++++++++---------------- js/modules/backup.js | 3 - ts/components/session/ActionsPanel.tsx | 4 - ts/data/data.ts | 52 -------- ts/session/group/index.ts | 19 +-- ts/util/accountManager.ts | 22 ---- 6 files changed, 61 insertions(+), 211 deletions(-) diff --git a/app/sql.js b/app/sql.js index 27d802427..5a31e37f1 100644 --- a/app/sql.js +++ b/app/sql.js @@ -25,12 +25,6 @@ module.exports = { getIdentityKeyById, - removeAllSignedPreKeys, - removeAllContactPreKeys, - removeAllContactSignedPreKeys, - removeAllPreKeys, - removeAllSessions, - createOrUpdateItem, getItemById, getAllItems, @@ -44,8 +38,6 @@ module.exports = { getConversationCount, saveConversation, getConversationById, - savePublicServerToken, - getPublicServerTokenByServerUrl, updateConversation, removeConversation, getAllConversations, @@ -125,16 +117,10 @@ module.exports = { const CONVERSATIONS_TABLE = 'conversations'; const MESSAGES_TABLE = 'messages'; -const SENDER_KEYS_TABLE = 'senderKeys'; -const SERVERS_TOKEN_TABLE = 'servers'; +const MESSAGES_FTS_TABLE = 'messages_fts'; const NODES_FOR_PUBKEY_TABLE = 'nodesForPubkey'; const OPEN_GROUP_ROOMS_V2_TABLE = 'openGroupRoomsV2'; const IDENTITY_KEYS_TABLE = 'identityKeys'; -const PRE_KEYS_TABLE = 'preKeys'; -const CONTACT_PRE_KEYS_TABLE = 'contactPreKeys'; -const CONTACT_SIGNED_PRE_KEYS_TABLE = 'contactSignedPreKeys'; -const SIGNED_PRE_KEYS_TABLE = 'signedPreKeys'; -const SESSIONS_TABLE = 'sessions'; const GUARD_NODE_TABLE = 'guardNodes'; const ITEMS_TABLE = 'items'; const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; @@ -403,7 +389,7 @@ async function updateToSchemaVersion4(currentVersion, instance) { await instance.run('BEGIN TRANSACTION;'); await instance.run( - `CREATE TABLE conversations( + `CREATE TABLE ${CONVERSATIONS_TABLE}( id STRING PRIMARY KEY ASC, json TEXT, @@ -415,11 +401,11 @@ async function updateToSchemaVersion4(currentVersion, instance) { );` ); - await instance.run(`CREATE INDEX conversations_active ON conversations ( + await instance.run(`CREATE INDEX conversations_active ON ${CONVERSATIONS_TABLE} ( active_at ) WHERE active_at IS NOT NULL;`); - await instance.run(`CREATE INDEX conversations_type ON conversations ( + await instance.run(`CREATE INDEX conversations_type ON ${CONVERSATIONS_TABLE} ( type ) WHERE type IS NOT NULL;`); @@ -438,7 +424,7 @@ async function updateToSchemaVersion6(currentVersion, instance) { // friendRequestStatus is no longer needed. So no need to add the column on new apps // await instance.run( - // `ALTER TABLE conversations + // `ALTER TABLE ${CONVERSATIONS_TABLE} // ADD COLUMN friendRequestStatus INTEGER;` // ); @@ -478,13 +464,13 @@ async function updateToSchemaVersion6(currentVersion, instance) { );` ); await instance.run( - `CREATE TABLE identityKeys( + `CREATE TABLE ${IDENTITY_KEYS_TABLE}( id STRING PRIMARY KEY ASC, json TEXT );` ); await instance.run( - `CREATE TABLE items( + `CREATE TABLE ${ITEMS_TABLE}( id STRING PRIMARY KEY ASC, json TEXT );` @@ -590,18 +576,18 @@ async function updateToSchemaVersion8(currentVersion, instance) { // Then we create our full-text search table and populate it await instance.run(` - CREATE VIRTUAL TABLE messages_fts + CREATE VIRTUAL TABLE ${MESSAGES_FTS_TABLE} USING fts5(id UNINDEXED, body); `); await instance.run(` - INSERT INTO messages_fts(id, body) + INSERT INTO ${MESSAGES_FTS_TABLE}(id, body) SELECT id, body FROM ${MESSAGES_TABLE}; `); // Then we set up triggers to keep the full-text search table up to date await instance.run(` CREATE TRIGGER messages_on_insert AFTER INSERT ON ${MESSAGES_TABLE} BEGIN - INSERT INTO messages_fts ( + INSERT INTO ${MESSAGES_FTS_TABLE} ( id, body ) VALUES ( @@ -612,13 +598,13 @@ async function updateToSchemaVersion8(currentVersion, instance) { `); await instance.run(` CREATE TRIGGER messages_on_delete AFTER DELETE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM messages_fts WHERE id = old.id; + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; END; `); await instance.run(` CREATE TRIGGER messages_on_update AFTER UPDATE ON ${MESSAGES_TABLE} BEGIN - DELETE FROM messages_fts WHERE id = old.id; - INSERT INTO messages_fts( + DELETE FROM ${MESSAGES_FTS_TABLE} WHERE id = old.id; + INSERT INTO ${MESSAGES_FTS_TABLE}( id, body ) VALUES ( @@ -644,7 +630,7 @@ async function updateToSchemaVersion9(currentVersion, instance) { console.log('updateToSchemaVersion9: starting...'); await instance.run('BEGIN TRANSACTION;'); - await instance.run(`CREATE TABLE attachment_downloads( + await instance.run(`CREATE TABLE ${ATTACHMENT_DOWNLOADS_TABLE}( id STRING primary key, timestamp INTEGER, pending INTEGER, @@ -652,11 +638,11 @@ async function updateToSchemaVersion9(currentVersion, instance) { );`); await instance.run(`CREATE INDEX attachment_downloads_timestamp - ON attachment_downloads ( + ON ${ATTACHMENT_DOWNLOADS_TABLE} ( timestamp ) WHERE pending = 0;`); await instance.run(`CREATE INDEX attachment_downloads_pending - ON attachment_downloads ( + ON ${ATTACHMENT_DOWNLOADS_TABLE} ( pending ) WHERE pending != 0;`); @@ -789,6 +775,7 @@ const LOKI_SCHEMA_VERSIONS = [ updateToLokiSchemaVersion11, updateToLokiSchemaVersion12, updateToLokiSchemaVersion13, + updateToLokiSchemaVersion14, ]; async function updateToLokiSchemaVersion1(currentVersion, instance) { @@ -803,8 +790,9 @@ async function updateToLokiSchemaVersion1(currentVersion, instance) { `ALTER TABLE ${MESSAGES_TABLE} ADD COLUMN serverId INTEGER;` ); + // servers is removed later await instance.run( - `CREATE TABLE ${SERVERS_TOKEN_TABLE}( + `CREATE TABLE servers( serverUrl STRING PRIMARY KEY ASC, token TEXT );` @@ -904,16 +892,6 @@ async function updateToLokiSchemaVersion4(currentVersion, instance) { );` ); - // Create a table for Sender Keys - await instance.run( - `CREATE TABLE ${SENDER_KEYS_TABLE} ( - groupId TEXT, - senderIdentity TEXT, - json TEXT, - PRIMARY KEY (groupId, senderIdentity) - );` - ); - // Add senderIdentity field to `unprocessed` needed // for medium size groups await instance.run(`ALTER TABLE unprocessed ADD senderIdentity TEXT`); @@ -1087,7 +1065,7 @@ async function updateToLokiSchemaVersion11(currentVersion, instance) { console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); await instance.run('BEGIN TRANSACTION;'); - await updateExistingClosedGroupToClosedGroup(instance); + await updateExistingClosedGroupV1ToClosedGroupV2(instance); await instance.run( `INSERT INTO loki_schema ( @@ -1152,6 +1130,33 @@ async function updateToLokiSchemaVersion13(currentVersion, instance) { console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); } +async function updateToLokiSchemaVersion14(currentVersion, instance) { + const targetVersion = 14; + if (currentVersion >= targetVersion) { + return; + } + console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); + await instance.run('BEGIN TRANSACTION;'); + await instance.run('DROP TABLE IF EXISTS servers;'); + await instance.run('DROP TABLE IF EXISTS sessions;'); + await instance.run('DROP TABLE IF EXISTS preKeys;'); + await instance.run('DROP TABLE IF EXISTS contactPreKeys;'); + await instance.run('DROP TABLE IF EXISTS contactSignedPreKeys;'); + await instance.run('DROP TABLE IF EXISTS signedPreKeys;'); + await instance.run('DROP TABLE IF EXISTS senderKeys;'); + + await instance.run( + `INSERT INTO loki_schema ( + version + ) values ( + ${targetVersion} + );` + ); + await instance.run('COMMIT TRANSACTION;'); + + console.log(`updateToLokiSchemaVersion${targetVersion}: success!`); +} + async function updateLokiSchema(instance) { const result = await instance.get( "SELECT name FROM sqlite_master WHERE type = 'table' AND name='loki_schema';" @@ -1334,25 +1339,6 @@ async function getIdentityKeyById(id, instance) { return getById(IDENTITY_KEYS_TABLE, id, instance); } -// those removeAll calls are currently only used to cleanup the db from old data -// TODO remove those and move those removeAll in a migration -async function removeAllPreKeys() { - return removeAllFromTable(PRE_KEYS_TABLE); -} -async function removeAllContactPreKeys() { - return removeAllFromTable(CONTACT_PRE_KEYS_TABLE); -} - -async function removeAllContactSignedPreKeys() { - return removeAllFromTable(CONTACT_SIGNED_PRE_KEYS_TABLE); -} -async function removeAllSignedPreKeys() { - return removeAllFromTable(SIGNED_PRE_KEYS_TABLE); -} -async function removeAllSessions() { - return removeAllFromTable(SESSIONS_TABLE); -} - async function getGuardNodes() { const nodes = await db.all(`SELECT ed25519PubKey FROM ${GUARD_NODE_TABLE};`); @@ -1394,7 +1380,7 @@ async function getItemById(id) { return getById(ITEMS_TABLE, id); } async function getAllItems() { - const rows = await db.all('SELECT json FROM items ORDER BY id ASC;'); + const rows = await db.all(`SELECT json FROM ${ITEMS_TABLE} ORDER BY id ASC;`); return map(rows, row => jsonToObject(row.json)); } async function removeItemById(id) { @@ -1589,37 +1575,6 @@ async function removeConversation(id) { ); } -// open groups v1 only -async function savePublicServerToken(data) { - const { serverUrl, token } = data; - await db.run( - `INSERT OR REPLACE INTO ${SERVERS_TOKEN_TABLE} ( - serverUrl, - token - ) values ( - $serverUrl, - $token - )`, - { - $serverUrl: serverUrl, - $token: token, - } - ); -} - -// open groups v1 only -async function getPublicServerTokenByServerUrl(serverUrl) { - const row = await db.get(`SELECT * FROM ${SERVERS_TOKEN_TABLE} WHERE serverUrl = $serverUrl;`, { - $serverUrl: serverUrl, - }); - - if (!row) { - return null; - } - - return row.token; -} - async function getConversationById(id) { const row = await db.get(`SELECT * FROM ${CONVERSATIONS_TABLE} WHERE id = $id;`, { $id: id, @@ -1720,7 +1675,7 @@ async function searchMessages(query, { limit } = {}) { `SELECT messages.json, snippet(messages_fts, -1, '<>', '<>', '...', 15) as snippet - FROM messages_fts + FROM ${MESSAGES_FTS_TABLE} INNER JOIN ${MESSAGES_TABLE} on messages_fts.id = messages.id WHERE messages_fts match $query @@ -2323,7 +2278,7 @@ async function getNextAttachmentDownloadJobs(limit, options = {}) { const timestamp = options.timestamp || Date.now(); const rows = await db.all( - `SELECT json FROM attachment_downloads + `SELECT json FROM ${ATTACHMENT_DOWNLOADS_TABLE} WHERE pending = 0 AND timestamp < $timestamp ORDER BY timestamp DESC LIMIT $limit;`, @@ -2342,7 +2297,7 @@ async function saveAttachmentDownloadJob(job) { } await db.run( - `INSERT OR REPLACE INTO attachment_downloads ( + `INSERT OR REPLACE INTO ${ATTACHMENT_DOWNLOADS_TABLE} ( id, pending, timestamp, @@ -2362,13 +2317,13 @@ async function saveAttachmentDownloadJob(job) { ); } async function setAttachmentDownloadJobPending(id, pending) { - await db.run('UPDATE attachment_downloads SET pending = $pending WHERE id = $id;', { + await db.run(`UPDATE ${ATTACHMENT_DOWNLOADS_TABLE} SET pending = $pending WHERE id = $id;`, { $id: id, $pending: pending, }); } async function resetAttachmentDownloadPending() { - await db.run('UPDATE attachment_downloads SET pending = 0 WHERE pending != 0;'); + await db.run(`UPDATE ${ATTACHMENT_DOWNLOADS_TABLE} SET pending = 0 WHERE pending != 0;`); } async function removeAttachmentDownloadJob(id) { return removeById(ATTACHMENT_DOWNLOADS_TABLE, id); @@ -2384,24 +2339,17 @@ async function removeAll() { db.serialize(() => { promise = Promise.all([ db.run('BEGIN TRANSACTION;'), - db.run('DELETE FROM identityKeys;'), - db.run('DELETE FROM items;'), - db.run('DELETE FROM preKeys;'), - db.run('DELETE FROM sessions;'), - db.run('DELETE FROM signedPreKeys;'), + db.run(`DELETE FROM ${IDENTITY_KEYS_TABLE};`), + db.run(`DELETE FROM ${ITEMS_TABLE};`), db.run('DELETE FROM unprocessed;'), - db.run('DELETE FROM contactPreKeys;'), - db.run('DELETE FROM contactSignedPreKeys;'), - db.run(`DELETE FROM ${SERVERS_TOKEN_TABLE};`), db.run('DELETE FROM lastHashes;'), - db.run(`DELETE FROM ${SENDER_KEYS_TABLE};`), db.run(`DELETE FROM ${NODES_FOR_PUBKEY_TABLE};`), db.run(`DELETE FROM ${CLOSED_GROUP_V2_KEY_PAIRS_TABLE};`), db.run('DELETE FROM seenMessages;'), db.run(`DELETE FROM ${CONVERSATIONS_TABLE};`), db.run(`DELETE FROM ${MESSAGES_TABLE};`), - db.run('DELETE FROM attachment_downloads;'), - db.run('DELETE FROM messages_fts;'), + db.run(`DELETE FROM ${ATTACHMENT_DOWNLOADS_TABLE};`), + db.run(`DELETE FROM ${MESSAGES_FTS_TABLE};`), db.run('COMMIT TRANSACTION;'), ]); }); @@ -2562,7 +2510,7 @@ async function removeKnownAttachments(allAttachments) { const conversationTotal = await getConversationCount(); console.log( - `removeKnownAttachments: About to iterate through ${conversationTotal} conversations` + `removeKnownAttachments: About to iterate through ${conversationTotal} ${CONVERSATIONS_TABLE}` ); while (!complete) { @@ -2594,7 +2542,7 @@ async function removeKnownAttachments(allAttachments) { count += conversations.length; } - console.log(`removeKnownAttachments: Done processing ${count} conversations`); + console.log(`removeKnownAttachments: Done processing ${count} ${CONVERSATIONS_TABLE}`); return Object.keys(lookup); } @@ -2698,7 +2646,7 @@ function remove05PrefixFromStringIfNeeded(str) { return str; } -async function updateExistingClosedGroupToClosedGroup(instance) { +async function updateExistingClosedGroupV1ToClosedGroupV2(instance) { // the migration is called only once, so all current groups not being open groups are v1 closed group. const allClosedGroupV1 = (await getAllClosedGroupConversations(instance)) || []; diff --git a/js/modules/backup.js b/js/modules/backup.js index 59cca4034..58b8ad155 100644 --- a/js/modules/backup.js +++ b/js/modules/backup.js @@ -213,10 +213,7 @@ async function importFromJsonString(jsonString, targetPath, options) { result.fullImport = false; delete importObject.items; - delete importObject.signedPreKeys; - delete importObject.preKeys; delete importObject.identityKeys; - delete importObject.sessions; delete importObject.unprocessed; window.log.info('This is a light import; contacts, groups and messages only'); diff --git a/ts/components/session/ActionsPanel.tsx b/ts/components/session/ActionsPanel.tsx index c09c58315..8cb800ba7 100644 --- a/ts/components/session/ActionsPanel.tsx +++ b/ts/components/session/ActionsPanel.tsx @@ -18,7 +18,6 @@ import { } from '../../data/data'; import { OnionPaths } from '../../session/onions'; import { getMessageQueue } from '../../session/sending'; -import { clearSessionsAndPreKeys } from '../../util/accountManager'; import { useDispatch, useSelector } from 'react-redux'; import { getOurNumber } from '../../state/selectors/user'; import { @@ -274,9 +273,6 @@ const doAppStartUp = (dispatch: Dispatch) => { // keep that one to make sure our users upgrade to new sessionIDS void showResetSessionIDDialogIfNeeded(); - // remove existing prekeys, sign prekeys and sessions - // FIXME audric, make this in a migration so we can remove this line - void clearSessionsAndPreKeys(); void removeAllV1OpenGroups(); // this generates the key to encrypt attachments locally diff --git a/ts/data/data.ts b/ts/data/data.ts index 3f2c07253..96a8ed32e 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -71,12 +71,6 @@ const channelsToMake = { removeDB, getPasswordHash, - getIdentityKeyById, - removeAllPreKeys, - removeAllSignedPreKeys, - removeAllContactPreKeys, - removeAllContactSignedPreKeys, - getGuardNodes, updateGuardNodes, @@ -85,8 +79,6 @@ const channelsToMake = { getAllItems, removeItemById, - removeAllSessions, - getSwarmNodesForPubkey, updateSwarmNodesForPubkey, @@ -99,8 +91,6 @@ const channelsToMake = { getAllConversationIds, getAllOpenGroupV1Conversations, getPubkeysInPublicConversation, - savePublicServerToken, - getPublicServerTokenByServerUrl, getAllGroupsInvolvingId, searchConversations, @@ -408,34 +398,6 @@ export async function getPasswordHash(): Promise { return channels.getPasswordHash(); } -// Identity Keys - -const IDENTITY_KEY_KEYS = ['publicKey']; - -// Identity Keys -// TODO: identity key has different shape depending on how it is called, -// so we need to come up with a way to make TS work with all of them - -export async function getIdentityKeyById(id: string): Promise { - const data = await channels.getIdentityKeyById(id); - return keysToArrayBuffer(IDENTITY_KEY_KEYS, data); -} - -// Those removeAll are not used anymore except to cleanup the app since we removed all of those tables -export async function removeAllPreKeys(): Promise { - await channels.removeAllPreKeys(); -} -const PRE_KEY_KEYS = ['privateKey', 'publicKey', 'signature']; -export async function removeAllSignedPreKeys(): Promise { - await channels.removeAllSignedPreKeys(); -} -export async function removeAllContactPreKeys(): Promise { - await channels.removeAllContactPreKeys(); -} -export async function removeAllContactSignedPreKeys(): Promise { - await channels.removeAllContactSignedPreKeys(); -} - // Guard Nodes export async function getGuardNodes(): Promise> { return channels.getGuardNodes(); @@ -493,10 +455,6 @@ export async function getAllItems(): Promise> { export async function removeItemById(id: string): Promise { await channels.removeItemById(id); } -// Sessions -export async function removeAllSessions(): Promise { - await channels.removeAllSessions(); -} // Swarm nodes export async function getSwarmNodesForPubkey(pubkey: string): Promise> { @@ -602,16 +560,6 @@ export async function getPubkeysInPublicConversation(id: string): Promise { - await channels.savePublicServerToken(data); -} - -// open groups v1 only -export async function getPublicServerTokenByServerUrl(serverUrl: string): Promise { - const token = await channels.getPublicServerTokenByServerUrl(serverUrl); - return token; -} export async function getAllGroupsInvolvingId(id: string): Promise { const conversations = await channels.getAllGroupsInvolvingId(id); diff --git a/ts/session/group/index.ts b/ts/session/group/index.ts index 7367016ad..3cafba12d 100644 --- a/ts/session/group/index.ts +++ b/ts/session/group/index.ts @@ -2,14 +2,12 @@ import { PubKey } from '../types'; import _ from 'lodash'; -import { fromHex, fromHexToArray, toHex } from '../utils/String'; +import { fromHexToArray, toHex } from '../utils/String'; import { BlockedNumberController } from '../../util/blockedNumberController'; import { ConversationController } from '../conversations'; import { addClosedGroupEncryptionKeyPair, - getIdentityKeyById, getLatestClosedGroupEncryptionKeyPair, - removeAllClosedGroupEncryptionKeyPairs, } from '../../../ts/data/data'; import uuid from 'uuid'; import { SignalService } from '../../protobuf'; @@ -65,21 +63,6 @@ export interface MemberChanges { leavingMembers?: Array; } -export async function getGroupSecretKey(groupId: string): Promise { - const groupIdentity = await getIdentityKeyById(groupId); - if (!groupIdentity) { - throw new Error(`Could not load secret key for group ${groupId}`); - } - - const secretKey = groupIdentity.secretKey; - - if (!secretKey) { - throw new Error(`Secret key not found in identity key record for group ${groupId}`); - } - - return new Uint8Array(fromHex(secretKey)); -} - export async function initiateGroupUpdate( groupId: string, groupName: string, diff --git a/ts/util/accountManager.ts b/ts/util/accountManager.ts index 21427f744..2138a161c 100644 --- a/ts/util/accountManager.ts +++ b/ts/util/accountManager.ts @@ -4,13 +4,6 @@ import { UserUtils } from '../session/utils'; import { fromArrayBufferToBase64, fromHex, toHex } from '../session/utils/String'; import { getOurPubKeyStrFromCache } from '../session/utils/User'; import { trigger } from '../shims/events'; -import { - removeAllContactPreKeys, - removeAllContactSignedPreKeys, - removeAllPreKeys, - removeAllSessions, - removeAllSignedPreKeys, -} from '../data/data'; import { forceSyncConfigurationNowIfNeeded } from '../session/utils/syncUtils'; import { actions as userActions } from '../state/ducks/user'; import { mn_decode, mn_encode } from '../session/crypto/mnemonic'; @@ -86,7 +79,6 @@ export async function signInByLinkingDevice(mnemonic: string, mnemonicLanguage: UserUtils.setSignInByLinking(true); await createAccount(identityKeyPair); UserUtils.saveRecoveryPhrase(mnemonic); - await clearSessionsAndPreKeys(); const pubKeyString = toHex(identityKeyPair.pubKey); // await for the first configuration message to come in. @@ -117,7 +109,6 @@ export async function registerSingleDevice( await createAccount(identityKeyPair); UserUtils.saveRecoveryPhrase(generatedMnemonic); - await clearSessionsAndPreKeys(); await UserUtils.setLastProfileUpdateTimestamp(Date.now()); const pubKeyString = toHex(identityKeyPair.pubKey); @@ -133,19 +124,6 @@ export async function generateMnemonic() { return mn_encode(hex); } -export async function clearSessionsAndPreKeys() { - window?.log?.info('clearing all sessions'); - // During secondary device registration we need to keep our prekeys sent - // to other pubkeys - await Promise.all([ - removeAllPreKeys(), - removeAllSignedPreKeys(), - removeAllContactPreKeys(), - removeAllContactSignedPreKeys(), - removeAllSessions(), - ]); -} - async function bouncyDeleteAccount(reason?: string) { const deleteEverything = async () => { window?.log?.info('configuration message sent successfully. Deleting everything'); From 9796f6fd52b98f6b142fa20f0c20d1282078fc79 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 11 Jun 2021 13:11:44 +1000 Subject: [PATCH 3/4] limit the number of returned members to the last 300 --- app/sql.js | 33 ++++++++++++++++++++++++++++++--- ts/data/data.ts | 3 +++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/app/sql.js b/app/sql.js index 5a31e37f1..ceca42568 100644 --- a/app/sql.js +++ b/app/sql.js @@ -126,6 +126,8 @@ const ITEMS_TABLE = 'items'; const ATTACHMENT_DOWNLOADS_TABLE = 'attachment_downloads'; const CLOSED_GROUP_V2_KEY_PAIRS_TABLE = 'encryptionKeyPairsForClosedGroupV2'; +const MAX_PUBKEYS_MEMBERS = 1000; + function objectToJSON(data) { return JSON.stringify(data); } @@ -1145,6 +1147,31 @@ async function updateToLokiSchemaVersion14(currentVersion, instance) { await instance.run('DROP TABLE IF EXISTS signedPreKeys;'); await instance.run('DROP TABLE IF EXISTS senderKeys;'); + console.time('removingOpengroupv1Messages'); + + let toRemoveCount = 0; + do { + // eslint-disable-next-line no-await-in-loop + const row = await instance.get(`SELECT count(*) from ${MESSAGES_TABLE} WHERE + conversationId LIKE 'publicChat:1@%';`); + toRemoveCount = row['count(*)']; + + if (toRemoveCount > 0) { + console.warn('toRemove count', toRemoveCount); + console.time('chunk'); + + // eslint-disable-next-line no-await-in-loop + await instance.all( + `DELETE FROM ${MESSAGES_TABLE} WHERE + conversationId LIKE 'publicChat:1@%' + ;` + ); + console.timeEnd('chunk'); + } + } while (toRemoveCount > 0); + + console.timeEnd('removingOpengroupv1Messages'); + await instance.run( `INSERT INTO loki_schema ( version @@ -1626,7 +1653,7 @@ async function getPubkeysInPublicConversation(id) { const rows = await db.all( `SELECT DISTINCT source FROM ${MESSAGES_TABLE} WHERE conversationId = $conversationId - ORDER BY id ASC;`, + ORDER BY received_at DESC LIMIT ${MAX_PUBKEYS_MEMBERS};`, { $conversationId: id, } @@ -2628,7 +2655,7 @@ async function createEncryptionKeyPairsForClosedGroup(instance) { ); } -async function getAllClosedGroupConversations(instance) { +async function getAllClosedGroupConversationsV1(instance) { const rows = await (db || instance).all( `SELECT json FROM ${CONVERSATIONS_TABLE} WHERE type = 'group' AND @@ -2648,7 +2675,7 @@ function remove05PrefixFromStringIfNeeded(str) { async function updateExistingClosedGroupV1ToClosedGroupV2(instance) { // the migration is called only once, so all current groups not being open groups are v1 closed group. - const allClosedGroupV1 = (await getAllClosedGroupConversations(instance)) || []; + const allClosedGroupV1 = (await getAllClosedGroupConversationsV1(instance)) || []; await Promise.all( allClosedGroupV1.map(async groupV1 => { diff --git a/ts/data/data.ts b/ts/data/data.ts index 96a8ed32e..7772af2fa 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -556,6 +556,9 @@ export async function getAllOpenGroupV1Conversations(): Promise> { return channels.getPubkeysInPublicConversation(id); } From 3f75fa54adf8af10fe93e8da633ea6be6a0fd808 Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Fri, 11 Jun 2021 15:14:52 +1000 Subject: [PATCH 4/4] delete one opengroupv1 message every 10 sec until there is no more --- app/sql.js | 58 +++++++++++++------------- ts/components/session/ActionsPanel.tsx | 13 ++++++ ts/data/data.ts | 6 +++ 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/app/sql.js b/app/sql.js index ceca42568..01a187724 100644 --- a/app/sql.js +++ b/app/sql.js @@ -113,6 +113,7 @@ module.exports = { getAllV2OpenGroupRooms, getV2OpenGroupRoomByRoomId, removeV2OpenGroupRoom, + removeOneOpenGroupV1Message, }; const CONVERSATIONS_TABLE = 'conversations'; @@ -1138,6 +1139,7 @@ async function updateToLokiSchemaVersion14(currentVersion, instance) { return; } console.log(`updateToLokiSchemaVersion${targetVersion}: starting...`); + await instance.run('BEGIN TRANSACTION;'); await instance.run('DROP TABLE IF EXISTS servers;'); await instance.run('DROP TABLE IF EXISTS sessions;'); @@ -1147,31 +1149,6 @@ async function updateToLokiSchemaVersion14(currentVersion, instance) { await instance.run('DROP TABLE IF EXISTS signedPreKeys;'); await instance.run('DROP TABLE IF EXISTS senderKeys;'); - console.time('removingOpengroupv1Messages'); - - let toRemoveCount = 0; - do { - // eslint-disable-next-line no-await-in-loop - const row = await instance.get(`SELECT count(*) from ${MESSAGES_TABLE} WHERE - conversationId LIKE 'publicChat:1@%';`); - toRemoveCount = row['count(*)']; - - if (toRemoveCount > 0) { - console.warn('toRemove count', toRemoveCount); - console.time('chunk'); - - // eslint-disable-next-line no-await-in-loop - await instance.all( - `DELETE FROM ${MESSAGES_TABLE} WHERE - conversationId LIKE 'publicChat:1@%' - ;` - ); - console.timeEnd('chunk'); - } - } while (toRemoveCount > 0); - - console.timeEnd('removingOpengroupv1Messages'); - await instance.run( `INSERT INTO loki_schema ( version @@ -1949,9 +1926,9 @@ async function saveMessages(arrayOfMessages) { await promise; } -async function removeMessage(id) { +async function removeMessage(id, instance) { if (!Array.isArray(id)) { - await db.run(`DELETE FROM ${MESSAGES_TABLE} WHERE id = $id;`, { $id: id }); + await (db || instance).run(`DELETE FROM ${MESSAGES_TABLE} WHERE id = $id;`, { $id: id }); return; } @@ -1960,7 +1937,7 @@ async function removeMessage(id) { } // Our node interface doesn't seem to allow you to replace one single ? with an array - await db.run( + await (db || instance).run( `DELETE FROM ${MESSAGES_TABLE} WHERE id IN ( ${id.map(() => '?').join(', ')} );`, id ); @@ -2849,3 +2826,28 @@ async function removeV2OpenGroupRoom(conversationId) { $conversationId: conversationId, }); } + +async function removeOneOpenGroupV1Message() { + // eslint-disable-next-line no-await-in-loop + const row = await db.get(`SELECT count(*) from ${MESSAGES_TABLE} WHERE + conversationId LIKE 'publicChat:1@%';`); + const toRemoveCount = row['count(*)']; + + if (toRemoveCount <= 0) { + return 0; + } + console.warn('left opengroupv1 message to remove: ', toRemoveCount); + const rowMessageIds = await db.all( + `SELECT id from ${MESSAGES_TABLE} WHERE conversationId LIKE 'publicChat:1@%' ORDER BY id LIMIT 1;` + ); + + const messagesIds = map(rowMessageIds, r => r.id)[0]; + + console.time('removeOneOpenGroupV1Message'); + + // eslint-disable-next-line no-await-in-loop + await removeMessage(messagesIds); + console.timeEnd('removeOneOpenGroupV1Message'); + + return toRemoveCount - 1; +} diff --git a/ts/components/session/ActionsPanel.tsx b/ts/components/session/ActionsPanel.tsx index 8cb800ba7..6a2fa7feb 100644 --- a/ts/components/session/ActionsPanel.tsx +++ b/ts/components/session/ActionsPanel.tsx @@ -15,6 +15,7 @@ import { hasSyncedInitialConfigurationItem, lastAvatarUploadTimestamp, removeConversation, + removeOneOpenGroupV1Message, } from '../../data/data'; import { OnionPaths } from '../../session/onions'; import { getMessageQueue } from '../../session/sending'; @@ -160,6 +161,16 @@ const triggerSyncIfNeeded = async () => { } }; +const scheduleDeleteOpenGroupV1Messages = async () => { + const leftToRemove = await removeOneOpenGroupV1Message(); + if (leftToRemove > 0) { + window?.log?.info(`We still have ${leftToRemove} opengroupv1 messages to remove...`); + setTimeout(scheduleDeleteOpenGroupV1Messages, 10000); + } else { + window?.log?.info('No more opengroupv1 messages to remove...'); + } +}; + const removeAllV1OpenGroups = async () => { const allV1Convos = (await getAllOpenGroupV1Conversations()).models || []; // do not remove messages of opengroupv1 for now. We have to find a way of doing it without making the whole app extremely slow @@ -180,6 +191,8 @@ const removeAllV1OpenGroups = async () => { window.log.warn(`failed to delete opengroupv1 ${v1Convo.id}`, e); } } + + setTimeout(scheduleDeleteOpenGroupV1Messages, 10000); }; const triggerAvatarReUploadIfNeeded = async () => { diff --git a/ts/data/data.ts b/ts/data/data.ts index 7772af2fa..cb2826eea 100644 --- a/ts/data/data.ts +++ b/ts/data/data.ts @@ -156,6 +156,7 @@ const channelsToMake = { addClosedGroupEncryptionKeyPair, isKeyPairAlreadySaved, removeAllClosedGroupEncryptionKeyPairs, + removeOneOpenGroupV1Message, // open group v2 ...channelstoMakeOpenGroupV2, @@ -947,3 +948,8 @@ export async function getSnodePoolFromDb(): Promise | null> { export async function updateSnodePoolOnDb(snodesAsJsonString: string): Promise { await exports.createOrUpdateItem({ id: SNODE_POOL_ITEM_ID, value: snodesAsJsonString }); } + +/** Returns the number of message left to remove (opengroupv1) */ +export async function removeOneOpenGroupV1Message(): Promise { + return channels.removeOneOpenGroupV1Message(); +}