From 561d60cfd589d2b05b79e0a009a200d9a5c3b752 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Mon, 21 Jan 2019 15:10:17 +1100 Subject: [PATCH 1/7] Sending messages now in a while loop like retrieving messages, added a failureCount and only delete node urls if they fail 3 times --- js/modules/loki_message_api.js | 74 ++++++++++++++++++++----------- js/modules/loki_snode_api.js | 32 +++++++++++-- libtextsecure/errors.js | 16 +++++++ libtextsecure/outgoing_message.js | 8 +--- 4 files changed, 94 insertions(+), 36 deletions(-) diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 1a3a47c54..55a674894 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -3,10 +3,6 @@ const fetch = require('node-fetch'); -// eslint-disable-next-line -const invert = p => new Promise((res, rej) => p.then(rej, res)); -const firstOf = ps => invert(Promise.all(ps.map(invert))); - // Will be raised (to 3?) when we get more nodes const MINIMUM_SUCCESSFUL_REQUESTS = 2; class LokiMessageAPI { @@ -15,11 +11,6 @@ class LokiMessageAPI { } async sendMessage(pubKey, data, messageTimeStamp, ttl) { - const swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); - if (!swarmNodes || swarmNodes.size === 0) { - throw Error('No swarm nodes to query!'); - } - const data64 = dcodeIO.ByteBuffer.wrap(data).toString('base64'); const timestamp = Math.floor(Date.now() / 1000); // Nonce is returned as a base64 string to include in header @@ -42,11 +33,12 @@ class LokiMessageAPI { // Something went horribly wrong throw err; } + let completedRequests = 0; - const requests = Array.from(swarmNodes).map(async node => { + const doRequest = async nodeUrl => { // TODO: Confirm sensible timeout const options = { - url: `${node}${this.messageServerPort}/store`, + url: `${nodeUrl}${this.messageServerPort}/store`, type: 'POST', responseType: undefined, timeout: 5000, @@ -69,8 +61,8 @@ class LokiMessageAPI { response = await fetch(options.url, fetchOptions); } catch (e) { log.error(options.type, options.url, 0, 'Error sending message'); - window.LokiSnodeAPI.unreachableNode(pubKey, node); - throw HTTPError('fetch error', 0, e.toString()); + window.LokiSnodeAPI.unreachableNode(pubKey, nodeUrl); + return; } let result; @@ -86,7 +78,8 @@ class LokiMessageAPI { } if (response.status >= 0 && response.status < 400) { - return result; + completedRequests += 1; + return; } log.error( options.type, @@ -95,13 +88,32 @@ class LokiMessageAPI { 'Error sending message' ); throw HTTPError('sendMessage: error response', response.status, result); - }); - try { - // TODO: Possibly change this to require more than a single response? - const result = await firstOf(requests); - return result; - } catch (err) { - throw err; + }; + + let swarmNodes; + while (completedRequests < MINIMUM_SUCCESSFUL_REQUESTS) { + try { + swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); + } catch (e) { + throw new window.textsecure.EmptySwarmError(pubKey, e); + } + if (!swarmNodes || swarmNodes.size === 0) { + if (completedRequests !== 0) { + // TODO: Decide how to handle some completed requests but not enough + return; + } + throw new window.textsecure.EmptySwarmError( + pubKey, + new Error('Ran out of swarm nodes to query') + ); + } + + const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedRequests; + await Promise.all( + Array.from(swarmNodes) + .splice(0, remainingRequests) + .map(nodeUrl => doRequest(nodeUrl)) + ); } } @@ -172,15 +184,27 @@ class LokiMessageAPI { }; while (completedRequests < MINIMUM_SUCCESSFUL_REQUESTS) { - const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedRequests; - const ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); - if (Object.keys(ourSwarmNodes).length < remainingRequests) { - // This means we don't have enough swarm nodes to meet the minimum threshold + let ourSwarmNodes; + try { + ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); + } catch (e) { + throw window.textsecure.EmptySwarmError( + window.textsecure.storage.user.getNumber(), + e + ); + } + if (!ourSwarmNodes || Object.keys(ourSwarmNodes).length === 0) { if (completedRequests !== 0) { // TODO: Decide how to handle some completed requests but not enough + return; } + throw window.textsecure.EmptySwarmError( + window.textsecure.storage.user.getNumber(), + new Error('Ran out of swarm nodes to query') + ); } + const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedRequests; await Promise.all( Object.entries(ourSwarmNodes) .splice(0, remainingRequests) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 80b999909..3ec6554a8 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -6,6 +6,7 @@ const dns = require('dns'); // Will be raised (to 3?) when we get more nodes const MINIMUM_SWARM_NODES = 1; +const FAILURE_THRESHOLD = 3; class LokiSnodeAPI { constructor({ url, swarmServerPort }) { @@ -16,6 +17,7 @@ class LokiSnodeAPI { this.swarmServerPort = swarmServerPort ? `:${swarmServerPort}` : ''; this.swarmsPendingReplenish = {}; this.ourSwarmNodes = {}; + this.contactSwarmNodes = {}; } getRandomSnodeAddress() { @@ -33,7 +35,26 @@ class LokiSnodeAPI { async unreachableNode(pubKey, nodeUrl) { if (pubKey === window.textsecure.storage.user.getNumber()) { - delete this.ourSwarmNodes[nodeUrl]; + if (!this.ourSwarmNodes[nodeUrl]) { + this.ourSwarmNodes[nodeUrl] = { + failureCount: 1, + }; + } else { + this.ourSwarmNodes[nodeUrl].failureCount += 1; + } + if (this.ourSwarmNodes[nodeUrl].failureCount >= FAILURE_THRESHOLD) { + delete this.ourSwarmNodes[nodeUrl]; + } + return; + } + if (!this.contactSwarmNodes[nodeUrl]) { + this.contactSwarmNodes[nodeUrl] = { + failureCount: 1, + }; + } else { + this.contactSwarmNodes[nodeUrl].failureCount += 1; + } + if (this.contactSwarmNodes[nodeUrl].failureCount < FAILURE_THRESHOLD) { return; } const conversation = window.ConversationController.get(pubKey); @@ -47,6 +68,7 @@ class LokiSnodeAPI { Conversation: Whisper.Conversation, } ); + delete this.contactSwarmNodes[nodeUrl]; } } @@ -74,7 +96,9 @@ class LokiSnodeAPI { } nodeAddresses.forEach(url => { - this.ourSwarmNodes[url] = {}; + this.ourSwarmNodes[url] = { + failureCount: 0, + }; }); } return this.ourSwarmNodes; @@ -155,7 +179,7 @@ class LokiSnodeAPI { 0, `Error getting swarm nodes for ${pubKey}` ); - throw HTTPError('fetch error', 0, e.toString()); + throw HTTPError('getSwarmNodes fetch error', 0, e.toString()); } let result; @@ -179,7 +203,7 @@ class LokiSnodeAPI { response.status, `Error getting swarm nodes for ${pubKey}` ); - throw HTTPError('sendMessage: error response', response.status, result); + throw HTTPError('getSwarmNodes: error response', response.status, result); } } diff --git a/libtextsecure/errors.js b/libtextsecure/errors.js index 51c879afe..a621ebf6f 100644 --- a/libtextsecure/errors.js +++ b/libtextsecure/errors.js @@ -127,6 +127,21 @@ } inherit(Error, UnregisteredUserError); + function EmptySwarmError(number, error) { + // eslint-disable-next-line prefer-destructuring + this.number = number.split('.')[0]; + + ReplayableError.call(this, { + name: 'EmptySwarmError', + message: 'Could not get any swarm nodes to query', + }); + + if (error) { + appendStack(this, error); + } + } + inherit(ReplayableError, PoWError); + function PoWError(number, error) { // eslint-disable-next-line prefer-destructuring this.number = number.split('.')[0]; @@ -151,4 +166,5 @@ window.textsecure.MessageError = MessageError; window.textsecure.SignedPreKeyRotationError = SignedPreKeyRotationError; window.textsecure.PoWError = PoWError; + window.textsecure.EmptySwarmError = EmptySwarmError; })(); diff --git a/libtextsecure/outgoing_message.js b/libtextsecure/outgoing_message.js index be11d8905..668e4067e 100644 --- a/libtextsecure/outgoing_message.js +++ b/libtextsecure/outgoing_message.js @@ -186,13 +186,7 @@ OutgoingMessage.prototype = { async transmitMessage(number, data, timestamp, ttl = 24 * 60 * 60) { const pubKey = number; try { - const result = await this.lokiMessageAPI.sendMessage( - pubKey, - data, - timestamp, - ttl - ); - return result; + await this.lokiMessageAPI.sendMessage(pubKey, data, timestamp, ttl); } catch (e) { if (e.name === 'HTTPError' && (e.code !== 409 && e.code !== 410)) { // 409 and 410 should bubble and be handled by doSendMessage From c83661ce3f1b4d3e63bc47ea6e3385f2624a134f Mon Sep 17 00:00:00 2001 From: Beaudan Date: Mon, 21 Jan 2019 17:34:27 +1100 Subject: [PATCH 2/7] Added DNS resolution error for when lokinet isn't working, now keeping track of the nodes that have been queried and not trying them again --- js/modules/loki_message_api.js | 75 +++++++++++++++++++++++++++------- js/modules/loki_snode_api.js | 5 ++- libtextsecure/errors.js | 11 +++++ 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 55a674894..358dd33c0 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -5,6 +5,7 @@ const fetch = require('node-fetch'); // Will be raised (to 3?) when we get more nodes const MINIMUM_SUCCESSFUL_REQUESTS = 2; + class LokiMessageAPI { constructor({ messageServerPort }) { this.messageServerPort = messageServerPort ? `:${messageServerPort}` : ''; @@ -33,7 +34,8 @@ class LokiMessageAPI { // Something went horribly wrong throw err; } - let completedRequests = 0; + const completedNodes = []; + let canResolve = true; const doRequest = async nodeUrl => { // TODO: Confirm sensible timeout @@ -60,8 +62,15 @@ class LokiMessageAPI { try { response = await fetch(options.url, fetchOptions); } catch (e) { + if (e.code === 'ENOTFOUND') { + // TODO: Handle the case where lokinet is not working + canResolve = false; + return; + } log.error(options.type, options.url, 0, 'Error sending message'); - window.LokiSnodeAPI.unreachableNode(pubKey, nodeUrl); + if (window.LokiSnodeAPI.unreachableNode(pubKey, nodeUrl)) { + completedNodes.push(nodeUrl); + } return; } @@ -78,7 +87,7 @@ class LokiMessageAPI { } if (response.status >= 0 && response.status < 400) { - completedRequests += 1; + completedNodes.push(nodeUrl); return; } log.error( @@ -91,14 +100,28 @@ class LokiMessageAPI { }; let swarmNodes; - while (completedRequests < MINIMUM_SUCCESSFUL_REQUESTS) { + while (completedNodes.length < MINIMUM_SUCCESSFUL_REQUESTS) { + if (!canResolve) { + throw new window.textsecure.DNSResolutionError('Sending messages'); + } try { swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); + // Filter out the nodes we have already got responses from + swarmNodes = Object.keys(swarmNodes) + .filter(node => !(node in completedNodes)) + .reduce( + // eslint-disable-next-line no-loop-func + (obj, node) => ({ + ...obj, + [node]: swarmNodes[node], + }), + {} + ); } catch (e) { throw new window.textsecure.EmptySwarmError(pubKey, e); } if (!swarmNodes || swarmNodes.size === 0) { - if (completedRequests !== 0) { + if (completedNodes.length !== 0) { // TODO: Decide how to handle some completed requests but not enough return; } @@ -108,7 +131,8 @@ class LokiMessageAPI { ); } - const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedRequests; + const remainingRequests = + MINIMUM_SUCCESSFUL_REQUESTS - completedNodes.length; await Promise.all( Array.from(swarmNodes) .splice(0, remainingRequests) @@ -119,7 +143,8 @@ class LokiMessageAPI { async retrieveMessages(callback) { const ourKey = window.textsecure.storage.user.getNumber(); - let completedRequests = 0; + const completedNodes = []; + let canResolve = true; const doRequest = async (nodeUrl, nodeData) => { // TODO: Confirm sensible timeout @@ -147,6 +172,11 @@ class LokiMessageAPI { try { response = await fetch(options.url, fetchOptions); } catch (e) { + if (e.code === 'ENOTFOUND') { + // TODO: Handle the case where lokinet is not working + canResolve = false; + return; + } // TODO: Maybe we shouldn't immediately delete? // And differentiate between different connectivity issues log.error( @@ -155,7 +185,9 @@ class LokiMessageAPI { 0, `Error retrieving messages from ${nodeUrl}` ); - window.LokiSnodeAPI.unreachableNode(ourKey, nodeUrl); + if (window.LokiSnodeAPI.unreachableNode(ourKey, nodeUrl)) { + completedNodes.push(nodeUrl); + } return; } @@ -170,7 +202,7 @@ class LokiMessageAPI { } else { result = await response.text(); } - completedRequests += 1; + completedNodes.push(nodeUrl); if (response.status === 200) { if (result.lastHash) { @@ -183,28 +215,43 @@ class LokiMessageAPI { log.error(options.type, options.url, response.status, 'Error'); }; - while (completedRequests < MINIMUM_SUCCESSFUL_REQUESTS) { + while (completedNodes.length < MINIMUM_SUCCESSFUL_REQUESTS) { + if (!canResolve) { + throw new window.textsecure.DNSResolutionError('Retrieving messages'); + } let ourSwarmNodes; try { ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); + // Filter out the nodes we have already got responses from + ourSwarmNodes = Object.keys(ourSwarmNodes) + .filter(node => !(node in completedNodes)) + .reduce( + // eslint-disable-next-line no-loop-func + (obj, node) => ({ + ...obj, + [node]: ourSwarmNodes[node], + }), + {} + ); } catch (e) { - throw window.textsecure.EmptySwarmError( + throw new window.textsecure.EmptySwarmError( window.textsecure.storage.user.getNumber(), e ); } if (!ourSwarmNodes || Object.keys(ourSwarmNodes).length === 0) { - if (completedRequests !== 0) { + if (completedNodes.length !== 0) { // TODO: Decide how to handle some completed requests but not enough return; } - throw window.textsecure.EmptySwarmError( + throw new window.textsecure.EmptySwarmError( window.textsecure.storage.user.getNumber(), new Error('Ran out of swarm nodes to query') ); } - const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedRequests; + const remainingRequests = + MINIMUM_SUCCESSFUL_REQUESTS - completedNodes.length; await Promise.all( Object.entries(ourSwarmNodes) .splice(0, remainingRequests) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 3ec6554a8..7dd4cffa6 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -45,7 +45,7 @@ class LokiSnodeAPI { if (this.ourSwarmNodes[nodeUrl].failureCount >= FAILURE_THRESHOLD) { delete this.ourSwarmNodes[nodeUrl]; } - return; + return false; } if (!this.contactSwarmNodes[nodeUrl]) { this.contactSwarmNodes[nodeUrl] = { @@ -55,7 +55,7 @@ class LokiSnodeAPI { this.contactSwarmNodes[nodeUrl].failureCount += 1; } if (this.contactSwarmNodes[nodeUrl].failureCount < FAILURE_THRESHOLD) { - return; + return false; } const conversation = window.ConversationController.get(pubKey); const swarmNodes = conversation.get('swarmNodes'); @@ -70,6 +70,7 @@ class LokiSnodeAPI { ); delete this.contactSwarmNodes[nodeUrl]; } + return true; } updateLastHash(nodeUrl, hash) { diff --git a/libtextsecure/errors.js b/libtextsecure/errors.js index a621ebf6f..f76fb3795 100644 --- a/libtextsecure/errors.js +++ b/libtextsecure/errors.js @@ -157,6 +157,16 @@ } inherit(ReplayableError, PoWError); + function DNSResolutionError(message) { + // eslint-disable-next-line prefer-destructuring + + ReplayableError.call(this, { + name: 'DNSResolutionError', + message: `Error resolving url: ${message}`, + }); + } + inherit(ReplayableError, DNSResolutionError); + window.textsecure.UnregisteredUserError = UnregisteredUserError; window.textsecure.SendMessageNetworkError = SendMessageNetworkError; window.textsecure.IncomingIdentityKeyError = IncomingIdentityKeyError; @@ -167,4 +177,5 @@ window.textsecure.SignedPreKeyRotationError = SignedPreKeyRotationError; window.textsecure.PoWError = PoWError; window.textsecure.EmptySwarmError = EmptySwarmError; + window.textsecure.DNSResolutionError = DNSResolutionError; })(); From 8b5106433f50cc40f73b8fb593a24d50b6976398 Mon Sep 17 00:00:00 2001 From: Mikunj Varsani Date: Wed, 23 Jan 2019 17:03:41 +1100 Subject: [PATCH 3/7] Update libtextsecure/errors.js Co-Authored-By: BeaudanBrown --- libtextsecure/errors.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libtextsecure/errors.js b/libtextsecure/errors.js index f76fb3795..7f972492e 100644 --- a/libtextsecure/errors.js +++ b/libtextsecure/errors.js @@ -140,7 +140,7 @@ appendStack(this, error); } } - inherit(ReplayableError, PoWError); + inherit(ReplayableError, EmptySwarmError); function PoWError(number, error) { // eslint-disable-next-line prefer-destructuring From dde63a552fb61b93e9839d0b5218e59442fa9dcb Mon Sep 17 00:00:00 2001 From: Beaudan Date: Fri, 25 Jan 2019 12:48:51 +1100 Subject: [PATCH 4/7] Fix bug with replayable errors, fix bug with set representation of swarmNodes not being handled properly --- js/models/messages.js | 2 ++ js/modules/data.js | 37 ++++++++++------------------------ js/modules/loki_message_api.js | 16 +++------------ js/modules/loki_snode_api.js | 6 +++--- 4 files changed, 19 insertions(+), 42 deletions(-) diff --git a/js/models/messages.js b/js/models/messages.js index fe684e961..790349056 100644 --- a/js/models/messages.js +++ b/js/models/messages.js @@ -847,6 +847,8 @@ e.name === 'SendMessageNetworkError' || e.name === 'SignedPreKeyRotationError' || e.name === 'OutgoingIdentityKeyError' || + e.name === 'DNSResolutionError' || + e.name === 'EmptySwarmError' || e.name === 'PoWError' ); }, diff --git a/js/modules/data.js b/js/modules/data.js index 1d937b521..c5489ae6c 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -663,18 +663,14 @@ async function removeAllSessions(id) { function setifyProperty(data, propertyName) { if (!data) return data; const returnData = data; - if (returnData[propertyName]) { + if (returnData[propertyName] && Array.isArray(returnData[propertyName])) { returnData[propertyName] = new Set(returnData[propertyName]); } return returnData; } async function getSwarmNodesByPubkey(pubkey) { - let swarmNodes = await channels.getSwarmNodesByPubkey(pubkey); - if (Array.isArray(swarmNodes)) { - swarmNodes = new Set(swarmNodes); - } - return swarmNodes; + return channels.getSwarmNodesByPubkey(pubkey); } async function getConversationCount() { @@ -682,11 +678,7 @@ async function getConversationCount() { } async function saveConversation(data) { - const storeData = data; - if (storeData.swarmNodes) { - storeData.swarmNodes = Array.from(storeData.swarmNodes); - } - await channels.saveConversation(storeData); + await channels.saveConversation(data); } async function saveConversations(data) { @@ -694,8 +686,7 @@ async function saveConversations(data) { } async function getConversationById(id, { Conversation }) { - const rawData = await channels.getConversationById(id); - const data = setifyProperty(rawData, 'swarmNodes'); + const data = await channels.getConversationById(id); return new Conversation(data); } @@ -704,8 +695,10 @@ async function updateConversation(id, data, { Conversation }) { if (!existing) { throw new Error(`Conversation ${id} does not exist!`); } + const setData = setifyProperty(data, 'swarmNodes'); + const setExisting = setifyProperty(existing.attributes, 'swarmNodes'); - const merged = merge({}, existing.attributes, data); + const merged = merge({}, setExisting, setData); if (merged.swarmNodes instanceof Set) { merged.swarmNodes = Array.from(merged.swarmNodes); } @@ -729,9 +722,7 @@ async function _removeConversations(ids) { } async function getAllConversations({ ConversationCollection }) { - const conversations = (await channels.getAllConversations()).map(c => - setifyProperty(c, 'swarmNodes') - ); + const conversations = await channels.getAllConversations(); const collection = new ConversationCollection(); collection.add(conversations); @@ -744,9 +735,7 @@ async function getAllConversationIds() { } async function getAllPrivateConversations({ ConversationCollection }) { - const conversations = (await channels.getAllPrivateConversations()).map(c => - setifyProperty(c, 'swarmNodes') - ); + const conversations = await channels.getAllPrivateConversations(); const collection = new ConversationCollection(); collection.add(conversations); @@ -754,9 +743,7 @@ async function getAllPrivateConversations({ ConversationCollection }) { } async function getAllGroupsInvolvingId(id, { ConversationCollection }) { - const conversations = (await channels.getAllGroupsInvolvingId(id)).map(c => - setifyProperty(c, 'swarmNodes') - ); + const conversations = await channels.getAllGroupsInvolvingId(id); const collection = new ConversationCollection(); collection.add(conversations); @@ -764,9 +751,7 @@ async function getAllGroupsInvolvingId(id, { ConversationCollection }) { } async function searchConversations(query, { ConversationCollection }) { - const conversations = (await channels.searchConversations(query)).map(c => - setifyProperty(c, 'swarmNodes') - ); + const conversations = await channels.searchConversations(query); const collection = new ConversationCollection(); collection.add(conversations); diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index 358dd33c0..bb333cd01 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -106,21 +106,11 @@ class LokiMessageAPI { } try { swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); - // Filter out the nodes we have already got responses from - swarmNodes = Object.keys(swarmNodes) - .filter(node => !(node in completedNodes)) - .reduce( - // eslint-disable-next-line no-loop-func - (obj, node) => ({ - ...obj, - [node]: swarmNodes[node], - }), - {} - ); + swarmNodes = swarmNodes.filter(node => !(node in completedNodes)); } catch (e) { throw new window.textsecure.EmptySwarmError(pubKey, e); } - if (!swarmNodes || swarmNodes.size === 0) { + if (!swarmNodes || swarmNodes.length === 0) { if (completedNodes.length !== 0) { // TODO: Decide how to handle some completed requests but not enough return; @@ -134,7 +124,7 @@ class LokiMessageAPI { const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedNodes.length; await Promise.all( - Array.from(swarmNodes) + swarmNodes .splice(0, remainingRequests) .map(nodeUrl => doRequest(nodeUrl)) ); diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 7dd4cffa6..1ba2a2124 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -108,7 +108,7 @@ class LokiSnodeAPI { async getSwarmNodesByPubkey(pubKey) { const swarmNodes = await window.Signal.Data.getSwarmNodesByPubkey(pubKey); // TODO: Check if swarm list is below a threshold rather than empty - if (swarmNodes && swarmNodes.size !== 0) { + if (swarmNodes && swarmNodes.length !== 0) { return swarmNodes; } return this.replenishSwarm(pubKey); @@ -120,10 +120,10 @@ class LokiSnodeAPI { this.swarmsPendingReplenish[pubKey] = new Promise(async resolve => { let newSwarmNodes; try { - newSwarmNodes = new Set(await this.getSwarmNodes(pubKey)); + newSwarmNodes = await this.getSwarmNodes(pubKey); } catch (e) { // TODO: Handle these errors sensibly - newSwarmNodes = new Set([]); + newSwarmNodes = []; } conversation.set({ swarmNodes: newSwarmNodes }); await window.Signal.Data.updateConversation( From 9ece4e1fd8c454d017fb81f1bad88e8e11e17482 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Fri, 25 Jan 2019 14:35:09 +1100 Subject: [PATCH 5/7] Review comments --- js/modules/data.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/modules/data.js b/js/modules/data.js index c5489ae6c..e8afe5b14 100644 --- a/js/modules/data.js +++ b/js/modules/data.js @@ -662,8 +662,8 @@ async function removeAllSessions(id) { function setifyProperty(data, propertyName) { if (!data) return data; - const returnData = data; - if (returnData[propertyName] && Array.isArray(returnData[propertyName])) { + const returnData = { ...data }; + if (Array.isArray(returnData[propertyName])) { returnData[propertyName] = new Set(returnData[propertyName]); } return returnData; From f6d167eda148efdd544a887ac4ec6aa29d9aba1a Mon Sep 17 00:00:00 2001 From: Beaudan Date: Wed, 30 Jan 2019 11:43:18 +1100 Subject: [PATCH 6/7] Split logic for getting swarm nodes from db, getting from lokinet and saving to db. Now tracking successfulRequests when making queries on top of the completedNodes list --- js/conversation_controller.js | 4 +- js/modules/loki_message_api.js | 105 ++++++++++++++++++--------------- js/modules/loki_snode_api.js | 29 +++++---- 3 files changed, 77 insertions(+), 61 deletions(-) diff --git a/js/conversation_controller.js b/js/conversation_controller.js index feaa2e1eb..298f4b931 100644 --- a/js/conversation_controller.js +++ b/js/conversation_controller.js @@ -181,8 +181,10 @@ return conversation; } - window.LokiSnodeAPI.replenishSwarm(id); try { + conversation.attributes.swarmNodes = await window.LokiSnodeAPI.getFreshSwarmNodes( + id + ); await window.Signal.Data.saveConversation(conversation.attributes, { Conversation: Whisper.Conversation, }); diff --git a/js/modules/loki_message_api.js b/js/modules/loki_message_api.js index bb333cd01..dfd67cc90 100644 --- a/js/modules/loki_message_api.js +++ b/js/modules/loki_message_api.js @@ -1,4 +1,5 @@ /* eslint-disable no-await-in-loop */ +/* eslint-disable no-loop-func */ /* global log, dcodeIO, window, callWorker */ const fetch = require('node-fetch'); @@ -35,6 +36,7 @@ class LokiMessageAPI { throw err; } const completedNodes = []; + let successfulRequests = 0; let canResolve = true; const doRequest = async nodeUrl => { @@ -43,7 +45,7 @@ class LokiMessageAPI { url: `${nodeUrl}${this.messageServerPort}/store`, type: 'POST', responseType: undefined, - timeout: 5000, + timeout: 10000, }; const fetchOptions = { @@ -70,6 +72,7 @@ class LokiMessageAPI { log.error(options.type, options.url, 0, 'Error sending message'); if (window.LokiSnodeAPI.unreachableNode(pubKey, nodeUrl)) { completedNodes.push(nodeUrl); + swarmNodes = swarmNodes.filter(node => node !== nodeUrl); } return; } @@ -88,6 +91,8 @@ class LokiMessageAPI { if (response.status >= 0 && response.status < 400) { completedNodes.push(nodeUrl); + swarmNodes = swarmNodes.filter(node => node !== nodeUrl); + successfulRequests += 1; return; } log.error( @@ -100,27 +105,30 @@ class LokiMessageAPI { }; let swarmNodes; - while (completedNodes.length < MINIMUM_SUCCESSFUL_REQUESTS) { + try { + swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); + } catch (e) { + throw new window.textsecure.EmptySwarmError(pubKey, e); + } + while (successfulRequests < MINIMUM_SUCCESSFUL_REQUESTS) { if (!canResolve) { throw new window.textsecure.DNSResolutionError('Sending messages'); } - try { - swarmNodes = await window.LokiSnodeAPI.getSwarmNodesByPubkey(pubKey); - swarmNodes = swarmNodes.filter(node => !(node in completedNodes)); - } catch (e) { - throw new window.textsecure.EmptySwarmError(pubKey, e); - } if (!swarmNodes || swarmNodes.length === 0) { - if (completedNodes.length !== 0) { - // TODO: Decide how to handle some completed requests but not enough - return; + swarmNodes = await window.LokiSnodeAPI.getFreshSwarmNodes(pubKey); + swarmNodes = swarmNodes.filter(node => !(node in completedNodes)); + if (!swarmNodes || swarmNodes.length === 0) { + if (successfulRequests !== 0) { + // TODO: Decide how to handle some completed requests but not enough + return; + } + throw new window.textsecure.EmptySwarmError( + pubKey, + new Error('Ran out of swarm nodes to query') + ); } - throw new window.textsecure.EmptySwarmError( - pubKey, - new Error('Ran out of swarm nodes to query') - ); + await window.LokiSnodeAPI.saveSwarmNodes(pubKey, swarmNodes); } - const remainingRequests = MINIMUM_SUCCESSFUL_REQUESTS - completedNodes.length; await Promise.all( @@ -135,6 +143,7 @@ class LokiMessageAPI { const ourKey = window.textsecure.storage.user.getNumber(); const completedNodes = []; let canResolve = true; + let successfulRequests = 0; const doRequest = async (nodeUrl, nodeData) => { // TODO: Confirm sensible timeout @@ -142,7 +151,7 @@ class LokiMessageAPI { url: `${nodeUrl}${this.messageServerPort}/retrieve`, type: 'GET', responseType: 'json', - timeout: 5000, + timeout: 10000, }; const headers = { @@ -167,8 +176,6 @@ class LokiMessageAPI { canResolve = false; return; } - // TODO: Maybe we shouldn't immediately delete? - // And differentiate between different connectivity issues log.error( options.type, options.url, @@ -177,6 +184,7 @@ class LokiMessageAPI { ); if (window.LokiSnodeAPI.unreachableNode(ourKey, nodeUrl)) { completedNodes.push(nodeUrl); + delete ourSwarmNodes[nodeUrl]; } return; } @@ -193,51 +201,54 @@ class LokiMessageAPI { result = await response.text(); } completedNodes.push(nodeUrl); + delete ourSwarmNodes[nodeUrl]; if (response.status === 200) { if (result.lastHash) { window.LokiSnodeAPI.updateLastHash(nodeUrl, result.lastHash); callback(result.messages); } + successfulRequests += 1; return; } // Handle error from snode log.error(options.type, options.url, response.status, 'Error'); }; - while (completedNodes.length < MINIMUM_SUCCESSFUL_REQUESTS) { + let ourSwarmNodes; + try { + ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); + } catch (e) { + throw new window.textsecure.EmptySwarmError( + window.textsecure.storage.user.getNumber(), + e + ); + } + while (successfulRequests < MINIMUM_SUCCESSFUL_REQUESTS) { if (!canResolve) { throw new window.textsecure.DNSResolutionError('Retrieving messages'); } - let ourSwarmNodes; - try { - ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); - // Filter out the nodes we have already got responses from - ourSwarmNodes = Object.keys(ourSwarmNodes) - .filter(node => !(node in completedNodes)) - .reduce( - // eslint-disable-next-line no-loop-func - (obj, node) => ({ - ...obj, - [node]: ourSwarmNodes[node], - }), - {} + if (Object.keys(ourSwarmNodes).length === 0) { + try { + ourSwarmNodes = await window.LokiSnodeAPI.getOurSwarmNodes(); + // Filter out the nodes we have already got responses from + completedNodes.forEach(nodeUrl => delete ourSwarmNodes[nodeUrl]); + } catch (e) { + throw new window.textsecure.EmptySwarmError( + window.textsecure.storage.user.getNumber(), + e + ); + } + if (Object.keys(ourSwarmNodes).length === 0) { + if (successfulRequests !== 0) { + // TODO: Decide how to handle some completed requests but not enough + return; + } + throw new window.textsecure.EmptySwarmError( + window.textsecure.storage.user.getNumber(), + new Error('Ran out of swarm nodes to query') ); - } catch (e) { - throw new window.textsecure.EmptySwarmError( - window.textsecure.storage.user.getNumber(), - e - ); - } - if (!ourSwarmNodes || Object.keys(ourSwarmNodes).length === 0) { - if (completedNodes.length !== 0) { - // TODO: Decide how to handle some completed requests but not enough - return; } - throw new window.textsecure.EmptySwarmError( - window.textsecure.storage.user.getNumber(), - new Error('Ran out of swarm nodes to query') - ); } const remainingRequests = diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 1ba2a2124..d6000c3e8 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -1,3 +1,4 @@ +/* eslint-disable class-methods-use-this */ /* global log, window, Whisper */ const fetch = require('node-fetch'); @@ -107,15 +108,25 @@ class LokiSnodeAPI { async getSwarmNodesByPubkey(pubKey) { const swarmNodes = await window.Signal.Data.getSwarmNodesByPubkey(pubKey); - // TODO: Check if swarm list is below a threshold rather than empty - if (swarmNodes && swarmNodes.length !== 0) { + if (swarmNodes) { return swarmNodes; } - return this.replenishSwarm(pubKey); + return []; } - async replenishSwarm(pubKey) { + async saveSwarmNodes(pubKey, swarmNodes) { const conversation = window.ConversationController.get(pubKey); + conversation.set({ swarmNodes }); + await window.Signal.Data.updateConversation( + conversation.id, + conversation.attributes, + { + Conversation: Whisper.Conversation, + } + ); + } + + async getFreshSwarmNodes(pubKey) { if (!(pubKey in this.swarmsPendingReplenish)) { this.swarmsPendingReplenish[pubKey] = new Promise(async resolve => { let newSwarmNodes; @@ -125,14 +136,6 @@ class LokiSnodeAPI { // TODO: Handle these errors sensibly newSwarmNodes = []; } - conversation.set({ swarmNodes: newSwarmNodes }); - await window.Signal.Data.updateConversation( - conversation.id, - conversation.attributes, - { - Conversation: Whisper.Conversation, - } - ); resolve(newSwarmNodes); }); } @@ -149,7 +152,7 @@ class LokiSnodeAPI { url: `http://${node}${this.swarmServerPort}/json_rpc`, type: 'POST', responseType: 'json', - timeout: 5000, + timeout: 10000, }; const body = { From f2acde219a216897a1b7f4902722eb1a01e839c9 Mon Sep 17 00:00:00 2001 From: Beaudan Date: Wed, 30 Jan 2019 11:57:31 +1100 Subject: [PATCH 7/7] Review suggestion --- js/conversation_controller.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/js/conversation_controller.js b/js/conversation_controller.js index 298f4b931..c92f07c9c 100644 --- a/js/conversation_controller.js +++ b/js/conversation_controller.js @@ -182,9 +182,8 @@ } try { - conversation.attributes.swarmNodes = await window.LokiSnodeAPI.getFreshSwarmNodes( - id - ); + const swarmNodes = await window.LokiSnodeAPI.getFreshSwarmNodes(id); + conversation.set({ swarmNodes}); await window.Signal.Data.saveConversation(conversation.attributes, { Conversation: Whisper.Conversation, });