From 99679741c148d1ce653289fddf344bb27cb9fb87 Mon Sep 17 00:00:00 2001 From: Ryan Tharp Date: Thu, 19 Mar 2020 14:23:23 -0700 Subject: [PATCH] some renames, improve refresh random pool failures better --- js/modules/loki_snode_api.js | 192 ++++++++++++++++++++--------------- 1 file changed, 110 insertions(+), 82 deletions(-) diff --git a/js/modules/loki_snode_api.js b/js/modules/loki_snode_api.js index 9f3efcb76..1b78cd6a3 100644 --- a/js/modules/loki_snode_api.js +++ b/js/modules/loki_snode_api.js @@ -1,10 +1,12 @@ /* eslint-disable class-methods-use-this */ -/* global window, ConversationController, _, log */ +/* global window, ConversationController, _, log, clearTimeout */ const is = require('@sindresorhus/is'); const { lokiRpc } = require('./loki_rpc'); -const RANDOM_SNODES_TO_USE = 3; +const RANDOM_SNODES_TO_USE_FOR_PUBKEY_SWARM = 3; +const RANDOM_SNODES_POOL_SIZE = 1024; +const SEED_NODE_RETRIES = 3; class LokiSnodeAPI { constructor({ serverUrl, localUrl }) { @@ -15,14 +17,14 @@ class LokiSnodeAPI { this.localUrl = localUrl; // localhost.loki this.randomSnodePool = []; this.swarmsPendingReplenish = {}; - this.initialiseRandomPoolPromise = false; + this.refreshRandomPoolPromise = false; } async getRandomSnodeAddress() { /* resolve random snode */ if (this.randomSnodePool.length === 0) { // allow exceptions to pass through upwards - await this.initialiseRandomPool(); + await this.refreshRandomPool(); } if (this.randomSnodePool.length === 0) { throw new window.textsecure.SeedNodeError('Invalid seed node response'); @@ -32,85 +34,110 @@ class LokiSnodeAPI { ]; } - async initialiseRandomPool( - seedNodes = [...window.seedNodeList], - consecutiveErrors = 0 + + async refreshRandomPool( + seedNodes = [...window.seedNodeList] ) { // if currently not in progress - if (this.initialiseRandomPoolPromise === false) { - // FIXME: add timeout + if (this.refreshRandomPoolPromise === false) { // set lock - this.initialiseRandomPoolPromise = new Promise(async resolve => { - const params = { - limit: 1024, - active_only: true, - fields: { - public_ip: true, - storage_port: true, - pubkey_x25519: true, - pubkey_ed25519: true, - }, - }; - const seedNode = seedNodes.splice( - Math.floor(Math.random() * seedNodes.length), - 1 - )[0]; - let snodes = []; - try { - log.info('loki_snodes: Refreshing random snode pool'); - const response = await lokiRpc( - `http://${seedNode.ip}`, - seedNode.port, - 'get_n_service_nodes', - params, - {}, // Options - '/json_rpc' // Seed request endpoint - ); - // Filter 0.0.0.0 nodes which haven't submitted uptime proofs - snodes = response.result.service_node_states.filter( - snode => snode.public_ip !== '0.0.0.0' - ); - this.randomSnodePool = snodes.map(snode => ({ - ip: snode.public_ip, - port: snode.storage_port, - pubkey_x25519: snode.pubkey_x25519, - pubkey_ed25519: snode.pubkey_ed25519, - })); - log.info( - 'loki_snodes: Refreshed random snode pool with', - this.randomSnodePool.length, - 'snodes' - ); - } catch (e) { - log.warn( - 'loki_snodes: initialiseRandomPool error', - e.code, - e.message - ); - if (consecutiveErrors < 3) { - // retry after a possible delay - setTimeout(() => { - log.info( - 'loki_snodes: Retrying initialising random snode pool, try #', - consecutiveErrors - ); - this.initialiseRandomPool(seedNodes, consecutiveErrors + 1); - }, consecutiveErrors * consecutiveErrors * 5000); - } else { - log.error('loki_snodes: Giving up trying to contact seed node'); - if (snodes.length === 0) { - throw new window.textsecure.SeedNodeError( - 'Failed to contact seed node' - ); + this.refreshRandomPoolPromise = new Promise(async (resolve, reject) => { + let timeoutTimer = null + // private retry container + const trySeedNode = async (consecutiveErrors = 0) => { + const params = { + limit: RANDOM_SNODES_POOL_SIZE, + active_only: true, + fields: { + public_ip: true, + storage_port: true, + pubkey_x25519: true, + pubkey_ed25519: true, + }, + }; + const seedNode = seedNodes.splice( + Math.floor(Math.random() * seedNodes.length), + 1 + )[0]; + let snodes = []; + try { + log.info('loki_snodes:::refreshRandomPoolPromise - Refreshing random snode pool'); + const response = await lokiRpc( + `http://${seedNode.ip}`, + seedNode.port, + 'get_n_service_nodes', + params, + {}, // Options + '/json_rpc' // Seed request endpoint + ); + // Filter 0.0.0.0 nodes which haven't submitted uptime proofs + snodes = response.result.service_node_states.filter( + snode => snode.public_ip !== '0.0.0.0' + ); + this.randomSnodePool = snodes.map(snode => ({ + ip: snode.public_ip, + port: snode.storage_port, + pubkey_x25519: snode.pubkey_x25519, + pubkey_ed25519: snode.pubkey_ed25519, + })); + log.info( + 'loki_snodes:::refreshRandomPoolPromise - Refreshed random snode pool with', + this.randomSnodePool.length, + 'snodes' + ); + // clear lock + this.refreshRandomPoolPromise = null; + if (timeoutTimer !== null) { + clearTimeout(timeoutTimer); + timeoutTimer = null; + } + resolve(); + } catch (e) { + log.warn( + 'loki_snodes:::refreshRandomPoolPromise - error', + e.code, + e.message + ); + if (consecutiveErrors < SEED_NODE_RETRIES) { + // retry after a possible delay + setTimeout(() => { + log.info( + 'loki_snodes:::refreshRandomPoolPromise - Retrying initialising random snode pool, try #', + consecutiveErrors + ); + trySeedNode(consecutiveErrors + 1); + }, consecutiveErrors * consecutiveErrors * 5000); + } else { + log.error('loki_snodes:::refreshRandomPoolPromise - Giving up trying to contact seed node'); + if (snodes.length === 0) { + this.refreshRandomPoolPromise = null; // clear lock + if (timeoutTimer !== null) { + clearTimeout(timeoutTimer); + timeoutTimer = null; + } + reject() + } } } } - // clear lock - this.initialiseRandomPoolPromise = null; - resolve(); + const delay = (SEED_NODE_RETRIES + 1) * (SEED_NODE_RETRIES + 1) * 5000; + timeoutTimer = setTimeout(() => { + log.warn('loki_snodes:::refreshRandomPoolPromise - TIMEDOUT after', delay, 's'); + reject(); + }, delay); + trySeedNode() }); } - await this.initialiseRandomPoolPromise; + try { + await this.refreshRandomPoolPromise; + } catch(e) { + // we will throw for each time initialiseRandomPool has been called in parallel + log.error('loki_snodes:::refreshRandomPoolPromise - error', e.code, e.message); + throw new window.textsecure.SeedNodeError( + 'Failed to contact seed node' + ); + } + log.info('loki_snodes:::refreshRandomPoolPromise - RESOLVED') } // unreachableNode.url is like 9hrje1bymy7hu6nmtjme9idyu3rm8gr3mkstakjyuw1997t7w4ny.snode @@ -119,7 +146,7 @@ class LokiSnodeAPI { const swarmNodes = [...conversation.get('swarmNodes')]; if (typeof unreachableNode === 'string') { log.warn( - 'loki_snodes::unreachableNode: String passed as unreachableNode to unreachableNode' + 'loki_snodes:::unreachableNode - String passed as unreachableNode to unreachableNode' ); return swarmNodes; } @@ -137,7 +164,7 @@ class LokiSnodeAPI { }); if (!found) { log.warn( - `loki_snodes::unreachableNode snode ${unreachableNode.ip}:${ + `loki_snodes:::unreachableNode - snode ${unreachableNode.ip}:${ unreachableNode.port } has already been marked as bad` ); @@ -196,7 +223,7 @@ class LokiSnodeAPI { try { newSwarmNodes = await this.getSwarmNodes(pubKey); } catch (e) { - log.error('loki_snodes: getFreshSwarmNodes error', e.code, e.message); + log.error('loki_snodes:::getFreshSwarmNodes - error', e.code, e.message); // TODO: Handle these errors sensibly newSwarmNodes = []; } @@ -223,7 +250,7 @@ class LokiSnodeAPI { ); if (!result) { log.warn( - `getSnodesForPubkey lokiRpc on ${snode.ip}:${ + `loki_snode:::getSnodesForPubkey - lokiRpc on ${snode.ip}:${ snode.port } returned falsish value`, result @@ -231,8 +258,9 @@ class LokiSnodeAPI { return []; } if (!result.snodes) { + // we hit this when snode gives 500s log.warn( - `getSnodesForPubkey lokiRpc on ${snode.ip}:${ + `loki_snode:::getSnodesForPubkey - lokiRpc on ${snode.ip}:${ snode.port } returned falsish value for snodes`, result @@ -244,7 +272,7 @@ class LokiSnodeAPI { } catch (e) { const randomPoolRemainingCount = this.markRandomNodeUnreachable(snode); log.error( - 'loki_snodes: getSnodesForPubkey error', + 'loki_snodes:::getSnodesForPubkey - error', e.code, e.message, `for ${snode.ip}:${ @@ -257,7 +285,7 @@ class LokiSnodeAPI { async getSwarmNodes(pubKey) { const snodes = []; - const questions = [...Array(RANDOM_SNODES_TO_USE).keys()]; + const questions = [...Array(RANDOM_SNODES_TO_USE_FOR_PUBKEY_SWARM).keys()]; await Promise.all( questions.map(async () => { // allow exceptions to pass through upwards