Merge branch 'clearnet' into height-fix

js/expire.js

@@ -1,4 +1,4 @@
-/* global LokiAppDotNetServerAPI, LokiFileServerAPI, semver, log */
+/* global LokiAppDotNetServerAPI, semver, log */
 // eslint-disable-next-line func-names
 (function() {
   'use strict';
@@ -12,9 +12,8 @@
   );
   // use the anonymous access token
   window.tokenlessFileServerAdnAPI.token = 'loki';
-  window.tokenlessFileServerAdnAPI.pubKey = window.Signal.Crypto.base64ToArrayBuffer(
+  // configure for file server comms
-    LokiFileServerAPI.secureRpcPubKey
+  window.tokenlessFileServerAdnAPI.getPubKeyForUrl();
-  );
 
   let nextWaitSeconds = 5;
   const checkForUpgrades = async () => {

js/modules/loki_app_dot_net_api.js

@@ -7,6 +7,8 @@ const FormData = require('form-data');
 const https = require('https');
 const path = require('path');
 
+const lokiRpcUtils = require('./loki_rpc');
+
 // Can't be less than 1200 if we have unauth'd requests
 const PUBLICCHAT_MSG_POLL_EVERY = 1.5 * 1000; // 1.5s
 const PUBLICCHAT_CHAN_POLL_EVERY = 20 * 1000; // 20s
@@ -14,6 +16,7 @@ const PUBLICCHAT_DELETION_POLL_EVERY = 5 * 1000; // 5s
 const PUBLICCHAT_MOD_POLL_EVERY = 30 * 1000; // 30s
 const PUBLICCHAT_MIN_TIME_BETWEEN_DUPLICATE_MESSAGES = 10 * 1000; // 10s
 
+// FIXME: replace with something on urlPubkeyMap...
 const FILESERVER_HOSTS = [
   'file-dev.lokinet.org',
   'file.lokinet.org',
@@ -21,6 +24,17 @@ const FILESERVER_HOSTS = [
   'file.getsession.org',
 ];
 
+const LOKIFOUNDATION_DEVFILESERVER_PUBKEY =
+  'BSZiMVxOco/b3sYfaeyiMWv/JnqokxGXkHoclEx8TmZ6';
+const LOKIFOUNDATION_FILESERVER_PUBKEY =
+  'BWJQnVm97sQE3Q1InB4Vuo+U/T1hmwHBv0ipkiv8tzEc';
+const urlPubkeyMap = {
+  'https://file-dev.getsession.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY,
+  'https://file-dev.lokinet.org': LOKIFOUNDATION_DEVFILESERVER_PUBKEY,
+  'https://file.getsession.org': LOKIFOUNDATION_FILESERVER_PUBKEY,
+  'https://file.lokinet.org': LOKIFOUNDATION_FILESERVER_PUBKEY,
+};
+
 const HOMESERVER_USER_ANNOTATION_TYPE = 'network.loki.messenger.homeserver';
 const AVATAR_USER_ANNOTATION_TYPE = 'network.loki.messenger.avatar';
 const SETTINGS_CHANNEL_ANNOTATION_TYPE = 'net.patter-app.settings';
@@ -34,30 +48,141 @@ const snodeHttpsAgent = new https.Agent({
 
 const timeoutDelay = ms => new Promise(resolve => setTimeout(resolve, ms));
 
-const sendToProxy = async (
+const sendViaOnion = async (srvPubKey, url, fetchOptions, options = {}) => {
-  srvPubKey,
-  endpoint,
-  pFetchOptions,
-  options = {}
-) => {
   if (!srvPubKey) {
     log.error(
-      'loki_app_dot_net:::sendToProxy - called without a server public key'
+      'loki_app_dot_net:::sendViaOnion - called without a server public key'
     );
     return {};
   }
 
-  const fetchOptions = pFetchOptions; // make lint happy
+  // set retry count
+  if (options.retry === undefined) {
+    // eslint-disable-next-line no-param-reassign
+    options.retry = 0;
+    // eslint-disable-next-line no-param-reassign
+    options.requestNumber = window.lokiSnodeAPI.assignOnionRequestNumber();
+  }
+
+  const payloadObj = {
+    method: fetchOptions.method,
+    body: fetchOptions.body,
     // safety issue with file server, just safer to have this
-  if (fetchOptions.headers === undefined) {
+    headers: fetchOptions.headers || {},
-    fetchOptions.headers = {};
+    // no initial /
+    endpoint: url.pathname.replace(/^\//, ''),
+  };
+  if (url.search) {
+    payloadObj.endpoint += `?${url.search}`;
+  }
+
+  // from https://github.com/sindresorhus/is-stream/blob/master/index.js
+  if (
+    payloadObj.body &&
+    typeof payloadObj.body === 'object' &&
+    typeof payloadObj.body.pipe === 'function'
+  ) {
+    const fData = payloadObj.body.getBuffer();
+    const fHeaders = payloadObj.body.getHeaders();
+    // update headers for boundary
+    payloadObj.headers = { ...payloadObj.headers, ...fHeaders };
+    // update body with base64 chunk
+    payloadObj.body = {
+      fileUpload: fData.toString('base64'),
+    };
+  }
+
+  let pathNodes = [];
+  try {
+    pathNodes = await lokiSnodeAPI.getOnionPath();
+  } catch (e) {
+    log.error(
+      `loki_app_dot_net:::sendViaOnion #${
+        options.requestNumber
+      } - getOnionPath Error ${e.code} ${e.message}`
+    );
+  }
+  if (!pathNodes || !pathNodes.length) {
+    log.warn(
+      `loki_app_dot_net:::sendViaOnion #${
+        options.requestNumber
+      } - failing, no path available`
+    );
+    // should we retry?
+    return {};
+  }
+
+  // do the request
+  let result;
+  try {
+    result = await lokiRpcUtils.sendOnionRequestLsrpcDest(
+      0,
+      pathNodes,
+      srvPubKey,
+      url.host,
+      payloadObj,
+      options.requestNumber
+    );
+  } catch (e) {
+    log.error(
+      'loki_app_dot_net:::sendViaOnion - lokiRpcUtils error',
+      e.code,
+      e.message
+    );
+    return {};
+  }
+
+  // handle error/retries
+  if (!result.status) {
+    log.error(
+      `loki_app_dot_net:::sendViaOnion #${options.requestNumber} - Retry #${
+        options.retry
+      } Couldnt handle onion request, retrying`,
+      payloadObj
+    );
+    return sendViaOnion(srvPubKey, url, fetchOptions, {
+      ...options,
+      retry: options.retry + 1,
+      counter: options.requestNumber,
+    });
+  }
+
+  // get the return variables we need
+  let response = {};
+  let txtResponse = '';
+  let body = '';
+  try {
+    body = JSON.parse(result.body);
+  } catch (e) {
+    log.error(
+      `loki_app_dot_net:::sendViaOnion #${
+        options.requestNumber
+      } - Cant decode JSON body`,
+      result.body
+    );
+  }
+  // result.status has the http response code
+  txtResponse = JSON.stringify(body);
+  response = body;
+  response.headers = result.headers;
+
+  return { result, txtResponse, response };
+};
+
+const sendToProxy = async (srvPubKey, endpoint, fetchOptions, options = {}) => {
+  if (!srvPubKey) {
+    log.error(
+      'loki_app_dot_net:::sendToProxy - called without a server public key'
+    );
+    return {};
   }
 
   const payloadObj = {
     body: fetchOptions.body, // might need to b64 if binary...
     endpoint,
     method: fetchOptions.method,
-    headers: fetchOptions.headers,
+    // safety issue with file server, just safer to have this
+    headers: fetchOptions.headers || {},
   };
 
   // from https://github.com/sindresorhus/is-stream/blob/master/index.js
@@ -87,7 +212,7 @@ const sendToProxy = async (
     log.warn('proxy random snode pool is not ready, retrying 10s', endpoint);
     // no nodes in the pool yet, give it some time and retry
     await timeoutDelay(1000);
-    return sendToProxy(srvPubKey, endpoint, pFetchOptions, options);
+    return sendToProxy(srvPubKey, endpoint, fetchOptions, options);
   }
   const url = `https://${randSnode.ip}:${randSnode.port}/file_proxy`;
 
@@ -98,7 +223,10 @@ const sendToProxy = async (
   payloadObj.body = false; // free memory
 
   // make temporary key for this request/response
-  const ephemeralKey = await libsignal.Curve.async.generateKeyPair();
+  // async maybe preferable to avoid cpu spikes
+  // tho I think sync might be more apt in certain cases here...
+  // like sending
+  const ephemeralKey = await libloki.crypto.generateEphemeralKeyPair();
 
   // mix server pub key with our priv key
   const symKey = await libsignal.Curve.async.calculateAgreement(
@@ -257,6 +385,21 @@ const serverRequest = async (endpoint, options = {}) => {
     const host = url.host.toLowerCase();
     // log.info('host', host, FILESERVER_HOSTS);
     if (
+      window.lokiFeatureFlags.useFileOnionRequests &&
+      FILESERVER_HOSTS.includes(host)
+    ) {
+      mode = 'sendViaOnion';
+      // url.search automatically includes the ? part
+      // const search = url.search || '';
+      // strip first slash
+      // const endpointWithQS = `${url.pathname}${search}`.replace(/^\//, '');
+      ({ response, txtResponse, result } = await sendViaOnion(
+        srvPubKey,
+        url,
+        fetchOptions,
+        options
+      ));
+    } else if (
       window.lokiFeatureFlags.useSnodeProxy &&
       FILESERVER_HOSTS.includes(host)
     ) {
@@ -317,6 +460,14 @@ const serverRequest = async (endpoint, options = {}) => {
       err: e,
     };
   }
+
+  if (!result) {
+    return {
+      err: 'noResult',
+      response,
+    };
+  }
+
   // if it's a response style with a meta
   if (result.status !== 200) {
     if (!forceFreshToken && (!response.meta || response.meta.code === 401)) {
@@ -418,6 +569,66 @@ class LokiAppDotNetServerAPI {
     this.channels.splice(i, 1);
   }
 
+  // set up pubKey & pubKeyHex properties
+  // optionally called for mainly file server comms
+  getPubKeyForUrl() {
+    // Hard coded
+    let pubKeyAB;
+    if (urlPubkeyMap) {
+      pubKeyAB = window.Signal.Crypto.base64ToArrayBuffer(
+        urlPubkeyMap[this.baseServerUrl]
+      );
+    }
+    // else will fail validation later
+
+    // if in proxy mode, don't allow "file-dev."...
+    // it only supports "file."... host.
+    if (
+      window.lokiFeatureFlags.useSnodeProxy &&
+      !window.lokiFeatureFlags.useOnionRequests
+    ) {
+      pubKeyAB = window.Signal.Crypto.base64ToArrayBuffer(
+        LOKIFOUNDATION_FILESERVER_PUBKEY
+      );
+    }
+
+    // do we have their pubkey locally?
+    // FIXME: this._server won't be set yet...
+    // can't really do this for the file server because we'll need the key
+    // before we can communicate with lsrpc
+    /*
+    // get remote pubKey
+    this._server.serverRequest('loki/v1/public_key').then(keyRes => {
+      // we don't need to delay to protect identity because the token request
+      // should only be done over lokinet-lite
+      this.delayToken = true;
+      if (keyRes.err || !keyRes.response || !keyRes.response.data) {
+        if (keyRes.err) {
+          log.error(`Error ${keyRes.err}`);
+        }
+      } else {
+        // store it
+        this.pubKey = dcodeIO.ByteBuffer.wrap(
+          keyRes.response.data,
+          'base64'
+        ).toArrayBuffer();
+        // write it to a file
+      }
+    });
+    */
+
+    // now that key is loaded, lets verify
+    if (pubKeyAB && pubKeyAB.byteLength && pubKeyAB.byteLength !== 33) {
+      log.error('FILESERVER PUBKEY is invalid, length:', pubKeyAB.byteLength);
+      process.exit(1);
+    }
+
+    this.pubKey = pubKeyAB;
+    this.pubKeyHex = StringView.arrayBufferToHex(pubKeyAB);
+
+    return pubKeyAB;
+  }
+
   async setProfileName(profileName) {
     // when we add an annotation, may need this
     /*

js/modules/loki_file_server_api.js

@@ -8,49 +8,11 @@ const LokiAppDotNetAPI = require('./loki_app_dot_net_api');
 const DEVICE_MAPPING_USER_ANNOTATION_TYPE =
   'network.loki.messenger.devicemapping';
 
-// const LOKIFOUNDATION_DEVFILESERVER_PUBKEY =
-//  'BSZiMVxOco/b3sYfaeyiMWv/JnqokxGXkHoclEx8TmZ6';
-const LOKIFOUNDATION_FILESERVER_PUBKEY =
-  'BWJQnVm97sQE3Q1InB4Vuo+U/T1hmwHBv0ipkiv8tzEc';
-
 // can have multiple of these instances as each user can have a
 // different home server
 class LokiFileServerInstance {
   constructor(ourKey) {
     this.ourKey = ourKey;
-
-    // do we have their pubkey locally?
-    /*
-    // get remote pubKey
-    this._server.serverRequest('loki/v1/public_key').then(keyRes => {
-      // we don't need to delay to protect identity because the token request
-      // should only be done over lokinet-lite
-      this.delayToken = true;
-      if (keyRes.err || !keyRes.response || !keyRes.response.data) {
-        if (keyRes.err) {
-          log.error(`Error ${keyRes.err}`);
-        }
-      } else {
-        // store it
-        this.pubKey = dcodeIO.ByteBuffer.wrap(
-          keyRes.response.data,
-          'base64'
-        ).toArrayBuffer();
-        // write it to a file
-      }
-    });
-    */
-    // Hard coded
-    this.pubKey = window.Signal.Crypto.base64ToArrayBuffer(
-      LOKIFOUNDATION_FILESERVER_PUBKEY
-    );
-    if (this.pubKey.byteLength && this.pubKey.byteLength !== 33) {
-      log.error(
-        'FILESERVER PUBKEY is invalid, length:',
-        this.pubKey.byteLength
-      );
-      process.exit(1);
-    }
   }
 
   // FIXME: this is not file-server specific
@@ -65,9 +27,8 @@ class LokiFileServerInstance {
     } else {
       this._server = new LokiAppDotNetAPI(this.ourKey, serverUrl);
     }
-
+    // make sure pubKey & pubKeyHex are set in _server
-    // configure proxy
+    this.pubKey = this._server.getPubKeyForUrl();
-    this._server.pubKey = this.pubKey;
 
     if (options !== undefined && options.skipToken) {
       return;
@@ -80,6 +41,7 @@ class LokiFileServerInstance {
       log.error('You are blacklisted form this home server');
     }
   }
+
   async getUserDeviceMapping(pubKey) {
     const annotations = await this._server.getUserAnnotations(pubKey);
     const deviceMapping = annotations.find(
@@ -333,7 +295,5 @@ class LokiFileServerFactoryAPI {
     return thisServer;
   }
 }
-// smuggle some data out of this joint (for expire.js/version upgrade check)
-LokiFileServerFactoryAPI.secureRpcPubKey = LOKIFOUNDATION_FILESERVER_PUBKEY;
 
 module.exports = LokiFileServerFactoryAPI;

js/modules/loki_rpc.js

@@ -11,88 +11,257 @@ const snodeHttpsAgent = new https.Agent({
 
 const endpointBase = '/storage_rpc/v1';
 
-// Request index for debugging
-let onionReqIdx = 0;
-
-const encryptForNode = async (node, payloadStr) => {
-  const textEncoder = new TextEncoder();
-  const plaintext = textEncoder.encode(payloadStr);
-
-  return libloki.crypto.encryptForPubkey(node.pubkey_x25519, plaintext);
-};
-
 // Returns the actual ciphertext, symmetric key that will be used
 // for decryption, and an ephemeral_key to send to the next hop
-const encryptForDestination = async (node, payload) => {
+const encryptForPubKey = async (pubKeyX25519hex, reqObj) => {
-  // Do we still need "headers"?
+  const reqStr = JSON.stringify(reqObj);
-  const reqStr = JSON.stringify({ body: payload, headers: '' });
+
+  const textEncoder = new TextEncoder();
+  const plaintext = textEncoder.encode(reqStr);
 
-  return encryptForNode(node, reqStr);
+  return libloki.crypto.encryptForPubkey(pubKeyX25519hex, plaintext);
 };
 
 // `ctx` holds info used by `node` to relay further
-const encryptForRelay = async (node, nextNode, ctx) => {
+const encryptForRelay = async (relayX25519hex, destination, ctx) => {
+  // ctx contains: ciphertext, symmetricKey, ephemeralKey
   const payload = ctx.ciphertext;
 
-  const reqJson = {
+  if (!destination.host && !destination.destination) {
+    log.warn(`loki_rpc::encryptForRelay - no destination`, destination);
+  }
+
+  const reqObj = {
+    ...destination,
     ciphertext: dcodeIO.ByteBuffer.wrap(payload).toString('base64'),
     ephemeral_key: StringView.arrayBufferToHex(ctx.ephemeralKey),
-    destination: nextNode.pubkey_ed25519,
   };
 
-  const reqStr = JSON.stringify(reqJson);
+  return encryptForPubKey(relayX25519hex, reqObj);
-
-  return encryptForNode(node, reqStr);
 };
 
-const BAD_PATH = 'bad_path';
+const makeGuardPayload = guardCtx => {
+  const ciphertextBase64 = dcodeIO.ByteBuffer.wrap(
+    guardCtx.ciphertext
+  ).toString('base64');
+
+  const guardPayloadObj = {
+    ciphertext: ciphertextBase64,
+    ephemeral_key: StringView.arrayBufferToHex(guardCtx.ephemeralKey),
+  };
+  return guardPayloadObj;
+};
 
-// May return false BAD_PATH, indicating that we should try a new
+// we just need the targetNode.pubkey_ed25519 for the encryption
-const sendOnionRequest = async (reqIdx, nodePath, targetNode, plaintext) => {
+// targetPubKey is ed25519 if snode is the target
-  const ctxes = [await encryptForDestination(targetNode, plaintext)];
+const makeOnionRequest = async (
+  nodePath,
+  destCtx,
+  targetED25519Hex,
+  finalRelayOptions = false,
+  id = ''
+) => {
+  const ctxes = [destCtx];
   // from (3) 2 to 0
   const firstPos = nodePath.length - 1;
 
   for (let i = firstPos; i > -1; i -= 1) {
-    // this nodePath points to the previous (i + 1) context
+    let dest;
-    ctxes.push(
+    const relayingToFinalDestination = i === 0; // if last position
+
+    if (relayingToFinalDestination && finalRelayOptions) {
+      dest = {
+        host: finalRelayOptions.host,
+        target: '/loki/v1/lsrpc',
+        method: 'POST',
+      };
+    } else {
+      // set x25519 if destination snode
+      let pubkeyHex = targetED25519Hex; // relayingToFinalDestination
+      // or ed25519 snode destination
+      if (!relayingToFinalDestination) {
+        pubkeyHex = nodePath[i + 1].pubkey_ed25519;
+        if (!pubkeyHex) {
+          log.error(
+            `loki_rpc:::makeOnionRequest ${id} - no ed25519 for`,
+            nodePath[i + 1],
+            'path node',
+            i + 1
+          );
+        }
+      }
+      // destination takes a hex key
+      dest = {
+        destination: pubkeyHex,
+      };
+    }
+    try {
       // eslint-disable-next-line no-await-in-loop
-      await encryptForRelay(
+      const ctx = await encryptForRelay(
-        nodePath[i],
+        nodePath[i].pubkey_x25519,
-        i === firstPos ? targetNode : nodePath[i + 1],
+        dest,
         ctxes[ctxes.length - 1]
-      )
       );
+      ctxes.push(ctx);
+    } catch (e) {
+      log.error(
+        `loki_rpc:::makeOnionRequest ${id} - encryptForRelay failure`,
+        e.code,
+        e.message
+      );
+      throw e;
+    }
   }
   const guardCtx = ctxes[ctxes.length - 1]; // last ctx
 
-  const ciphertextBase64 = dcodeIO.ByteBuffer.wrap(
+  const payloadObj = makeGuardPayload(guardCtx);
-    guardCtx.ciphertext
-  ).toString('base64');
 
-  const payload = {
+  // all these requests should use AesGcm
-    ciphertext: ciphertextBase64,
+  return payloadObj;
-    ephemeral_key: StringView.arrayBufferToHex(guardCtx.ephemeralKey),
+};
-  };
 
-  const fetchOptions = {
+// finalDestOptions is an object
+// FIXME: internally track reqIdx, not externally
+const sendOnionRequest = async (
+  reqIdx,
+  nodePath,
+  destX25519Any,
+  finalDestOptions,
+  finalRelayOptions = false,
+  lsrpcIdx
+) => {
+  if (!destX25519Any) {
+    log.error('loki_rpc::sendOnionRequest - no destX25519Any given');
+    return {};
+  }
+
+  // loki-storage may need this to function correctly
+  // but ADN calls will not always have a body
+  /*
+  if (!finalDestOptions.body) {
+    finalDestOptions.body = '';
+  }
+  */
+
+  let id = '';
+  if (lsrpcIdx !== undefined) {
+    id += `${lsrpcIdx}=>`;
+  }
+  if (reqIdx !== undefined) {
+    id += `${reqIdx}`;
+  }
+
+  // get destination pubkey in array buffer format
+  let destX25519hex = destX25519Any;
+  if (typeof destX25519hex !== 'string') {
+    // convert AB to hex
+    destX25519hex = StringView.arrayBufferToHex(destX25519Any);
+  }
+
+  // safely build destination
+  let targetEd25519hex;
+  if (finalDestOptions) {
+    if (finalDestOptions.destination_ed25519_hex) {
+      // snode destination
+      targetEd25519hex = finalDestOptions.destination_ed25519_hex;
+      // eslint-disable-next-line no-param-reassign
+      delete finalDestOptions.destination_ed25519_hex;
+    }
+    // else it's lsrpc...
+  } else {
+    // eslint-disable-next-line no-param-reassign
+    finalDestOptions = {};
+    log.warn(`loki_rpc::sendOnionRequest ${id} - no finalDestOptions`);
+    return {};
+  }
+
+  const options = finalDestOptions; // lint
+  // do we need this?
+  if (options.headers === undefined) {
+    options.headers = '';
+  }
+
+  let destCtx;
+  try {
+    destCtx = await encryptForPubKey(destX25519hex, options);
+  } catch (e) {
+    log.error(
+      `loki_rpc::sendOnionRequest ${id} - encryptForPubKey failure [`,
+      e.code,
+      e.message,
+      '] destination X25519',
+      destX25519hex.substr(0, 32),
+      '...',
+      destX25519hex.substr(32),
+      'options',
+      options
+    );
+    throw e;
+  }
+
+  const payloadObj = await makeOnionRequest(
+    nodePath,
+    destCtx,
+    targetEd25519hex,
+    finalRelayOptions,
+    id
+  );
+
+  const guardFetchOptions = {
     method: 'POST',
-    body: JSON.stringify(payload),
+    body: JSON.stringify(payloadObj),
+    // we are talking to a snode...
+    agent: snodeHttpsAgent,
   };
 
-  const url = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`;
+  const guardUrl = `https://${nodePath[0].ip}:${nodePath[0].port}/onion_req`;
-
+  const response = await nodeFetch(guardUrl, guardFetchOptions);
-  // we only proxy to snodes...
-  process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
-  const response = await nodeFetch(url, fetchOptions);
-  process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1';
 
-  return processOnionResponse(reqIdx, response, ctxes[0].symmetricKey, true);
+  return processOnionResponse(reqIdx, response, destCtx.symmetricKey, true);
 };
 
+const sendOnionRequestSnodeDest = async (
+  reqIdx,
+  nodePath,
+  targetNode,
+  plaintext
+) =>
+  sendOnionRequest(reqIdx, nodePath, targetNode.pubkey_x25519, {
+    destination_ed25519_hex: targetNode.pubkey_ed25519,
+    body: plaintext,
+  });
+
+// need relay node's pubkey_x25519_hex
+// always the same target: /loki/v1/lsrpc
+const sendOnionRequestLsrpcDest = async (
+  reqIdx,
+  nodePath,
+  destX25519Any,
+  host,
+  payloadObj,
+  lsrpcIdx = 0
+) =>
+  sendOnionRequest(
+    reqIdx,
+    nodePath,
+    destX25519Any,
+    payloadObj,
+    { host },
+    lsrpcIdx
+  );
+
+const BAD_PATH = 'bad_path';
+
 // Process a response as it arrives from `nodeFetch`, handling
 // http errors and attempting to decrypt the body with `sharedKey`
-const processOnionResponse = async (reqIdx, response, sharedKey, useAesGcm) => {
+// May return false BAD_PATH, indicating that we should try a new path.
+const processOnionResponse = async (
+  reqIdx,
+  response,
+  sharedKey,
+  useAesGcm,
+  debug
+) => {
   // FIXME: 401/500 handling?
 
   // detect SNode is not ready (not in swarm; not done syncing)
@@ -115,16 +284,26 @@ const processOnionResponse = async (reqIdx, response, sharedKey, useAesGcm) => {
 
   if (response.status !== 200) {
     log.warn(
-      `(${reqIdx}) [path] fetch unhandled error code: ${response.status}`
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - fetch unhandled error code: ${
+        response.status
+      }`
     );
     return false;
   }
 
   const ciphertext = await response.text();
   if (!ciphertext) {
-    log.warn(`(${reqIdx}) [path]: Target node return empty ciphertext`);
+    log.warn(
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - Target node return empty ciphertext`
+    );
     return false;
   }
+  if (debug) {
+    log.debug(
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - ciphertext`,
+      ciphertext
+    );
+  }
 
   let plaintext;
   let ciphertextBuffer;
@@ -134,22 +313,52 @@ const processOnionResponse = async (reqIdx, response, sharedKey, useAesGcm) => {
       'base64'
     ).toArrayBuffer();
 
+    if (debug) {
+      log.debug(
+        `(${reqIdx}) [path] lokiRpc::processOnionResponse - ciphertextBuffer`,
+        StringView.arrayBufferToHex(ciphertextBuffer),
+        'useAesGcm',
+        useAesGcm
+      );
+    }
+
     const decryptFn = useAesGcm
-      ? window.libloki.crypto.DecryptGCM
+      ? libloki.crypto.DecryptGCM
-      : window.libloki.crypto.DHDecrypt;
+      : libloki.crypto.DHDecrypt;
 
-    const plaintextBuffer = await decryptFn(sharedKey, ciphertextBuffer);
+    const plaintextBuffer = await decryptFn(sharedKey, ciphertextBuffer, debug);
+    if (debug) {
+      log.debug(
+        'lokiRpc::processOnionResponse - plaintextBuffer',
+        plaintextBuffer.toString()
+      );
+    }
 
     const textDecoder = new TextDecoder();
     plaintext = textDecoder.decode(plaintextBuffer);
   } catch (e) {
-    log.error(`(${reqIdx}) [path] decode error`);
+    log.error(
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - decode error`,
+      e.code,
+      e.message
+    );
+    log.error(
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - symKey`,
+      StringView.arrayBufferToHex(sharedKey)
+    );
     if (ciphertextBuffer) {
-      log.error(`(${reqIdx}) [path] ciphertextBuffer`, ciphertextBuffer);
+      log.error(
+        `(${reqIdx}) [path] lokiRpc::processOnionResponse - ciphertextBuffer`,
+        StringView.arrayBufferToHex(ciphertextBuffer)
+      );
     }
     return false;
   }
 
+  if (debug) {
+    log.debug('lokiRpc::processOnionResponse - plaintext', plaintext);
+  }
+
   try {
     const jsonRes = JSON.parse(plaintext);
     // emulate nodeFetch response...
@@ -158,13 +367,22 @@ const processOnionResponse = async (reqIdx, response, sharedKey, useAesGcm) => {
         const res = JSON.parse(jsonRes.body);
         return res;
       } catch (e) {
-        log.error(`(${reqIdx}) [path] parse error json: `, jsonRes.body);
+        log.error(
+          `(${reqIdx}) [path] lokiRpc::processOnionResponse - parse error inner json: `,
+          jsonRes.body
+        );
       }
       return false;
     };
     return jsonRes;
   } catch (e) {
-    log.error('[path] parse error', e.code, e.message, `json:`, plaintext);
+    log.error(
+      `(${reqIdx}) [path] lokiRpc::processOnionResponse - parse error outer json`,
+      e.code,
+      e.message,
+      `json:`,
+      plaintext
+    );
     return false;
   }
 };
@@ -206,7 +424,7 @@ const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => {
 
   const snPubkeyHex = StringView.hexToArrayBuffer(targetNode.pubkey_x25519);
 
-  const myKeys = await window.libloki.crypto.generateEphemeralKeyPair();
+  const myKeys = await libloki.crypto.generateEphemeralKeyPair();
 
   const symmetricKey = await libsignal.Curve.async.calculateAgreement(
     snPubkeyHex,
@@ -217,7 +435,7 @@ const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => {
   const body = JSON.stringify(options);
 
   const plainText = textEncoder.encode(body);
-  const ivAndCiphertext = await window.libloki.crypto.DHEncrypt(
+  const ivAndCiphertext = await libloki.crypto.DHEncrypt(
     symmetricKey,
     plainText
   );
@@ -279,6 +497,7 @@ const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => {
     // grab a fresh random one
     return sendToProxy(options, targetNode, pRetryNumber);
   }
+  // 502 is "Next node not found"
 
   // detect SNode is not ready (not in swarm; not done syncing)
   // 503 can be proxy target or destination in pre 2.0.3
@@ -364,7 +583,7 @@ const sendToProxy = async (options = {}, targetNode, retryNumber = 0) => {
       'base64'
     ).toArrayBuffer();
 
-    const plaintextBuffer = await window.libloki.crypto.DHDecrypt(
+    const plaintextBuffer = await libloki.crypto.DHDecrypt(
       symmetricKey,
       ciphertextBuffer
     );
@@ -460,6 +679,7 @@ const lokiFetch = async (url, options = {}, targetNode = null) => {
     // Wrong PoW difficulty
     if (response.status === 432) {
       const result = await response.json();
+      log.error(`lokirpc:::lokiFetch ${type} - WRONG POW`, result);
       throw new textsecure.WrongDifficultyError(result.difficulty);
     }
 
@@ -480,11 +700,10 @@ const lokiFetch = async (url, options = {}, targetNode = null) => {
         // Get a path excluding `targetNode`:
         // eslint-disable-next-line no-await-in-loop
         const path = await lokiSnodeAPI.getOnionPath(targetNode);
-        const thisIdx = onionReqIdx;
+        const thisIdx = window.lokiSnodeAPI.assignOnionRequestNumber();
-        onionReqIdx += 1;
 
         // eslint-disable-next-line no-await-in-loop
-        const result = await sendOnionRequest(
+        const result = await sendOnionRequestSnodeDest(
           thisIdx,
           path,
           targetNode,
@@ -640,4 +859,5 @@ const lokiRpc = (
 
 module.exports = {
   lokiRpc,
+  sendOnionRequestLsrpcDest,
 };

js/modules/loki_snode_api.js

@@ -23,8 +23,17 @@ const compareSnodes = (current, search) =>
 
 // just get the filtered list
 async function tryGetSnodeListFromLokidSeednode(
-  seedNodes = [...window.seedNodeList]
+  seedNodes = window.seedNodeList
 ) {
+  if (!seedNodes.length) {
+    log.error(
+      `loki_snodes:::tryGetSnodeListFromLokidSeednode - no seedNodes given`,
+      seedNodes,
+      'window',
+      window.seedNodeList
+    );
+    return [];
+  }
   // Removed limit until there is a way to get snode info
   // for individual nodes (needed for guard nodes);  this way
   // we get all active nodes
@@ -42,6 +51,13 @@ async function tryGetSnodeListFromLokidSeednode(
     Math.floor(Math.random() * seedNodes.length),
     1
   )[0];
+  if (!seedNode) {
+    log.error(
+      `loki_snodes:::tryGetSnodeListFromLokidSeednode - seedNode selection failure - seedNodes`,
+      seedNodes
+    );
+    return [];
+  }
   let snodes = [];
   try {
     const getSnodesFromSeedUrl = async urlObj => {
@@ -53,6 +69,30 @@ async function tryGetSnodeListFromLokidSeednode(
         {}, // Options
         '/json_rpc' // Seed request endpoint
       );
+      if (!response) {
+        log.error(
+          `loki_snodes:::tryGetSnodeListFromLokidSeednode - invalid response from seed ${urlObj.toString()}:`,
+          response
+        );
+        return [];
+      }
+
+      // should we try to JSON.parse this?
+      if (typeof response === 'string') {
+        log.error(
+          `loki_snodes:::tryGetSnodeListFromLokidSeednode - invalid string response from seed ${urlObj.toString()}:`,
+          response
+        );
+        return [];
+      }
+
+      if (!response.result) {
+        log.error(
+          `loki_snodes:::tryGetSnodeListFromLokidSeednode - invalid result from seed ${urlObj.toString()}:`,
+          response
+        );
+        return [];
+      }
       // Filter 0.0.0.0 nodes which haven't submitted uptime proofs
       return response.result.service_node_states.filter(
         snode => snode.public_ip !== '0.0.0.0'
@@ -72,6 +112,13 @@ async function tryGetSnodeListFromLokidSeednode(
         );
       }
     }
+    if (snodes.length) {
+      log.info(
+        `loki_snodes:::tryGetSnodeListFromLokidSeednode - got ${
+          snodes.length
+        } service nodes from seed`
+      );
+    }
     return snodes;
   } catch (e) {
     log.warn(
@@ -87,9 +134,18 @@ async function tryGetSnodeListFromLokidSeednode(
 }
 
 async function getSnodeListFromLokidSeednode(
-  seedNodes = [...window.seedNodeList],
+  seedNodes = window.seedNodeList,
   retries = 0
 ) {
+  if (!seedNodes.length) {
+    log.error(
+      `loki_snodes:::getSnodeListFromLokidSeednode - no seedNodes given`,
+      seedNodes,
+      'window',
+      window.seedNodeList
+    );
+    return [];
+  }
   let snodes = [];
   try {
     snodes = await tryGetSnodeListFromLokidSeednode(seedNodes);
@@ -129,6 +185,12 @@ class LokiSnodeAPI {
 
     this.onionPaths = [];
     this.guardNodes = [];
+    this.onionRequestCounter = 0; // Request index for debugging
+  }
+
+  assignOnionRequestNumber() {
+    this.onionRequestCounter += 1;
+    return this.onionRequestCounter;
   }
 
   async getRandomSnodePool() {
@@ -202,7 +264,7 @@ class LokiSnodeAPI {
     // FIXME: handle rejections
     let nodePool = await this.getRandomSnodePool();
     if (nodePool.length === 0) {
-      log.error(`Could not select guarn nodes: node pool is empty`);
+      log.error(`Could not select guard nodes: node pool is empty`);
       return [];
     }
 
@@ -213,7 +275,7 @@ class LokiSnodeAPI {
     const DESIRED_GUARD_COUNT = 3;
     if (shuffled.length < DESIRED_GUARD_COUNT) {
       log.error(
-        `Could not select guarn nodes: node pool is not big enough, pool size ${
+        `Could not select guard nodes: node pool is not big enough, pool size ${
           shuffled.length
         }, need ${DESIRED_GUARD_COUNT}, attempting to refresh randomPool`
       );
@@ -222,7 +284,7 @@ class LokiSnodeAPI {
       shuffled = _.shuffle(nodePool);
       if (shuffled.length < DESIRED_GUARD_COUNT) {
         log.error(
-          `Could not select guarn nodes: node pool is not big enough, pool size ${
+          `Could not select guard nodes: node pool is not big enough, pool size ${
             shuffled.length
           }, need ${DESIRED_GUARD_COUNT}, failing...`
         );
@@ -278,12 +340,15 @@ class LokiSnodeAPI {
         `Must have at least 2 good onion paths, actual: ${goodPaths.length}`
       );
       await this.buildNewOnionPaths();
+      // should we add a delay? buildNewOnionPaths should act as one
+      // reload goodPaths now
+      return this.getOnionPath(toExclude);
     }
 
     const paths = _.shuffle(goodPaths);
 
     if (!toExclude) {
-      return paths[0];
+      return paths[0].path;
     }
 
     // Select a path that doesn't contain `toExclude`
@@ -294,6 +359,19 @@ class LokiSnodeAPI {
 
     if (otherPaths.length === 0) {
       // This should never happen!
+      // well it did happen, should we
+      // await this.buildNewOnionPaths();
+      // and restart call?
+      log.error(
+        `loki_snode_api::getOnionPath - no paths without`,
+        toExclude.pubkey_ed25519,
+        'path count',
+        paths.length,
+        'goodPath count',
+        goodPaths.length,
+        'paths',
+        paths
+      );
       throw new Error('No onion paths available after filtering');
     }
 
@@ -569,7 +647,17 @@ class LokiSnodeAPI {
     );
   }
 
-  async refreshRandomPool(seedNodes = [...window.seedNodeList]) {
+  async refreshRandomPool(seedNodes = window.seedNodeList) {
+    if (!seedNodes.length) {
+      if (!window.seedNodeList || !window.seedNodeList.length) {
+        log.error(
+          `loki_snodes:::refreshRandomPool - seedNodeList has not been loaded yet`
+        );
+        return [];
+      }
+      // eslint-disable-next-line no-param-reassign
+      seedNodes = window.seedNodeList;
+    }
     return primitives.allowOnlyOneAtATime('refreshRandomPool', async () => {
       // are we running any _getAllVerionsForRandomSnodePool
       if (this.stopGetAllVersionPromiseControl !== false) {

preload.js

@@ -416,6 +416,7 @@ window.lokiFeatureFlags = {
   privateGroupChats: true,
   useSnodeProxy: !process.env.USE_STUBBED_NETWORK,
   useOnionRequests: true,
+  useFileOnionRequests: false,
   onionRequestHops: 1,
 };
 

test/index.html

@@ -492,13 +492,14 @@
   <script type="text/javascript" src="test.js"></script>
 
   <script type="text/javascript" src="../js/registration.js" data-cover></script>
-  <script type="text/javascript" src="../js/expire.js" data-cover></script>
   <script type="text/javascript" src="../js/chromium.js" data-cover></script>
   <script type="text/javascript" src="../js/database.js" data-cover></script>
   <script type="text/javascript" src="../js/storage.js" data-cover></script>
   <script type="text/javascript" src="../js/signal_protocol_store.js" data-cover></script>
   <script type="text/javascript" src="../js/libtextsecure.js" data-cover></script>
   <script type="text/javascript" src="../js/libloki.js" data-cover></script>
+  <!-- needs the network comms libraries to work -->
+  <script type="text/javascript" src="../js/expire.js" data-cover></script>
 
   <script type="text/javascript" src="../js/models/messages.js" data-cover></script>
   <script type="text/javascript" src="../js/models/conversations.js" data-cover></script>