Merge pull request #317 from sachaaaaa/https_p2p

Clearnet p2p

js/background.js

@@ -217,6 +217,11 @@
     }
   }
 
+  function startLocalLokiServer() {
+    const pems = window.getSelfSignedCert();
+    window.localLokiServer = new window.LocalLokiServer(pems);
+  }
+
   // We need this 'first' check because we don't want to start the app up any other time
   //   than the first time. And storage.fetch() will cause onready() to fire.
   let first = true;
@@ -225,6 +230,11 @@
       return;
     }
     first = false;
+
+    if (Whisper.Registration.isDone()) {
+      startLocalLokiServer();
+    }
+
     window.lokiP2pAPI = new window.LokiP2pAPI(
       textsecure.storage.user.getNumber()
     );
@@ -309,6 +319,9 @@
       },
 
       shutdown: async () => {
+
+        await window.localLokiServer.close();
+
         // Stop background processing
         window.Signal.AttachmentDownloads.stop();
         if (idleDetector) {
@@ -535,9 +548,11 @@
     window.log.info('Cleanup: complete');
 
     window.log.info('listening for registration events');
-    Whisper.events.on('registration_done', () => {
+    Whisper.events.on('registration_done', async () => {
       window.log.info('handling registration event');
 
+      startLocalLokiServer();
+
       // listeners
       Whisper.RotateSignedPreKeyListener.init(Whisper.events, newVersion);
       // window.Signal.RefreshSenderCertificate.initialize({

js/modules/loki_snode_api.js

@@ -5,6 +5,7 @@ const is = require('@sindresorhus/is');
 const dns = require('dns');
 const process = require('process');
 const { rpc } = require('./loki_rpc');
+const natUpnp = require('nat-upnp');
 
 const resolve4 = url =>
   new Promise((resolve, reject) => {
@@ -43,6 +44,18 @@ class LokiSnodeAPI {
     }
   }
 
+  async getMyClearIp() {
+    const upnpClient = natUpnp.createClient();
+    return new Promise((resolve, reject) => {
+      upnpClient.externalIp((err, ip) => {
+        if (err) reject(err);
+        else {
+          resolve(ip);
+        }
+      });
+    });
+  }
+
   async getMyLokiIp() {
     try {
       const address = await resolveCname(this.localUrl);

libloki/api.js

@@ -26,10 +26,29 @@
   }
 
   async function sendOnlineBroadcastMessage(pubKey, isPing = false) {
-    const myLokiAddress = await window.lokiSnodeAPI.getMyLokiAddress();
+    let p2pAddress = null;
+    let p2pPort = null;
+    let type;
+
+    if (!window.localLokiServer.isListening()) {
+      // Skip if server is not running AND we're not trying to ping a contact
+      if (!isPing)
+        return;
+
+      type = textsecure.protobuf.LokiAddressMessage.Type.HOST_UNREACHABLE;
+    } else {
+      // clearnet change: getMyLokiAddress -> getMyClearIP
+      // const myLokiAddress = await window.lokiSnodeAPI.getMyLokiAddress();
+      const myIp = await window.lokiSnodeAPI.getMyClearIp();
+      p2pAddress = `https://${myIp}`;
+      p2pPort = window.localLokiServer.getPublicPort();
+      type = textsecure.protobuf.LokiAddressMessage.Type.HOST_REACHABLE;
+    }
+
     const lokiAddressMessage = new textsecure.protobuf.LokiAddressMessage({
-      p2pAddress: `http://${myLokiAddress}`,
+      p2pAddress,
-      p2pPort: parseInt(window.localServerPort, 10),
+      p2pPort,
+      type,
     });
     const content = new textsecure.protobuf.Content({
       lokiAddressMessage,

libloki/modules/local_loki_server.js

@@ -1,5 +1,7 @@
-const http = require('http');
+/* global textsecure */
+const https = require('https');
 const EventEmitter = require('events');
+const natUpnp = require('nat-upnp');
 
 const STATUS = {
   OK: 200,
@@ -14,9 +16,16 @@ class LocalLokiServer extends EventEmitter {
    * Creates an instance of LocalLokiServer.
    * Sends out a `message` event when a new message is received.
    */
-  constructor() {
+  constructor(pems, options = {}) {
     super();
-    this.server = http.createServer((req, res) => {
+    const httpsOptions = {
+      key: pems.private,
+      cert: pems.cert,
+    };
+    if (!options.skipUpnp) {
+      this.upnpClient = natUpnp.createClient();
+    }
+    this.server = https.createServer(httpsOptions, (req, res) => {
       let body = [];
 
       const sendResponse = (statusCode, message = null) => {
@@ -79,18 +88,92 @@ class LocalLokiServer extends EventEmitter {
 
     // Start a listening on new server
     return new Promise((res, rej) => {
-      this.server.listen(port, ip, err => {
+      this.server.listen(port, ip, async (err) => {
         if (err) {
           rej(err);
+        } else if (this.upnpClient) {
+          try {
+            const publicPort = await this.punchHole();
+            res(publicPort);
+          } catch (e) {
+            if (e instanceof textsecure.HolePunchingError) {
+              await this.close();
+            }
+            rej(e);
+          }
         } else {
-          res(this.server.address().port);
+          res(port);
         }
       });
     });
   }
 
+  async punchHole() {
+    const privatePort = this.server.address().port;
+    const portStart = 22100;
+    const portEnd = 22200;
+    const ttl = 60 * 15; // renew upnp every 15 minutes
+    const publicPortsInUse = await new Promise((resolve, reject) => {
+      this.upnpClient.getMappings({ local: true }, (err, results) => {
+        if (err) {
+          // We assume an error here means upnp not enabled
+          reject(new textsecure.HolePunchingError('Could not get mapping from upnp. Upnp not available?', err));
+        }
+        else {
+          // remove the current private port from the current mapping
+          // to allow reusing that port.
+          resolve(results
+            .filter(entry => entry.private.port !== privatePort)
+            .map(entry => entry.public.port));
+        }
+      });
+    });
+
+    for (let publicPort = portStart; publicPort <= portEnd; publicPort += 1) {
+      if (publicPortsInUse.includes(publicPort)) {
+        // eslint-disable-next-line no-continue
+        continue;
+      }
+      const p = new Promise((resolve, reject) => {
+        this.upnpClient.portMapping({
+          public: publicPort,
+          private: privatePort,
+          ttl,
+        }, (err) => {
+          if (err)
+            reject(err);
+          else
+            resolve();
+        });
+      });
+      try {
+        // eslint-disable-next-line no-await-in-loop
+        await p;
+        this.publicPort = publicPort;
+        this.timerHandler = setTimeout(async () => {
+          try {
+            this.publicPort = await this.punchHole();
+          } catch (e) {
+            this.close();
+          }
+        }, ttl * 1000);
+        return publicPort;
+      } catch (e) {
+        throw new textsecure.HolePunchingError('Could not punch hole. Disabled upnp?', e);
+      }
+    }
+    const e = new Error();
+    throw new textsecure.HolePunchingError(`Could not punch hole: no available port. Public ports: ${portStart}-${portEnd}`, e);
+  }
   // Async wrapper for http server close
   close() {
+    clearInterval(this.timerHandler);
+    if (this.upnpClient) {
+      this.upnpClient.portUnmapping({
+        public: this.publicPort,
+      });
+      this.publicPort = null;
+    }
     if (this.server) {
       return new Promise(res => {
         this.server.close(() => res());
@@ -107,6 +190,14 @@ class LocalLokiServer extends EventEmitter {
 
     return null;
   }
+
+  getPublicPort() {
+    return this.publicPort;
+  }
+
+  isListening() {
+    return this.server.listening;
+  }
 }
 
 module.exports = LocalLokiServer;

libloki/test/node/local_loki_server_test.js

@@ -1,20 +1,39 @@
 const axios = require('axios');
 const { assert } = require('chai');
 const LocalLokiServer = require('../../modules/local_loki_server');
+const selfsigned = require('selfsigned');
+const https = require('https');
+
+class HolePunchingError extends Error {
+  constructor(message, err) {
+    super(message);
+    this.name = 'HolePunchingError';
+    this.error = err;
+  }
+}
 
 describe('LocalLokiServer', () => {
   before(async () => {
-    this.server = new LocalLokiServer();
+    const attrs = [{ name: 'commonName', value: 'mypubkey' }];
+    const pems = selfsigned.generate(attrs, { days: 365 * 10 });
+    global.textsecure = {};
+    global.textsecure.HolePunchingError = HolePunchingError;
+    this.server = new LocalLokiServer(pems, { skipUpnp: true });
     await this.server.start(8000);
+    this.axiosClient = axios.create({
+      httpsAgent: new https.Agent({
+        rejectUnauthorized: false,
+      }),
+    });
   });
 
-  after(() => {
+  after(async () => {
-    this.server.close();
+    await this.server.close();
   });
 
   it('should return 405 if not a POST request', async () => {
     try {
-      await axios.get('http://localhost:8000');
+      await this.axiosClient.get('https://localhost:8000');
       assert.fail('Got a successful response');
     } catch (error) {
       if (error.response) {
@@ -27,7 +46,7 @@ describe('LocalLokiServer', () => {
 
   it('should return 404 if no endpoint provided', async () => {
     try {
-      await axios.post('http://localhost:8000', { name: 'Test' });
+      await this.axiosClient.post('https://localhost:8000', { name: 'Test' });
       assert.fail('Got a successful response');
     } catch (error) {
       if (error.response) {
@@ -40,7 +59,7 @@ describe('LocalLokiServer', () => {
 
   it('should return 404 and a string if invalid enpoint is provided', async () => {
     try {
-      await axios.post('http://localhost:8000/invalid', { name: 'Test' });
+      await this.axiosClient.post('https://localhost:8000/invalid', { name: 'Test' });
       assert.fail('Got a successful response');
     } catch (error) {
       if (error.response) {
@@ -54,7 +73,9 @@ describe('LocalLokiServer', () => {
 
   describe('/store', async () => {
     it('should pass the POSTed data to the callback', async () => {
-      const server = new LocalLokiServer();
+      const attrs = [{ name: 'commonName', value: 'mypubkey' }];
+      const pems = selfsigned.generate(attrs, { days: 365 * 10 });
+      const server = new LocalLokiServer(pems, { skipUpnp: true });
       await server.start(8001);
       const messageData = {
         method: 'store',
@@ -74,7 +95,7 @@ describe('LocalLokiServer', () => {
       });
 
       try {
-        await axios.post('http://localhost:8001/storage_rpc/v1', messageData);
+        await this.axiosClient.post('https://localhost:8001/storage_rpc/v1', messageData);
       } catch (error) {
         assert.isNotOk(error, 'Error occured');
       }

libtextsecure/errors.js

@@ -163,6 +163,22 @@
   }
   inherit(ReplayableError, DNSResolutionError);
 
+  function HolePunchingError(message, error) {
+    this.name = 'HolePunchingError';
+    this.message = message;
+    this.error = error;
+
+    Error.call(this, message);
+
+    // Maintains proper stack trace, where our error was thrown (only available on V8)
+    //   via https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this);
+    }
+
+    appendStack(this, error);
+  }
+
   function LokiIpError(message, resolutionError) {
     this.name = 'LokiIpError';
     this.message = message;
@@ -260,6 +276,7 @@
   window.textsecure.SeedNodeError = SeedNodeError;
   window.textsecure.DNSResolutionError = DNSResolutionError;
   window.textsecure.LokiIpError = LokiIpError;
+  window.textsecure.HolePunchingError = HolePunchingError;
   window.textsecure.HTTPError = HTTPError;
   window.textsecure.NotFoundError = NotFoundError;
   window.textsecure.WrongSwarmError = WrongSwarmError;

libtextsecure/message_receiver.js

@@ -117,7 +117,9 @@ MessageReceiver.prototype.extend({
   },
   async startLocalServer() {
     try {
-      const myLokiIp = await window.lokiSnodeAPI.getMyLokiIp();
+      // clearnet change: getMyLokiIp -> getMyClearIp
+      // const myLokiIp = await window.lokiSnodeAPI.getMyLokiIp();
+      const myLokiIp = '0.0.0.0';
       const myServerPort = await localLokiServer.start(
         localServerPort,
         myLokiIp
@@ -125,7 +127,12 @@ MessageReceiver.prototype.extend({
       window.log.info(`Local Server started at ${myLokiIp}:${myServerPort}`);
       libloki.api.broadcastOnlineStatus();
     } catch (e) {
-      if (e instanceof textsecure.LokiIpError) {
+      if (e instanceof textsecure.HolePunchingError) {
+        window.log.warn(e.message);
+        window.log.warn('Abdandoning starting p2p server.');
+        return;
+      }
+      else if (e instanceof textsecure.LokiIpError) {
         window.log.warn(
           'Failed to get my loki address to bind server to, will retry in 30 seconds'
         );
@@ -995,13 +1002,15 @@ MessageReceiver.prototype.extend({
     );
   },
   async handleLokiAddressMessage(envelope, lokiAddressMessage) {
-    const { p2pAddress, p2pPort } = lokiAddressMessage;
+    const { p2pAddress, p2pPort, type } = lokiAddressMessage;
+    if (type === textsecure.protobuf.LokiAddressMessage.Type.HOST_REACHABLE) {
       lokiP2pAPI.updateContactP2pDetails(
         envelope.source,
         p2pAddress,
         p2pPort,
         envelope.isP2p
       );
+    }
     return this.removeFromCache(envelope);
   },
   handleDataMessage(envelope, msg) {

package.json

@@ -32,7 +32,7 @@
     "test-lib-view": "NODE_ENV=test-lib yarn run start",
     "test-loki-view": "NODE_ENV=test-loki yarn run start",
     "test-electron": "yarn grunt test",
-    "test-node": "mocha --recursive test/app test/modules ts/test libloki/test/node",
+    "test-node": "mocha --recursive --exit test/app test/modules ts/test libloki/test/node",
     "test-node-coverage": "nyc --reporter=lcov --reporter=text mocha --recursive test/app test/modules ts/test libloki/test/node",
     "test-node-coverage-html": "nyc --reporter=lcov --reporter=html mocha --recursive test/app test/modules ts/test libloki/test/node",
     "eslint": "eslint .",
@@ -86,6 +86,7 @@
     "mkdirp": "0.5.1",
     "moment": "2.21.0",
     "mustache": "2.3.0",
+    "nat-upnp": "^1.1.1",
     "node-fetch": "2.3.0",
     "node-gyp": "3.8.0",
     "node-sass": "4.9.3",
@@ -105,6 +106,7 @@
     "redux-promise-middleware": "6.1.0",
     "reselect": "4.0.0",
     "rimraf": "2.6.2",
+    "selfsigned": "^1.10.4",
     "semver": "5.4.1",
     "spellchecker": "3.5.1",
     "tar": "4.4.8",

preload.js

@@ -3,6 +3,7 @@
 const path = require('path');
 const electron = require('electron');
 const semver = require('semver');
+const selfsigned = require('selfsigned');
 
 const { deferredToPromise } = require('./js/modules/deferred_to_promise');
 const { JobQueue } = require('./js/modules/job_queue');
@@ -50,6 +51,19 @@ window.isBeforeVersion = (toCheck, baseVersion) => {
 
 // temporary clearnet fix
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+window.getSelfSignedCert = () => {
+  let pems = window.storage.get('self-signed-certificate', null);
+  if (!pems) {
+    const pubKey = window.storage.get('number_id');
+    const attrs = [{ name: 'commonName', value: pubKey }];
+    pems = selfsigned.generate(attrs, { days: 365 * 10 });
+    window.storage.put('self-signed-certificate', pems);
+    window.log.info(`Created PEM for p2p:\n${pems}`);
+  } else {
+    window.log.info(`Found existing PEM for p2p:\n${pems}`);
+  }
+  return pems;
+};
 
 window.wrapDeferred = deferredToPromise;
 
@@ -304,10 +318,9 @@ window.LokiP2pAPI = require('./js/modules/loki_p2p_api');
 
 window.LokiMessageAPI = require('./js/modules/loki_message_api');
 
-const LocalLokiServer = require('./libloki/modules/local_loki_server');
+window.LocalLokiServer = require('./libloki/modules/local_loki_server');
 
 window.localServerPort = config.localServerPort;
-window.localLokiServer = new LocalLokiServer();
 
 window.mnemonic = require('./libloki/modules/mnemonic');
 const WorkerInterface = require('./js/modules/util_worker_interface');
@@ -319,7 +332,7 @@ window.callWorker = (fnName, ...args) => utilWorker.callWorker(fnName, ...args);
 
 // Linux seems to periodically let the event loop stop, so this is a global workaround
 setInterval(() => {
-  window.nodeSetImmediate(() => {});
+  window.nodeSetImmediate(() => { });
 }, 1000);
 
 const { autoOrientImage } = require('./js/modules/auto_orient_image');

protos/SignalService.proto

@@ -39,8 +39,13 @@ message Content {
 }
 
 message LokiAddressMessage {
+  enum Type {
+    HOST_REACHABLE = 0;
+    HOST_UNREACHABLE = 1;
+  }
   optional string p2pAddress    = 1;
   optional uint32 p2pPort       = 2;
+  optional Type   type          = 3;
 }
 
 message PreKeyBundleMessage {

yarn.lock

@@ -754,6 +754,13 @@ async@^2.1.4:
   dependencies:
     lodash "^4.14.0"
 
+async@^2.1.5:
+  version "2.6.2"
+  resolved "https://registry.yarnpkg.com/async/-/async-2.6.2.tgz#18330ea7e6e313887f5d2f2a904bac6fe4dd5381"
+  integrity sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==
+  dependencies:
+    lodash "^4.17.11"
+
 async@~0.9.0:
   version "0.9.2"
   resolved "https://registry.yarnpkg.com/async/-/async-0.9.2.tgz#aea74d5e61c1f899613bf64bda66d4c78f2fd17d"
@@ -4599,7 +4606,7 @@ ip-regex@^1.0.1:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-1.0.3.tgz#dc589076f659f419c222039a33316f1c7387effd"
 
-ip@^1.1.0, ip@^1.1.5:
+ip@^1.1.0, ip@^1.1.4, ip@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
 
@@ -5502,7 +5509,7 @@ lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
 
-lodash@4.17.11, lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.1, lodash@^4.3.0, lodash@^4.8.0:
+lodash@4.17.11, lodash@^4.0.0, lodash@^4.13.1, lodash@^4.14.0, lodash@^4.17.11, lodash@^4.17.4, lodash@^4.2.1, lodash@^4.3.0, lodash@^4.8.0:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
   integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@@ -6092,6 +6099,16 @@ nanomatch@^1.2.9:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
+nat-upnp@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/nat-upnp/-/nat-upnp-1.1.1.tgz#b18365e4faf44652549bb593c69e6b690df22043"
+  integrity sha512-b1Q+sf9fHGCXhlWErNgTTEto8A02MnNysw3vx3kD1657+/Ae23vPEAB6QBh+9RqLL4+xw/LmjVTiLy6A7Cx0xw==
+  dependencies:
+    async "^2.1.5"
+    ip "^1.1.4"
+    request "^2.79.0"
+    xml2js "~0.1.14"
+
 native-or-lie@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/native-or-lie/-/native-or-lie-1.0.2.tgz#c870ee0ba0bf0ff11350595d216cfea68a6d8086"
@@ -6164,6 +6181,11 @@ node-forge@0.7.1:
   version "0.7.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.1.tgz#9da611ea08982f4b94206b3beb4cc9665f20c300"
 
+node-forge@0.7.5:
+  version "0.7.5"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.5.tgz#6c152c345ce11c52f465c2abd957e8639cd674df"
+  integrity sha512-MmbQJ2MTESTjt3Gi/3yG1wGpIMhUfcIypUCGtTizFR9IiccFwxSpfp0vtIZlkFclEqERemxfnSdZEMR9VqqEFQ==
+
 node-gyp@3.8.0, node-gyp@^3.8.0:
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c"
@@ -8413,14 +8435,14 @@ sass-graph@^2.2.4:
     scss-tokenizer "^0.2.3"
     yargs "^7.0.0"
 
+sax@>=0.1.1, sax@^1.2.4, sax@~1.2.1:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
+
 sax@>=0.6.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
 
-sax@^1.2.4, sax@~1.2.1:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
-
 scheduler@^0.13.3:
   version "0.13.3"
   resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.13.3.tgz#bed3c5850f62ea9c716a4d781f9daeb9b2a58896"
@@ -8447,6 +8469,13 @@ select-hose@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
 
+selfsigned@^1.10.4:
+  version "1.10.4"
+  resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.4.tgz#cdd7eccfca4ed7635d47a08bf2d5d3074092e2cd"
+  integrity sha512-9AukTiDmHXGXWtWjembZ5NDmVvP2695EtpgbCsxCa68w3c88B+alqbmZ4O3hZ4VWGXeGWzEVdvqgAJD8DQPCDw==
+  dependencies:
+    node-forge "0.7.5"
+
 selfsigned@^1.9.1:
   version "1.10.2"
   resolved "https://registry.yarnpkg.com/selfsigned/-/selfsigned-1.10.2.tgz#b4449580d99929b65b10a48389301a6592088758"
@@ -10208,6 +10237,13 @@ xml2js@^0.4.5:
     sax ">=0.6.0"
     xmlbuilder "^4.1.0"
 
+xml2js@~0.1.14:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw=
+  dependencies:
+    sax ">=0.1.1"
+
 xmlbuilder@^4.1.0:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-4.2.1.tgz#aa58a3041a066f90eaa16c2f5389ff19f3f461a5"