Incorporate geobuf for passing geojson

According to mapbox/mapbox-gl-js#1504 the transferring of GeoJSON files between main and worker threads can cause a bottleneck. We should be able to reduce this bottleneck by converting and passing the GeoJSON file created in the Worker as a geobuf and decoding it on the main thread. This commit is broken because of a possible bug with Webworkify not pulling in correct dependencies. See browserify/webworkify#14
nickpeihl · Feb 19, 2016 · 1482953 · 1482953
1 parent 036f79f
commit 1482953
Show file tree

Hide file tree

Showing 4 changed files with 115 additions and 96 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 /node_modules/
+/.tern-port
diff --git a/main.js b/main.js
@@ -2,123 +2,127 @@ var work = require('webworkify');
 var mapboxgl = require('mapbox-gl');
 var concave = require('turf-concave');
 var geojsonhint = require('geojsonhint');
+var geobuf = require('geobuf');
+var Pbf = require('pbf');
 
 var w = work(require('./worker.js'));
 w.addEventListener('message', function(ev) {
-  console.log(ev.data[0]);
-  turfResult.setData(ev.data[1]);
+    console.log(ev.data);
+    var gj = geobuf.decode(ev.data[1]);
+    console.log(gj);
+    turfResult.setData(gj);
 
 });
 w.addEventListener('error', function(err) {
-  console.log(err);
+    console.log(err);
 });
 
 mapboxgl.accessToken = 'pk.eyJ1IjoibnBlaWhsIiwiYSI6InVmU21qeVUifQ.jwa9V6XsmccKsEHKh5QfmQ';
 
 var map = new mapboxgl.Map({
-  container: 'map',
-  style: 'mapbox://styles/mapbox/streets-v8',
-  center: [-122.963104,48.569792],
-  zoom: 10
+    container: 'map',
+    style: 'mapbox://styles/mapbox/streets-v8',
+    center: [-122.963104,48.569792],
+    zoom: 10
 });
 
 var dataUrl = 'assets/hydrants.geojson';
 
 var hydrants = new mapboxgl.GeoJSONSource({
-  data: dataUrl
+    data: dataUrl
 });
 
 var turfResult = new mapboxgl.GeoJSONSource({});
 
 map.on('style.load', function() {
-  console.log('style loaded');
-  map.addSource('points', hydrants);
-
-  map.addSource('results', turfResult);
-
-  map.addLayer({
-    'id': 'hydrants',
-    'type': 'circle',
-    'source': 'points'
-  });
-
-  map.addLayer({
-    'id': 'results',
-    'type': 'fill',
-    'source': 'results',
-    'paint': {
-      'fill-opacity': 0.6,
-      'fill-color': '#00FF00'
-    }
-  });
+    console.log('style loaded');
+    map.addSource('points', hydrants);
+
+    map.addSource('results', turfResult);
+
+    map.addLayer({
+        'id': 'hydrants',
+        'type': 'circle',
+        'source': 'points'
+    });
+
+    map.addLayer({
+        'id': 'results',
+        'type': 'fill',
+        'source': 'results',
+        'paint': {
+            'fill-opacity': 0.6,
+            'fill-color': '#00FF00'
+        }
+    });
 });
 
 map.on('source.load', function() {
-  console.log('source loaded');
+    console.log('source loaded');
 });
 
 addButton('Turf Sync', 'turf-sync');
 addButton('Turf Async','turf-async');
 
 function addButton(name, id) {
-  var link = document.createElement('a');
-  link.href = '#';
-  link.id = id;
-  link.className = 'active';
-  link.textContent = name;
-
-  link.onclick = function(e) {
-    e.preventDefault();
-    e.stopPropagation();
-
-    switch (id) {
-    case 'turf-sync':
-      turfSync();
-      break;
-    case 'turf-async':
-      turfAsync();
-      break;
-    }
-  };
-  var menu = document.getElementById('menu');
-  menu.appendChild(link);
+    var link = document.createElement('a');
+    link.href = '#';
+    link.id = id;
+    link.className = 'active';
+    link.textContent = name;
+
+    link.onclick = function(e) {
+        e.preventDefault();
+        e.stopPropagation();
+
+        switch (id) {
+        case 'turf-sync':
+            turfSync();
+            break;
+        case 'turf-async':
+            turfAsync();
+            break;
+        }
+    };
+    var menu = document.getElementById('menu');
+    menu.appendChild(link);
 }
 
 
 function turfSync() {
-  turfResult.setData('');
-  var absUrl = window.location + dataUrl;
-  makeRequest(absUrl);
+    turfResult.setData('');
+    var absUrl = window.location + dataUrl;
+    makeRequest(absUrl);
 }
 
 function makeRequest(url) {
-  var req = new XMLHttpRequest();
-  req.addEventListener('load', validateData);
-  req.addEventListener('error', transferFailed);
-  req.open('GET', url);
-  req.send();
+    var req = new XMLHttpRequest();
+    req.addEventListener('load', validateData);
+    req.addEventListener('error', transferFailed);
+    req.open('GET', url);
+    req.send();
 }
 
 function validateData() {
-  var errors = geojsonhint.hint(this.responseText);
-  if (errors.len > 0) {
-    console.log('Errors', errors.join(', '));
-  } else {
-    turfIt(this.responseText);
-  }
+    var errors = geojsonhint.hint(this.responseText);
+    if (errors.len > 0) {
+        console.log('Errors', errors.join(', '));
+    } else {
+        turfIt(this.responseText);
+    }
 }
 
 function transferFailed() {
-  console.log('Error', this.responseText);
+    console.log('Error', this.responseText);
 }
 
 function turfIt(data) {
-  var results = concave(JSON.parse(data), 1, 'miles');
-  turfResult.setData(results);
+    var results = concave(JSON.parse(data), 1, 'miles');
+    turfResult.setData(results);
 }
 
 function turfAsync() {
-  turfResult.setData('');
-  var absUrl = window.location + dataUrl;
-  w.postMessage([absUrl]);
+    turfResult.setData('');
+    var absUrl = window.location + dataUrl;
+    w.postMessage([absUrl]);
 }
diff --git a/package.json b/package.json
@@ -9,8 +9,10 @@
   "author": "Nick Peihl <[email protected]>",
   "license": "ISC",
   "dependencies": {
+    "geobuf": "^2.0.0",
     "geojsonhint": "^1.2.0",
     "mapbox-gl": "^0.14.1",
+    "pbf": "^1.3.5",
     "turf-concave": "^1.1.3",
     "webworkify": "^1.1.0"
   },

diff --git a/worker.js b/worker.js
@@ -1,36 +1,48 @@
 var concave = require('turf-concave');
 var geojsonhint = require('geojsonhint');
+var geobuf = require('geobuf');
+var Pbf = require('pbf');
 
+debugger;
 
 module.exports = function(self) {
-  self.addEventListener('message', function(ev) {
-    var dataUrl = ev.data[0];
-    self.makeRequest(dataUrl);
-  });
+    self.addEventListener('message', function(ev) {
+        var dataUrl = ev.data[0];
+        self.makeRequest(dataUrl);
+    });
 
-  self.makeRequest = function(url) {
-    var req = new XMLHttpRequest();
-    req.addEventListener('load', self.validateData);
-    req.addEventListener('error', self.transferFailed);
-    req.open('GET', url);
-    req.send();
-  };
+    self.makeRequest = function(url) {
+        var req = new XMLHttpRequest();
+        req.addEventListener('load', self.validateData);
+        req.addEventListener('error', self.transferFailed);
+        req.open('GET', url);
+        req.send();
+    };
 
-  self.validateData = function() {
-    var errors = geojsonhint.hint(this.responseText);
-    if (errors.len > 0) {
-      self.postMessage(['Errors', errors.join(', ') ]);
-    } else {
-      self.turfIt(this.responseText);
-    }
-  };
+    self.validateData = function() {
+        var errors = geojsonhint.hint(this.responseText);
+        if (errors.len > 0) {
+            self.postMessage(['Errors', errors.join(', ') ]);
+        } else {
+            self.turfIt(this.responseText);
+        }
+    };
 
-  self.transferFailed = function() {
-    self.postMessage(['Error', this.responseText]);
-  };
+    self.transferFailed = function() {
+        self.postMessage(['Error', this.responseText]);
+    };
 
-  self.turfIt = function(data) {
-    var results = concave(JSON.parse(data), 1, 'miles');
-    self.postMessage(['Done', results]);
-  };
+    self.turfIt = function(data) {
+        var results = concave(JSON.parse(data), 1, 'miles');
+        var pbf = new Pbf(geobuf.encode(results, new Pbf()));
+        /* The pbf does not contain the readFields method when called via
+         the web worker. However, I can set a breakpoint on the line below,
+         then rewrite the above line using a different variable (e.g. pbf2).
+         The pbf2 variable does contain the readFields method. I believe this
+         is due to the scope of the pbf buffer existing in the browser, but not
+         in the web worker. Possibly releated to this issue
+         (https://github.com/substack/webworkify/issues/14).
+        */
+        self.postMessage(['Done', pbf]);
+    };
 };