luma_third_party: Multi-device view-hierarchy fix

View hierarchy generation can take more than 1 second to
arrive after a request is sent to a device. This adds
a request timeout for view hierarchies and updates the
queue service to handle request timeouts across multiple
devices.

Change-Id: Iafb608fff465397ab3a63dc63313da334205e964
diff --git a/crowdstf/lib/db/api.js b/crowdstf/lib/db/api.js
index 1189dec..1569ade 100644
--- a/crowdstf/lib/db/api.js
+++ b/crowdstf/lib/db/api.js
@@ -1,5 +1,6 @@
 var r = require('rethinkdb')
 var util = require('util')
+var Promise = require('bluebird');
 
 var db = require('./')
 var wireutil = require('../wire/util')
@@ -433,10 +434,20 @@
   }).then(function() {
     return dbapi.deleteUser(token);
   }).then(function() {
-    return dbapi.publishKickedSerial(serial);
+    if (serial) {
+      return dbapi.publishKickedSerial(serial);
+    } else {
+      // End the chain early.
+      return Promise.resolve();
+    }
   }).then(function() {
-    return dbapi.unsetDeviceOwner(serial);
-  });
+    if (serial) {
+      return dbapi.unsetDeviceOwner(serial);
+    } else {
+      // End the chain.
+      return Promise.resolve();
+    }
+  })
 };
 
 module.exports = dbapi
diff --git a/crowdstf/lib/units/app/index.js b/crowdstf/lib/units/app/index.js
index afaa970..6543d76 100644
--- a/crowdstf/lib/units/app/index.js
+++ b/crowdstf/lib/units/app/index.js
@@ -78,10 +78,6 @@
   , keys: [options.secret]
   }))
 
-  app.get('/task-end', function(req, res) {
-    res.render('taskend', {contactEmail: config.hitAccounts.contactEmail});
-  });
-
   app.use(auth({
     secret: options.secret
   , authUrl: options.authUrl
diff --git a/crowdstf/lib/units/auth/token.js b/crowdstf/lib/units/auth/token.js
index 8b63aee..71eda24 100644
--- a/crowdstf/lib/units/auth/token.js
+++ b/crowdstf/lib/units/auth/token.js
@@ -16,6 +16,7 @@
 var urlutil = require('../../util/urlutil');
 var lifecycle = require('../../util/lifecycle');
 var dbapi = require('../../db/api');
+var config = require('../../../config');
 
 const JWT_EXPIRE_LENGTH = 24 * 3600;
 const DEFAULT_EXPIRE_MINS = 5.0;
@@ -26,6 +27,11 @@
   var app = express();
   var server = Promise.promisifyAll(http.createServer(app));
 
+  // The auth module usually redirects, but use jade to display configurable
+  // logout & session end messaging.
+  app.set('view engine', 'jade');
+  app.set('views', pathutil.resource('app/views'));
+
   lifecycle.observe(function() {
     log.info('Waiting for client connections to end');
     return server.closeAsync();
@@ -77,7 +83,9 @@
     res.clearCookie('XSRF-TOKEN');
     res.clearCookie('ssid');
     res.clearCookie('ssid.sig');
-    res.redirect('/task-end');
+
+    // Show token expiration messaging.
+    res.render('taskend', {contactEmail: config.hitAccounts.contactEmail});
   };
 
   app.get('/auth/token', resetSession);
@@ -149,7 +157,7 @@
         }
       }).catch(function(err) {
         log.error('Failed to load token "%s": ', token, err.stack);
-        return res.redirect('/task-end');
+        return res.redirect('/auth/token');
       });
     } else {
       return res.status(400).json({
diff --git a/crowdstf/lib/units/device/plugins/viewbridge.js b/crowdstf/lib/units/device/plugins/viewbridge.js
index 2cda2be..1db5023 100644
--- a/crowdstf/lib/units/device/plugins/viewbridge.js
+++ b/crowdstf/lib/units/device/plugins/viewbridge.js
@@ -55,8 +55,8 @@
                     log.info('Starting view bridge.');
                     return openViewBridge(options.serial);
                   })
-                  .then(function(logcat) {
-                    activeViewBridge = logcat;
+                  .then(function(viewBridge) {
+                    activeViewBridge = viewBridge;
 
                     function entryListener(entry) {
                       try {
diff --git a/crowdstf/lib/units/websocket/index.js b/crowdstf/lib/units/websocket/index.js
index 1b992d6..44ec62d 100644
--- a/crowdstf/lib/units/websocket/index.js
+++ b/crowdstf/lib/units/websocket/index.js
@@ -27,10 +27,11 @@
 var layoutCaptureService = require('./layoutcaptureservice')
 
 const START_LOGCAT_DELIM = 'RicoBegin';
-const START_DELIM_CHAR = START_LOGCAT_DELIM.substr(0, 1);
 const END_LOGCAT_DELIM = 'RicoEnd';
-const START_DELIM_LEN = START_LOGCAT_DELIM.length;
 const VIEW_JSON_END_DELIMITER = 'RICO_JSON_END';
+const VIEW_REQ_XMIT_TIMEOUT = 1000;
+const VIEW_REQ_ERR_MSG = 'Err:View hierarchy request exceeded ' +
+  VIEW_REQ_XMIT_TIMEOUT + 'ms.';
 
 module.exports = function(options) {
   var log = logger.createLogger('websocket')
@@ -40,8 +41,8 @@
       , transports: ['websocket']
       })
   var channelRouter = new events.EventEmitter()
-  var viewHierarchyJSON = '';
-  var viewResHandler = null;
+  var deviceViewJson = {};
+  var viewResHandlers = {};
 
   // Output
   var push = zmqutil.socket('push')
@@ -238,9 +239,22 @@
         }
       })
       .on(wire.DeviceViewBridgeEntryMessage, function(channel, message) {
-        viewHierarchyJSON += message.message;
+        deviceViewJson[message.serial] = (deviceViewJson[message.serial] || '')
+            + message.message;
         if (message.message.indexOf(VIEW_JSON_END_DELIMITER) > -1) {
-          viewResHandler(viewHierarchyJSON);
+          // Check if the response handler hung up due to timeout.
+          if(message.serial && viewResHandlers[message.serial]) {
+            // Stash a reference to the handler.
+            var viewResHandler = viewResHandlers[message.serial];
+
+            // Ready the handlers for the next handler.
+            viewResHandlers[message.serial] = null;
+
+            // Invoke the save response defined in the gesture.
+            viewResHandler(deviceViewJson[message.serial]);
+          } else {
+            log.warn('Ignoring view response for serial %s', message.serial);
+          }
         }
       })
       .on(wire.AirplaneModeEvent, function(channel, message) {
@@ -540,8 +554,9 @@
               ))
             ]);
           }, function(callback) {
-            viewHierarchyJSON = '';
-            viewResHandler = function(viewHierarchy) {
+            deviceViewJson[data.serial] = '';
+
+            viewResHandlers[data.serial] = function(viewHierarchy) {
               data.viewHierarchy = viewHierarchy;
               deviceEventStore.storeEvent('input.gestureStart', data);
               callback();
@@ -550,9 +565,24 @@
             // Send a request to the TCP view bridge.
             push.send([channel,
               wireutil.envelope(new wire.ViewBridgeGetMessage(
-                  data.imgId.split('_')[1]
+                  data.imgId.split('_')[1],
+                  data.serial
               ))
             ]);
+
+            // If we don't hear back from the device's view hierarchy service,
+            // ignore the request, log a warning, and move on.
+            setTimeout(function noResponse(){
+              if (viewResHandlers[data.serial]) {
+                log.warn('View hierarchy response timed out, ' +
+                  'skipping request.');
+                var viewResHandler = viewResHandlers[data.serial];
+                viewResHandlers[data.serial] = null;
+
+                // Invoke the gesture save with an error message.
+                viewResHandler(VIEW_REQ_ERR_MSG);
+              }
+            }, VIEW_REQ_XMIT_TIMEOUT);
           }, data.serial);
         })
         .on('input.gestureStop', function(channel, data) {
diff --git a/crowdstf/lib/units/websocket/layoutcaptureservice.js b/crowdstf/lib/units/websocket/layoutcaptureservice.js
index 5fa2d4f..4f9e71a 100644
--- a/crowdstf/lib/units/websocket/layoutcaptureservice.js
+++ b/crowdstf/lib/units/websocket/layoutcaptureservice.js
@@ -55,11 +55,12 @@
         if (eventActionObj.wireEvent === wire.GestureStartMessage) {
           eventActionObj.fetchView(function(err, layoutJSON) {
             if (err) {
-              console.error(err);
+              log.error('Fetch view failed for serial %s:', serial, err);
             } else {
               eventActionObj.actionFn(layoutJSON);
-              nextItem();
             }
+
+            nextItem();
           });
         } else {
           eventActionObj.actionFn();
diff --git a/crowdstf/lib/wire/wire.proto b/crowdstf/lib/wire/wire.proto
index cc26319..7ba6a0b 100644
--- a/crowdstf/lib/wire/wire.proto
+++ b/crowdstf/lib/wire/wire.proto
@@ -402,6 +402,7 @@
 
 message ViewBridgeGetMessage {
   optional string seq = 1;
+  required string serial = 2;
 }
 
 message LogcatApplyFiltersMessage {