perf(core): fix memory leak on rebuild es index

.gitignore

@@ -62,3 +62,5 @@ backups/
 public/uploads/assets/upload/
 
 stats\.json
+
+*.heapsnapshot

package.json

@@ -68,6 +68,7 @@
     "moment-timezone": "0.5.23",
     "mongoose": "5.4.5",
     "mongoose-autopopulate": "0.9.1",
+    "mongoose-lean-virtuals": "0.4.2",
     "nconf": "0.10.0",
     "netmask": "1.0.6",
     "node-cache": "4.2.0",

src/cache/index.js

@@ -13,17 +13,11 @@
  */
 
 var NodeCache = require('node-cache')
-
 var async = require('async')
-
 var path = require('path')
-
 var nconf = require('nconf')
-
 var _ = require('lodash')
-
 var winston = require('winston')
-
 var moment = require('moment')
 
 var truCache = {}
@@ -237,7 +231,7 @@ truCache.refreshCache = function (callback) {
       if (err) return winston.warn(err)
       // Send to parent
       process.send({ cache: cache })
-
+      cache.flushAll()
       if (_.isFunction(callback)) {
         return callback(err)
       }

src/cache/ticketStats.js

@@ -96,11 +96,8 @@ var init = function (tickets, callback) {
     .minute(59)
     .second(59)
   var e30 = today.clone().subtract(30, 'd')
-
   var e60 = today.clone().subtract(60, 'd')
-
   var e90 = today.clone().subtract(90, 'd')
-
   var e180 = today.clone().subtract(180, 'd')
   // e365 = today.clone().subtract(365, 'd');
 

src/controllers/api/v2/elasticsearch.js

@@ -83,9 +83,9 @@ apiElasticSearch.search = function (req, res) {
         var g = _.map(groups, function (i) {
           return i._id
         })
-
+        // For docker we need to add a unique ID for the index.
         var obj = {
-          index: 'trudesk',
+          index: es.indexName,
           body: {
             size: limit,
             from: 0,

src/elasticsearch/index.js

@@ -212,6 +212,7 @@ ES.rebuildIndex = function () {
     })
 
     esFork.on('exit', function () {
+      winston.debug('Rebuilding Process Closed: ' + esFork.pid)
       global.esRebuilding = false
       global.forks = _.filter(global.forks, function (i) {
         return i.name !== 'elasticsearchRebuild'

src/elasticsearch/rebuildIndexChild.js

@@ -20,7 +20,7 @@ winston.add(winston.transports.Console, {
       ' ' +
       date.toTimeString().substr(0, 8) +
       ' [Child:ElasticSearch:' +
-      global.process.pid +
+      process.pid +
       ']'
     )
   },
@@ -54,6 +54,7 @@ function setupDatabase (callback) {
 }
 
 function setupClient () {
+  ES.indexName = process.env.ELASTICSEARCH_INDEX_NAME || 'trudesk'
   ES.esclient = new elasticsearch.Client({
     host: process.env.ELASTICSEARCH_URI,
     pingTimeout: 10000,
@@ -64,14 +65,14 @@ function setupClient () {
 function deleteIndex (callback) {
   ES.esclient.indices.exists(
     {
-      index: 'trudesk'
+      index: ES.indexName
     },
     function (err, exists) {
       if (err) return callback(err)
       if (exists) {
         ES.esclient.indices.delete(
           {
-            index: 'trudesk'
+            index: ES.indexName
           },
           function (err) {
             if (err) return callback(err)
@@ -87,7 +88,7 @@ function deleteIndex (callback) {
 function createIndex (callback) {
   ES.esclient.indices.create(
     {
-      index: 'trudesk',
+      index: ES.indexName,
       body: {
         settings: {
           index: {
@@ -198,12 +199,12 @@ function sendAndEmptyQueue (bulk, callback) {
     ES.esclient.bulk(
       {
         body: bulk,
-        timeout: '2m'
+        timeout: '3m'
       },
       function (err) {
         if (err) {
           process.send({ success: false })
-          throw err
+          return process.exit()
         } else {
           winston.debug('Sent ' + bulk.length + ' documents to Elasticsearch!')
           if (typeof callback === 'function') return callback()
@@ -267,9 +268,10 @@ function crawlTickets (callback) {
 
   stream
     .on('data', function (doc) {
+      stream.pause()
       count += 1
 
-      bulk.push({ index: { _index: 'trudesk', _type: 'doc', _id: doc._id } })
+      bulk.push({ index: { _index: ES.indexName, _type: 'doc', _id: doc._id } })
       var comments = []
       if (doc.comments !== undefined) {
         doc.comments.forEach(function (c) {
@@ -325,6 +327,8 @@ function crawlTickets (callback) {
       })
 
       if (count % 200 === 1) bulk = sendAndEmptyQueue(bulk)
+
+      stream.resume()
     })
     .on('err', function (err) {
       winston.error(err)
@@ -370,12 +374,14 @@ function rebuild (callback) {
   setupClient()
   rebuild(function (err) {
     if (err) {
-      return process.send({ success: false, error: err })
+      process.send({ success: false, error: err })
+      return process.exit(0)
     }
 
     //  Kill it in 10sec to offset refresh timers
     setTimeout(function () {
-      return process.send({ success: true })
+      process.send({ success: true })
+      return process.exit()
     }, 6000)
   })
 })()

src/middleware/middleware.js

@@ -222,20 +222,27 @@ middleware.canUser = function (action) {
 middleware.isAdmin = function (req, res, next) {
   var roles = global.roles
   var role = _.find(roles, { _id: req.user.role._id })
+  role.isAdmin = role.grants.indexOf('admin:*') !== -1
+
   if (role.isAdmin) return next()
 
   return res.status(401).json({ success: false, error: 'Not Authorized for this API call.' })
 }
 
 middleware.isAgentOrAdmin = function (req, res, next) {
   var role = _.find(global.roles, { _id: req.user.role._id })
+  role.isAdmin = role.grants.indexOf('admin:*') !== -1
+  role.isAgent = role.grants.indexOf('agent:*') !== -1
+
   if (role.isAgent || role.isAdmin) return next()
 
   return res.status(401).json({ success: false, error: 'Not Authorized for this API call.' })
 }
 
 middleware.isAgent = function (req, res, next) {
   var role = _.find(global.roles, { _id: req.user.role._id })
+  role.isAgent = role.grants.indexOf('agent:*') !== -1
+
   if (role.isAgent) return next()
 
   return res.status(401).json({ success: false, error: 'Not Authorized for this API call.' })

src/models/role.js

@@ -13,6 +13,7 @@
  **/
 
 var mongoose = require('mongoose')
+var mongooseLeanVirtuals = require('mongoose-lean-virtuals')
 var _ = require('lodash')
 
 var COLLECTION = 'roles'
@@ -26,7 +27,7 @@ var roleSchema = mongoose.Schema(
     hierarchy: { type: Boolean, required: true, default: true }
   },
   {
-    toObject: { getters: true },
+    toObject: { getters: true, virtuals: true },
     toJSON: { virtuals: true }
   }
 )
@@ -47,6 +48,8 @@ roleSchema.virtual('isAgent').get(function () {
   return _.indexOf(role.grants, 'agent:*') !== -1
 })
 
+roleSchema.plugin(mongooseLeanVirtuals)
+
 roleSchema.pre('save', function (next) {
   this.name = this.name.trim()
   this.normalized = this.name.toLowerCase().trim()
@@ -71,6 +74,13 @@ roleSchema.statics.getRoles = function (callback) {
     .exec(callback)
 }
 
+roleSchema.statics.getRolesLean = function (callback) {
+  return this.model(COLLECTION)
+    .find({})
+    .lean({ virtuals: true })
+    .exec(callback)
+}
+
 roleSchema.statics.getRole = function (id, callback) {
   var q = this.model(COLLECTION).findOne({ _id: id })
 

src/models/roleorder.js

@@ -27,6 +27,13 @@ roleOrder.statics.getOrder = function (callback) {
     .exec(callback)
 }
 
+roleOrder.statics.getOrderLean = function (callback) {
+  return this.model(COLLECTION)
+    .findOne({})
+    .lean()
+    .exec(callback)
+}
+
 roleOrder.methods.updateOrder = function (order, callback) {
   this.order = order
   this.save(callback)

src/permissions/index.js

@@ -19,10 +19,10 @@ var roleOrder = require('../models/roleorder')
 
 var register = function (callback) {
   // Register Roles
-  roleSchema.getRoles(function (err, roles) {
+  roleSchema.getRolesLean(function (err, roles) {
     if (err) return callback(err)
 
-    roleOrder.getOrder(function (err, ro) {
+    roleOrder.getOrderLean(function (err, ro) {
       if (err) return callback(err)
 
       winston.debug('Registering Permissions...')

yarn.lock

@@ -8505,6 +8505,13 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mongoose-autopopulate/-/mongoose-autopopulate-0.9.1.tgz#ee497b50a40e065fb54f4be81eef3004d33c17bb"
   integrity sha512-Sh0eJXwENYtZbNBzkGyxxbcwwXeSd8Zi9QLHxLsVwwRoiYPC5qCas8+3mQcyKdWoZJOVV6rBVzS2Q1M/d+3j0g==
 
+[email protected]:
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/mongoose-lean-virtuals/-/mongoose-lean-virtuals-0.4.2.tgz#e476a7f0074e3cdd1fdd441348f9a88c71ed795f"
+  integrity sha512-2zsJBxhNotbRqeVy1uNcS460WylTMG8QliEvjbGC/INAvxA+irPplhyJpg586SuT2lS3sNCFxKOJj3GngCCU/g==
+  dependencies:
+    mpath "0.5.x"
+
 [email protected]:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz#3ba9f91fa507b5186d399fb40854bff18fb563e4"
@@ -8545,6 +8552,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.5.1.tgz#17131501f1ff9e6e4fbc8ffa875aa7065b5775ab"
   integrity sha512-H8OVQ+QEz82sch4wbODFOz+3YQ61FYz/z3eJ5pIdbMEaUzDqA268Wd+Vt4Paw9TJfvDgVKaayC0gBzMIw2jhsg==
 
+[email protected]:
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/mpath/-/mpath-0.5.2.tgz#b1eac586dffb5175d2f51ca9aacba35d9940dd41"
+  integrity sha512-NOeCoW6AYc3hLi30npe7uzbD9b4FQZKH40YKABUCCvaKKL5agj6YzvHoNx8jQpDMNPgIa5bvSZQbQpWBAVD0Kw==
+
 [email protected]:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/mquery/-/mquery-3.2.0.tgz#e276472abd5109686a15eb2a8e0761db813c81cc"