From 16bd9f8c83e4dd0d1deaaca90b9c371778009d48 Mon Sep 17 00:00:00 2001 From: Chris Brame Date: Fri, 17 May 2019 02:46:13 -0400 Subject: [PATCH] perf(cache): improvements --- .codacy.yml | 3 +- app.js | 22 +-- src/backup/backup.js | 2 +- src/backup/restore.js | 2 +- src/cache/cache.js | 53 +++++ src/cache/index.js | 15 +- src/cache/node-cache.js | 415 --------------------------------------- src/middleware/index.js | 2 +- src/settings/defaults.js | 27 +-- 9 files changed, 80 insertions(+), 461 deletions(-) create mode 100644 src/cache/cache.js delete mode 100644 src/cache/node-cache.js diff --git a/.codacy.yml b/.codacy.yml index d9359fabd..a21de7f5b 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -9,5 +9,4 @@ exclude_paths: - src/public/js/plugins/**/* - src/public/js/vendor/** - src/public/js/vendor/**/* - - test/**/* - - src/cache/node-cache.js \ No newline at end of file + - test/**/* \ No newline at end of file diff --git a/app.js b/app.js index f0e386141..dfaf906b7 100644 --- a/app.js +++ b/app.js @@ -193,12 +193,7 @@ function launchServer (db) { // return next() // }, function (next) { - var NodeCache = require('./src/cache/node-cache') - global.cache = new NodeCache({ checkperiod: 0 }) - var fork = require('child_process').fork - var memLimit = nconf.get('memlimit') || '2048' - - var env = { FORK: 1, NODE_ENV: global.env } + var cache = require('./src/cache/cache') if (isDocker) { var envDocker = { TRUDESK_DOCKER: process.env.TRUDESK_DOCKER, @@ -209,21 +204,10 @@ function launchServer (db) { TD_MONGODB_DATABASE: process.env.TD_MONGODB_DATABASE } - env = _.merge(env, envDocker) + cache.env = envDocker } - var n = fork(path.join(__dirname, '/src/cache/index.js'), { - execArgv: ['--max-old-space-size=' + memLimit], - env: env - }) - - global.forks.push({ name: 'cache', fork: n }) - - n.on('message', function (data) { - if (data.cache) { - global.cache.data = data.cache.data - } - }) + cache.init() return next() }, diff --git a/src/backup/backup.js b/src/backup/backup.js index 9328c8339..500ba5b69 100644 --- a/src/backup/backup.js +++ b/src/backup/backup.js @@ -39,7 +39,7 @@ winston.add(winston.transports.Console, { ' ' + date.toTimeString().substr(0, 8) + ' [Child:Backup:' + - global.process.pid + + process.pid + ']' ) }, diff --git a/src/backup/restore.js b/src/backup/restore.js index aac5e7c9d..838674f7e 100644 --- a/src/backup/restore.js +++ b/src/backup/restore.js @@ -41,7 +41,7 @@ winston.add(winston.transports.Console, { ' ' + date.toTimeString().substr(0, 8) + ' [Child:Backup:' + - global.process.pid + + process.pid + ']' ) }, diff --git a/src/cache/cache.js b/src/cache/cache.js new file mode 100644 index 000000000..a9e276bb9 --- /dev/null +++ b/src/cache/cache.js @@ -0,0 +1,53 @@ +/* + * . .o8 oooo + * .o8 "888 `888 + * .o888oo oooo d8b oooo oooo .oooo888 .ooooo. .oooo.o 888 oooo + * 888 `888""8P `888 `888 d88' `888 d88' `88b d88( "8 888 .8P' + * 888 888 888 888 888 888 888ooo888 `"Y88b. 888888. + * 888 . 888 888 888 888 888 888 .o o. )88b 888 `88b. + * "888" d888b `V88V"V8P' `Y8bod88P" `Y8bod8P' 8""888P' o888o o888o + * ======================================================================== + * Author: Chris Brame + * Updated: 5/17/19 2:03 AM + * Copyright (c) 2014-2019. All rights reserved. + */ + +var _ = require('lodash') +var NodeCache = require('node-cache') +var path = require('path') +var cache = {} + +cache.init = function () { + global.cache = new NodeCache({ checkperiod: 0 }) + cache.memLimit = process.env.CACHE_MEMLIMIT || '2048' + var env = { FORK: 1, NODE_ENV: global.env } + cache.env = _.merge(cache.env, env) + + spawnCache() + setInterval(spawnCache, 55 * 60 * 1000) +} + +function spawnCache () { + var fork = require('child_process').fork + console.log(cache.env) + var n = fork(path.join(__dirname, './index.js'), { + execArgv: ['--max-old-space-size=' + cache.memLimit], + env: cache.env + }) + + global.forks.push({ name: 'cache', fork: n }) + + n.on('message', function (data) { + if (data.cache) { + global.cache.data = data.cache.data + } + }) + + n.on('close', function () { + _.remove(global.forks, function (i) { + return i.name === 'cache' + }) + }) +} + +module.exports = cache diff --git a/src/cache/index.js b/src/cache/index.js index b8dd474e6..7eb498e32 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -39,7 +39,7 @@ winston.add(winston.transports.Console, { ' ' + date.toTimeString().substr(0, 8) + ' [Child:Cache:' + - global.process.pid + + process.pid + ']' ) }, @@ -66,7 +66,7 @@ truCache.init = function (callback) { truCache.refreshCache(function () { winston.debug('Cache Loaded') - restartRefreshClock() + // restartRefreshClock() return callback() }) @@ -103,9 +103,7 @@ truCache.refreshCache = function (callback) { [ function (done) { var ticketStats = require('./ticketStats') - // console.time('test'); ticketStats(tickets, function (err, stats) { - // console.timeEnd('test'); if (err) return done(err) var expire = 3600 // 1 hour cache.set('tickets:overview:lastUpdated', stats.lastUpdated, expire) @@ -135,11 +133,6 @@ truCache.refreshCache = function (callback) { cache.set('tickets:overview:e365:responseTime', stats.e365.avgResponse, expire) cache.set('tickets:overview:e365:graphData', stats.e365.graphData, expire) - // cache.set('tickets:overview:lifetime:ticketCount', stats.lifetime.tickets, expire); - // cache.set('tickets:overview:lifetime:closedTickets', stats.lifetime.closedTickets, expire); - // cache.set('tickets:overview:lifetime:responseTime', stats.lifetime.avgResponse, expire); - // cache.set('tickets:overview:lifetime:graphData', stats.lifetime.graphData, expire); - return done() }) }, @@ -232,7 +225,9 @@ truCache.refreshCache = function (callback) { if (err) return winston.warn(err) // Send to parent process.send({ cache: cache }) + cache.flushAll() + if (_.isFunction(callback)) { return callback(err) } @@ -278,6 +273,8 @@ truCache.refreshCache = function (callback) { winston.error(err) throw new Error(err) } + + return process.exit(0) }) }) })() diff --git a/src/cache/node-cache.js b/src/cache/node-cache.js deleted file mode 100644 index 565492eae..000000000 --- a/src/cache/node-cache.js +++ /dev/null @@ -1,415 +0,0 @@ -;(function () { - var EventEmitter - var _ - var clone - - var bind = function (fn, me) { - return function () { - return fn.apply(me, arguments) - } - } - - var extend = function (child, parent) { - for (var key in parent) if (hasProp.call(parent, key)) child[key] = parent[key] - function ctor () { - this.constructor = child - } - ctor.prototype = parent.prototype - child.prototype = new ctor() - child.__super__ = parent.prototype - return child - } - - var hasProp = {}.hasOwnProperty - var slice = [].slice - _ = require('lodash') - clone = require('clone') - EventEmitter = require('events').EventEmitter - - module.exports = (function (superClass) { - extend(NodeCache, superClass) - - function NodeCache (options) { - this.options = options != null ? options : {} - this._initErrors = bind(this._initErrors, this) - this._error = bind(this._error, this) - this._getValLength = bind(this._getValLength, this) - this._wrap = bind(this._wrap, this) - this._check = bind(this._check, this) - this._checkData = bind(this._checkData, this) - this.close = bind(this.close, this) - this.flushAll = bind(this.flushAll, this) - this.getStats = bind(this.getStats, this) - this.keys = bind(this.keys, this) - this.ttl = bind(this.ttl, this) - this.del = bind(this.del, this) - this.set = bind(this.set, this) - this.mget = bind(this.mget, this) - this.get = bind(this.get, this) - this._initErrors() - this.data = {} - if (options.data) { - this.data = options.data - } - this.options = _.extend( - { - forceString: false, - objectValueSize: 80, - arrayValueSize: 40, - stdTTL: 0, - checkperiod: 600, - useClones: true, - errorOnMissing: false - }, - this.options - ) - this.stats = { - hits: 0, - misses: 0, - keys: 0, - ksize: 0, - vsize: 0 - } - this._checkData() - } - - NodeCache.prototype.get = function (key, cb, errorOnMissing) { - var _err, _ret - if (typeof cb === 'boolean' && arguments.length === 2) { - errorOnMissing = cb - cb = void 0 - } - if (this.data[key] != null && this._check(key, this.data[key])) { - this.stats.hits++ - _ret = this._unwrap(this.data[key]) - if (cb != null) { - cb(null, _ret) - } - - return _ret - } else { - this.stats.misses++ - if (this.options.errorOnMissing || errorOnMissing) { - _err = this._error( - 'ENOTFOUND', - { - key: key - }, - cb - ) - if (_err != null) { - throw _err - } - - return - } else { - if (cb != null) { - cb(null, void 0) - } - } - return void 0 - } - } - - NodeCache.prototype.mget = function (keys, cb) { - var _err, i, key, len, oRet - if (!_.isArray(keys)) { - _err = this._error('EKEYSTYPE') - if (cb != null) { - cb(_err) - } - - return _err - } - oRet = {} - for (i = 0, len = keys.length; i < len; i++) { - key = keys[i] - if (this.data[key] != null && this._check(key, this.data[key])) { - this.stats.hits++ - oRet[key] = this._unwrap(this.data[key]) - } else { - this.stats.misses++ - } - } - if (cb != null) { - cb(null, oRet) - } - - return oRet - } - - NodeCache.prototype.set = function (key, value, ttl, cb) { - var existend - existend = false - if (this.options.forceString && !_.isString(value)) { - value = JSON.stringify(value) - } - - if (arguments.length === 3 && _.isFunction(ttl)) { - cb = ttl - ttl = this.options.stdTTL - } - if (this.data[key]) { - existend = true - this.stats.vsize -= this._getValLength(this._unwrap(this.data[key], false)) - } - this.data[key] = this._wrap(value, ttl) - this.stats.vsize += this._getValLength(value) - if (!existend) { - this.stats.ksize += this._getKeyLength(key) - this.stats.keys++ - } - this.emit('set', key, value) - if (cb != null) { - cb(null, true) - } - - return true - } - - NodeCache.prototype.del = function (keys, cb) { - var delCount, i, key, len, oldVal - if (_.isString(keys)) { - keys = [keys] - } - - delCount = 0 - for (i = 0, len = keys.length; i < len; i++) { - key = keys[i] - if (this.data[key] != null) { - this.stats.vsize -= this._getValLength(this._unwrap(this.data[key], false)) - this.stats.ksize -= this._getKeyLength(key) - this.stats.keys-- - delCount++ - oldVal = this.data[key] - delete this.data[key] - this.emit('del', key, oldVal.v) - } else { - this.stats.misses++ - } - } - if (cb != null) { - cb(null, delCount) - } - - return delCount - } - - NodeCache.prototype.ttl = function () { - var arg, args, cb, i, key, len, ttl - ;(key = arguments[0]), (args = arguments.length >= 2 ? slice.call(arguments, 1) : []) - for (i = 0, len = args.length; i < len; i++) { - arg = args[i] - switch (typeof arg) { - case 'number': - ttl = arg - break - case 'function': - cb = arg - } - } - ttl || (ttl = this.options.stdTTL) - if (!key) { - if (cb != null) { - cb(null, false) - } - - return false - } - if (this.data[key] != null && this._check(key, this.data[key])) { - if (ttl > 0) { - this.data[key] = this._wrap(this.data[key].v, ttl, false) - } else { - this.del(key) - } - - if (cb != null) { - cb(null, true) - } - - return true - } else { - if (cb != null) { - cb(null, false) - } - - return false - } - } - - NodeCache.prototype.keys = function (cb) { - var _keys - _keys = Object.keys(this.data) - if (cb != null) { - cb(null, _keys) - } - - return _keys - } - - NodeCache.prototype.getStats = function () { - return this.stats - } - - NodeCache.prototype.flushAll = function (_startPeriod) { - if (_startPeriod == null) { - _startPeriod = true - } - - this.data = {} - this.stats = { - hits: 0, - misses: 0, - keys: 0, - ksize: 0, - vsize: 0 - } - this._killCheckPeriod() - this._checkData(_startPeriod) - this.emit('flush') - } - - NodeCache.prototype.close = function () { - this._killCheckPeriod() - } - - NodeCache.prototype._checkData = function (startPeriod) { - var key, ref, value - if (startPeriod == null) { - startPeriod = true - } - - ref = this.data - for (key in ref) { - value = ref[key] - this._check(key, value) - } - if (startPeriod && this.options.checkperiod > 0) { - this.checkTimeout = setTimeout(this._checkData, this.options.checkperiod * 1000) - if (this.checkTimeout.unref != null) { - this.checkTimeout.unref() - } - } - } - - NodeCache.prototype._killCheckPeriod = function () { - if (this.checkTimeout != null) { - return clearTimeout(this.checkTimeout) - } - } - - NodeCache.prototype._check = function (key, data) { - if (data.t !== 0 && data.t < Date.now()) { - this.del(key) - this.emit('expired', key, this._unwrap(data)) - return false - } else { - return true - } - } - - NodeCache.prototype._wrap = function (value, ttl, asClone) { - var livetime, now, ttlMultiplicator - if (asClone === null) { - asClone = true - } - - if (!this.options.useClones) { - asClone = false - } - - now = Date.now() - livetime = 0 - ttlMultiplicator = 1000 - if (ttl === 0) { - livetime = 0 - } else if (ttl) { - livetime = now + ttl * ttlMultiplicator - } else { - if (this.options.stdTTL === 0) { - livetime = this.options.stdTTL - } else { - livetime = now + this.options.stdTTL * ttlMultiplicator - } - } - return { - t: livetime, - v: asClone ? clone(value) : value - } - } - - NodeCache.prototype._unwrap = function (value, asClone) { - if (asClone === null) { - asClone = true - } - - if (!this.options.useClones) { - asClone = false - } - - if (value.v !== null) { - if (asClone) { - return clone(value.v) - } else { - return value.v - } - } - return null - } - - NodeCache.prototype._getKeyLength = function (key) { - return key.length - } - - NodeCache.prototype._getValLength = function (value) { - if (_.isString(value)) { - return value.length - } else if (this.options.forceString) { - return JSON.stringify(value).length - } else if (_.isArray(value)) { - return this.options.arrayValueSize * value.length - } else if (_.isNumber(value)) { - return 8 - } else if (_.isObject(value)) { - return this.options.objectValueSize * _.size(value) - } else { - return 0 - } - } - - NodeCache.prototype._error = function (type, data, cb) { - var error - if (data == null) { - data = {} - } - - error = new Error() - error.name = type - error.errorcode = type - error.message = this.ERRORS[type] != null ? this.ERRORS[type](data) : '-' - error.data = data - if (cb && _.isFunction(cb)) { - cb(error, null) - } else { - return error - } - } - - NodeCache.prototype._initErrors = function () { - var _errMsg, _errT, ref - this.ERRORS = {} - ref = this._ERRORS - for (_errT in ref) { - _errMsg = ref[_errT] - this.ERRORS[_errT] = _.template(_errMsg) - } - } - - NodeCache.prototype._ERRORS = { - ENOTFOUND: 'Key `<%= key %>` not found', - EKEYSTYPE: 'The keys argument has to be an array.' - } - - return NodeCache - })(EventEmitter) -}.call(this)) diff --git a/src/middleware/index.js b/src/middleware/index.js index f874bfdc2..4d7f7d698 100644 --- a/src/middleware/index.js +++ b/src/middleware/index.js @@ -34,7 +34,7 @@ module.exports = function (app, db, callback) { app.disable('x-powered-by') app.set('views', path.join(__dirname, '../views/')) - global.HandleBars = HandleBars + app.engine( 'hbs', hbs.express4({ diff --git a/src/settings/defaults.js b/src/settings/defaults.js index 4902f29af..f184f1a90 100644 --- a/src/settings/defaults.js +++ b/src/settings/defaults.js @@ -23,9 +23,10 @@ var SettingsSchema = require('../models/setting') var PrioritySchema = require('../models/ticketpriority') var settingsDefaults = {} +var roleDefaults = {} -settingsDefaults.userGrants = ['tickets:create view update', 'comments:create view update'] -settingsDefaults.supportGrants = [ +roleDefaults.userGrants = ['tickets:create view update', 'comments:create view update'] +roleDefaults.supportGrants = [ 'tickets:*', 'agent:*', 'accounts:create update view import', @@ -34,7 +35,7 @@ settingsDefaults.supportGrants = [ 'reports:view create', 'notices:*' ] -settingsDefaults.adminGrants = [ +roleDefaults.adminGrants = [ 'admin:*', 'agent:*', 'chat:*', @@ -64,7 +65,7 @@ function rolesDefault (callback) { { name: 'User', description: 'Default role for users', - grants: settingsDefaults.userGrants + grants: roleDefaults.userGrants }, function (err, userRole) { if (err) return done(err) @@ -95,7 +96,7 @@ function rolesDefault (callback) { { name: 'Support', description: 'Default role for agents', - grants: settingsDefaults.supportGrants + grants: roleDefaults.supportGrants }, done ) @@ -111,7 +112,7 @@ function rolesDefault (callback) { { name: 'Admin', description: 'Default role for admins', - grants: settingsDefaults.adminGrants + grants: roleDefaults.adminGrants }, done ) @@ -144,6 +145,9 @@ function rolesDefault (callback) { ], function (err) { if (err) throw err + + roleDefaults = null + return callback() } ) @@ -151,7 +155,7 @@ function rolesDefault (callback) { function defaultUserRole (callback) { var roleOrderSchema = require('../models/roleorder') - roleOrderSchema.getOrder(function (err, roleOrder) { + roleOrderSchema.getOrderLean(function (err, roleOrder) { if (err) return callback(err) if (!roleOrder) return callback() @@ -686,9 +690,6 @@ settingsDefaults.init = function (callback) { function (done) { return timezoneDefault(done) }, - function (done) { - return showTourSettingDefault(done) - }, function (done) { return ticketTypeSettingDefault(done) }, @@ -708,14 +709,14 @@ settingsDefaults.init = function (callback) { return mailTemplates(done) }, function (done) { - elasticSearchConfToDB(done) + return elasticSearchConfToDB(done) }, function (done) { - installationID(done) + return installationID(done) } ], function (err) { - if (err) winston.debug(err) + if (err) winston.warn(err) if (_.isFunction(callback)) return callback() } )