feat: add redis cache to import list all versions api perf (#1441)

common/cache.js

@@ -0,0 +1,16 @@
+'use strict';
+
+const debug = require('debug')('cnpmjs.org:cache');
+const Redis = require('ioredis');
+const config = require('../config');
+
+let client;
+
+if (config.redisCache.enable) {
+  client = new Redis(config.redisCache.connectOptions);
+  client.on('ready', () => {
+    debug('connect ready, getBuiltinCommands: %j', client.getBuiltinCommands());
+  });
+}
+
+module.exports = client;

common/urllib.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - common/urllib.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var urlparse = require('url').parse;
 var urllib = require('urllib');
 var HttpAgent = require('agentkeepalive');

config/index.js

@@ -266,6 +266,12 @@ var config = {
   opensearch: {
     host: '',
   },
+
+  // redis cache
+  redisCache: {
+    enable: false,
+    connectOptions: null,
+  },
 };
 
 if (process.env.NODE_ENV === 'test') {

controllers/registry/package/list.js

@@ -6,6 +6,8 @@ var packageService = require('../../../services/package');
 var common = require('../../../lib/common');
 var SyncModuleWorker = require('../../sync_module_worker');
 var config = require('../../../config');
+const cache = require('../../../common/cache');
+const logger = require('../../../common/logger');
 
 // https://forum.nginx.org/read.php?2,240120,240120#msg-240120
 // should set weak etag avoid nginx remove it
@@ -18,7 +20,33 @@ function etag(objs) {
  * GET /:name
  */
 module.exports = function* list() {
-  var name = this.params.name || this.params[0];
+  const name = this.params.name || this.params[0];
+  // sync request will contain this query params
+  let noCache = this.query.cache === '0';
+  const isJSONPRequest = this.query.callback;
+  let cacheKey;
+  let needAbbreviatedMeta = false;
+  let abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
+  if (config.enableAbbreviatedMetadata && this.accepts([ 'json', abbreviatedMetaType ]) === abbreviatedMetaType) {
+    needAbbreviatedMeta = true;
+    if (cache && !isJSONPRequest) {
+      cacheKey = `list-${name}-v1`;
+    }
+  }
+
+  if (cacheKey && !noCache) {
+    const values = yield cache.hmget(cacheKey, 'etag', 'body');
+    if (values && values[0] && values[1]) {
+      this.body = values[1];
+      this.type = 'json';
+      this.etag = values[0];
+      this.set('x-hit-cache', cacheKey);
+      debug('hmget %s success, etag:%j', cacheKey, values[0]);
+      return;
+    }
+    debug('hmget %s missing, %j', cacheKey, values);
+  }
+
   var rs = yield [
     packageService.getModuleLastModified(name),
     packageService.listModuleTags(name),
@@ -46,11 +74,10 @@ module.exports = function* list() {
     }
   }
 
-  var abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
-  if (config.enableAbbreviatedMetadata && this.accepts([ 'json', abbreviatedMetaType ]) === abbreviatedMetaType) {
+  if (needAbbreviatedMeta) {
     var rows = yield packageService.listModuleAbbreviatedsByName(name);
     if (rows.length > 0) {
-      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows);
+      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows, cacheKey);
       return;
     }
     var fullRows = yield packageService.listModulesByName(name);
@@ -228,8 +255,9 @@ module.exports = function* list() {
   ]);
 };
 
-function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows) {
+function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cacheKey) {
   debug('show %s got %d rows, %d tags, modifiedTime: %s', name, rows.length, tags.length, modifiedTime);
+  const isJSONPRequest = ctx.query.callback;
   var latestMod = null;
   // set tags
   var distTags = {};
@@ -274,13 +302,32 @@ function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows) {
   };
 
   debug('show %j', info);
-  ctx.jsonp = info;
   // use faster etag
-  ctx.etag = etag([
+  const resultEtag = etag([
     modifiedTime,
     distTags,
     allVersionString,
   ]);
+
+  if (isJSONPRequest) {
+    ctx.jsonp = info;
+  } else {
+    ctx.body = JSON.stringify(info);
+    ctx.type = 'json';
+    // set cache
+    if (cacheKey) {
+      // set cache async, dont block the response
+      cache.pipeline()
+        .hmset(cacheKey, 'etag', resultEtag, 'body', ctx.body)
+        // cache 120s
+        .expire(cacheKey, 120)
+        .exec()
+        .catch(err => {
+          logger.error(err);
+        });
+    }
+  }
+  ctx.etag = resultEtag;
 }
 
 function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags, rows) {

controllers/sync_module_worker.js

@@ -25,6 +25,7 @@ var downloadTotalService = require('../services/download_total');
 var hook = require('../services/hook');
 var User = require('../models').User;
 var os = require('os');
+const cache = require('../common/cache');
 
 var USER_AGENT = 'sync.cnpmjs.org/' + config.version +
   ' hostname/' + os.hostname() +
@@ -190,6 +191,14 @@ SyncModuleWorker.prototype.add = function (name) {
 };
 
 SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
+  // clean cache
+  if (cache) {
+    const cacheKey = `list-${name}-v1`;
+    cache.del(cacheKey).catch(err => {
+      logger.error(err);
+    });
+  }
+
   this.log('----------------- Synced %s %s -------------------',
     name, success ? 'success' : 'fail');
   if (success) {
@@ -629,7 +638,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   // get package AbbreviatedMetadata
   var remoteAbbreviatedMetadatas = {};
   if (config.enableAbbreviatedMetadata) {
-    var packageUrl = '/' + name.replace('/', '%2f');
+    // use ?cache=0 tell registry dont use cache result
+    var packageUrl = '/' + name.replace('/', '%2f') + '?cache=0';
     var result = yield npmSerivce.request(packageUrl, {
       dataType: 'text',
       registry: config.sourceNpmRegistry,

package.json

@@ -35,6 +35,7 @@
     "he": "^1.2.0",
     "humanize-ms": "^1.2.1",
     "humanize-number": "~0.0.2",
+    "ioredis": "^4.6.2",
     "is-type-of": "^1.2.0",
     "kcors": "^1.2.1",
     "koa": "^1.2.0",