Release 3.0.0-alpha.3

- https://github.com/cnpm/cnpmjs.org/issues/1149
- https://github.com/cnpm/cnpmjs.org/issues/1148

History.md

@@ -1,4 +1,28 @@
 
+3.0.0-alpha.2 / 2017-03-27
+==================
+
+  * fix: only get from package_readme table
+
+3.0.0-alpha.1 / 2017-03-27
+==================
+
+  * chore: start 3.x
+  * fix: ignore sync npm registry status 502
+  * feat: remove readme from package
+  * feat: [BREAKING_CHANGE] support abbreviated meta
+
+2.19.4 / 2017-03-26
+==================
+
+  * feat: need to sync sourceNpmRegistry also (#1153)
+  * docs: change user.json to utf8mb4
+
+2.19.3 / 2017-02-22
+==================
+
+  * fix: should get package from orginal registry when package is unpublished (#1130)
+
 2.19.2 / 2017-01-05
 ==================
 

Makefile

@@ -1,6 +1,6 @@
 TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
 REPORTER = spec
-TIMEOUT = 30000
+TIMEOUT = 60000
 MOCHA_OPTS =
 DB = sqlite
 
@@ -8,7 +8,7 @@ jshint:
 	@node_modules/.bin/jshint .
 
 init-database:
-	@node test/init_db.js
+	@NODE_ENV=test node test/init_db.js
 
 init-mysql:
 	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
@@ -22,8 +22,8 @@ test: init-database
 	@NODE_ENV=test DB=${DB} node_modules/.bin/mocha \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -47,8 +47,8 @@ test-cov cov: init-database
 		-- -u exports \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -68,8 +68,8 @@ test-travis: init-database
 		-- -u exports \
 		--reporter dot \
 		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -99,6 +99,6 @@ autod:
 		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
 		--dep mysql \
-		--keep should,supertest,should-http,chunkstream,mm,pedding
+		--keep should,supertest,chunkstream,mm,pedding
 
 .PHONY: test

config/index.js

@@ -221,8 +221,16 @@ var config = {
 
   // snyk.io root url
   snykUrl: 'https://snyk.io',
+
+  // https://github.com/cnpm/cnpmjs.org/issues/1149
+  // if enable this option, must create module_abbreviated and package_readme table in database
+  enableAbbreviatedMetadata: false,
 };
 
+if (process.env.NODE_ENV === 'test') {
+  config.enableAbbreviatedMetadata = true;
+}
+
 if (process.env.NODE_ENV !== 'test') {
   var customConfig;
   if (process.env.NODE_ENV === 'development') {

controllers/registry/package/list.js

@@ -1,9 +1,5 @@
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
 var packageService = require('../../../services/package');
 var common = require('../../../lib/common');
@@ -15,16 +11,15 @@ var config = require('../../../config');
  * GET /:name
  */
 module.exports = function* list() {
-  var orginalName = this.params.name || this.params[0];
-  var name = orginalName;
+  var name = this.params.name || this.params[0];
   var rs = yield [
     packageService.getModuleLastModified(name),
-    packageService.listModuleTags(name)
+    packageService.listModuleTags(name),
   ];
   var modifiedTime = rs[0];
   var tags = rs[1];
 
-  debug('show %s(%s), last modified: %s, tags: %j', name, orginalName, modifiedTime, tags);
+  debug('show %s, last modified: %s, tags: %j', name, modifiedTime, tags);
   if (modifiedTime) {
     // find out the latest modfied time
     // because update tags only modfied tag, wont change module gmt_modified
@@ -44,6 +39,21 @@ module.exports = function* list() {
     }
   }
 
+  var abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
+  if (config.enableAbbreviatedMetadata && this.accepts([ 'json', abbreviatedMetaType ]) === abbreviatedMetaType) {
+    var rows = yield packageService.listModuleAbbreviatedsByName(name);
+    if (rows.length > 0) {
+      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows);
+      return;
+    }
+    var fullRows = yield packageService.listModulesByName(name);
+    if (fullRows.length > 0) {
+      // no abbreviated meta rows, use the full meta convert to abbreviated meta
+      yield handleAbbreviatedMetaRequestWithFullMeta(this, name, modifiedTime, tags, fullRows);
+      return;
+    }
+  }
+
   var r = yield [
     packageService.listModulesByName(name),
     packageService.listStarUserNames(name),
@@ -67,18 +77,18 @@ module.exports = function* list() {
 
   if (rows.length === 0) {
     // check if unpublished
-    var unpublishedInfo = yield* packageService.getUnpublishedModule(name);
+    var unpublishedInfo = yield packageService.getUnpublishedModule(name);
     debug('show unpublished %j', unpublishedInfo);
     if (unpublishedInfo) {
       this.status = 404;
-      this.body = {
-        _id: orginalName,
-        name: orginalName,
+      this.jsonp = {
+        _id: name,
+        name: name,
         time: {
           modified: unpublishedInfo.package.time,
           unpublished: unpublishedInfo.package,
         },
-        _attachments: {}
+        _attachments: {},
       };
       return;
     }
@@ -89,7 +99,7 @@ module.exports = function* list() {
   if (rows.length === 0) {
     if (!this.allowSync) {
       this.status = 404;
-      this.body = {
+      this.jsonp = {
         error: 'not_found',
         reason: 'document not found',
       };
@@ -97,7 +107,7 @@ module.exports = function* list() {
     }
 
     // start sync
-    var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
+    var logId = yield SyncModuleWorker.sync(name, 'sync-by-install');
     debug('start sync %s, get log id %s', name, logId);
 
     return this.redirect(config.officialNpmRegistry + this.url);
@@ -170,12 +180,19 @@ module.exports = function* list() {
     distTags.latest = pkg.version;
   }
 
+  if (!readme && config.enableAbbreviatedMetadata) {
+    var packageReadme = yield packageService.getPackageReadme(name);
+    if (packageReadme) {
+      readme = packageReadme.readme;
+    }
+  }
+
   var info = {
-    _id: orginalName,
+    _id: name,
     _rev: rev,
-    name: orginalName,
+    name: name,
     description: pkg.description,
-    "dist-tags": distTags,
+    'dist-tags': distTags,
     maintainers: pkg.maintainers,
     time: times,
     users: starUsers,
@@ -191,6 +208,118 @@ module.exports = function* list() {
   info.bugs = pkg.bugs;
   info.license = pkg.license;
 
-  debug('show module %s: %s, latest: %s', orginalName, rev, latestMod.version);
+  debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
   this.jsonp = info;
 };
+
+function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows) {
+  debug('show %s got %d rows, %d tags, modifiedTime: %s', name, rows.length, tags.length, modifiedTime);
+  var latestMod = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    var pkg = row.package;
+    common.setDownloadURL(pkg, ctx);
+    pkg._cnpm_publish_time = undefined;
+    pkg.publish_time = undefined;
+
+    versions[pkg.version] = pkg;
+
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+    }
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = latestMod.package.version;
+  }
+
+  var info = {
+    name: name,
+    modified: modifiedTime,
+    'dist-tags': distTags,
+    versions: versions,
+  };
+
+  debug('show %j', info);
+  ctx.jsonp = info;
+}
+
+function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags, rows) {
+  debug('show %s got %d rows, %d tags',
+    name, rows.length, tags.length);
+  var latestMod = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    // pkg is string ... ignore it
+    if (typeof row.package === 'string') {
+      continue;
+    }
+    // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+    var pkg = {
+      name: row.package.name,
+      version: row.package.version,
+      dependencies: row.package.dependencies,
+      optionalDependencies: row.package.optionalDependencies,
+      devDependencies: row.package.devDependencies,
+      bundleDependencies: row.package.bundleDependencies,
+      peerDependencies: row.package.peerDependencies,
+      bin: row.package.bin,
+      directories: row.package.directories,
+      dist: row.package.dist,
+      engines: row.package.engines,
+      _hasShrinkwrap: row.package._hasShrinkwrap,
+      _publish_on_cnpm: row.package._publish_on_cnpm,
+    };
+    common.setDownloadURL(pkg, ctx);
+
+    versions[pkg.version] = pkg;
+
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+    }
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = latestMod.package.version;
+  }
+
+  var info = {
+    name: name,
+    modified: modifiedTime,
+    'dist-tags': distTags,
+    versions: versions,
+  };
+
+  debug('show %j', info);
+  ctx.jsonp = info;
+}

controllers/registry/package/list_all.js

@@ -21,7 +21,7 @@ var packageService = require('../../../services/package');
 // https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L86
 module.exports = function* () {
   var updated = Date.now();
-  var names = yield* packageService.listAllPublicModuleNames();
+  var names = yield packageService.listAllPublicModuleNames();
   var result = { _updated: updated };
   names.forEach(function (name) {
     result[name] = true;

controllers/sync_module_worker.js

@@ -292,6 +292,19 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return setImmediate(this.finish.bind(this));
   }
 
+  // try to sync from official replicate when source npm registry is not cnpmjs.org
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+
+  let versions = yield this.syncByName(concurrencyId, name, registry);
+  if (versions && versions.length === 0 && registry === config.officialNpmReplicate) {
+    // need to sync sourceNpmRegistry also
+    // make sure package data be update event replicate down.
+    // https://github.com/npm/registry/issues/129
+    versions = yield this.syncByName(concurrencyId, name, config.officialNpmRegistry);
+  }
+};
+
+SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry) {
   var that = this;
   that.syncingNames[name] = true;
   var pkg = null;
@@ -309,8 +322,6 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
 
   // get from npm
   const packageUrl = '/' + name.replace('/', '%2f');
-  // try to sync from official replicate when source npm registry is not cnpmjs.org
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
   try {
     var result = yield npmSerivce.request(packageUrl, { registry: registry });
     pkg = result.data;
@@ -326,6 +337,26 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     }
   }
 
+  if (status === 404 && pkg && pkg.reason === 'deleted' && registry === config.officialNpmReplicate) {
+    // unpublished package on replicate.npmjs.com
+    // 404 { error: 'not_found', reason: 'deleted' }
+    // try to read from registry.npmjs.com and get the whole unpublished info
+    try {
+      var result = yield npmSerivce.request(packageUrl, { registry: config.sourceNpmRegistry });
+      pkg = result.data;
+      status = result.status;
+    } catch (err) {
+      // if 404
+      if (!err.res || err.res.statusCode !== 404) {
+        var errMessage = err.name + ': ' + err.message;
+        that.log('[c#%s] [error] [%s] get package(%s%s) error: %s, status: %s',
+          concurrencyId, name, config.sourceNpmRegistry, packageUrl, errMessage, status);
+        yield that._doneOne(concurrencyId, name, false);
+        return;
+      }
+    }
+  }
+
   var unpublishedInfo = null;
   if (status === 404) {
     // check if it's unpublished
@@ -335,17 +366,25 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     } else {
       pkg = null;
     }
+  } else {
+    // unpublished package status become to 200
+    if (name.length < 80 && pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+      unpublishedInfo = pkg.time.unpublished;
+    }
   }
 
   if (!pkg) {
     that.log('[c#%s] [error] [%s] get package(%s%s) error: package not exists, status: %s',
       concurrencyId, name, registry, packageUrl, status);
     yield that._doneOne(concurrencyId, name, true);
-    return;
+    // return empty versions, try again on officialNpmRegistry
+    return [];
   }
 
-  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, start...',
-    concurrencyId, name, registry, packageUrl, status, pkg['dist-tags'], pkg.time && pkg.time.modified);
+  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, unpublished: %j, start...',
+    concurrencyId, name, registry, packageUrl, status,
+    pkg['dist-tags'], pkg.time && pkg.time.modified,
+    unpublishedInfo);
 
   if (unpublishedInfo) {
     try {
@@ -375,6 +414,8 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   this.log('[c#%d] [%s] synced success, %d versions: %s',
     concurrencyId, name, versions.length, versions.join(', '));
   yield this._doneOne(concurrencyId, name, true);
+
+  return versions;
 };
 
 function* _listStarUsers(modName) {
@@ -407,14 +448,14 @@ function* _saveNpmUser(username) {
 
 function* _saveMaintainer(modName, username, action) {
   if (action === 'add') {
-    yield* packageService.addPublicModuleMaintainer(modName, username);
+    yield packageService.addPublicModuleMaintainer(modName, username);
   } else if (action === 'remove') {
-    yield* packageService.removePublicModuleMaintainer(modName, username);
+    yield packageService.removePublicModuleMaintainer(modName, username);
   }
 }
 
 SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
-  var mods = yield* packageService.listModulesByName(name);
+  var mods = yield packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
   if (common.isLocalModule(mods)) {
@@ -423,13 +464,16 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
     return [];
   }
 
-  var r = yield* packageService.saveUnpublishedModule(name, unpublishedInfo);
+  var r = yield packageService.saveUnpublishedModule(name, unpublishedInfo);
   this.log('    [%s] save unpublished info: %j to row#%s',
     name, unpublishedInfo, r.id);
   if (mods.length === 0) {
     return;
   }
-  yield [packageService.removeModulesByName(name), packageService.removeModuleTags(name)];
+  yield [
+    packageService.removeModulesByName(name),
+    packageService.removeModuleTags(name),
+  ];
   var keys = [];
   for (var i = 0; i < mods.length; i++) {
     var row = mods[i];
@@ -463,11 +507,13 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     packageService.listModuleTags(name),
     _listStarUsers(name),
     packageService.listPublicModuleMaintainers(name),
+    packageService.listModuleAbbreviatedsByName(name),
   ];
   var moduleRows = result[0];
   var tagRows = result[1];
   var existsStarUsers = result[2];
   var existsNpmMaintainers = result[3];
+  var existsModuleAbbreviateds = result[4];
 
   if (common.isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
@@ -475,6 +521,12 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     return [];
   }
 
+  var missingModuleAbbreviateds = [];
+  var existsModuleAbbreviatedsMap = {};
+  for (var item of existsModuleAbbreviateds) {
+    existsModuleAbbreviatedsMap[item.version] = item;
+  }
+
   hasModules = moduleRows.length > 0;
   var map = {};
   var localVersionNames = [];
@@ -491,6 +543,11 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     localVersionNames.push(r.version);
   }
 
+  var latestVersionPackageReadme = {
+    version: pkg['dist-tags'].latest,
+    readme: pkg.readme,
+  };
+
   var tags = {};
   for (var i = 0; i < tagRows.length; i++) {
     var r = tagRows[i];
@@ -501,6 +558,37 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     tags[r.tag] = r.version;
   }
 
+  // get package AbbreviatedMetadata
+  var abbreviatedMetadatas = {};
+  if (config.enableAbbreviatedMetadata) {
+    var packageUrl = '/' + name.replace('/', '%2f');
+    var result = yield npmSerivce.request(packageUrl, {
+      dataType: 'text',
+      registry: config.sourceNpmRegistry,
+      headers: {
+        Accept: 'application/vnd.npm.install-v1+json',
+      },
+    });
+    if (result.status === 200) {
+      var data;
+      try {
+        data = JSON.parse(result.data);
+      } catch (err) {
+        that.log('  [%s] get abbreviated meta error: %s, headers: %j, %j',
+          name, err, result.headers, result.data);
+      }
+      if (data) {
+        var versions = data && data.versions || {};
+        for (var version in versions) {
+          const item = versions[version];
+          if (item && typeof item._hasShrinkwrap === 'boolean') {
+            abbreviatedMetadatas[version] = { _hasShrinkwrap: item._hasShrinkwrap };
+          }
+        }
+      }
+    }
+  }
+
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
@@ -511,6 +599,11 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   // [[user, 'add or remove'], ...]
   var diffNpmMaintainers = [];
 
+  // [
+  //   { name, version, _hasShrinkwrap }
+  // ]
+  var missingAbbreviatedMetadatas = [];
+
   // find out new maintainers
   var pkgMaintainers = pkg.maintainers || [];
   if (Array.isArray(pkgMaintainers)) {
@@ -584,17 +677,29 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     if (!version || !version.dist || !version.dist.tarball) {
       continue;
     }
-    //patch for readme
-    if (!version.readme) {
-      version.readme = pkg.readme;
+    // remove readme
+    if (config.enableAbbreviatedMetadata) {
+      version.readme = undefined;
+    } else {
+      // patch for readme
+      if (!version.readme) {
+        version.readme = pkg.readme;
+      }
     }
+
     var publish_time = times[v];
     version.publish_time = publish_time ? Date.parse(publish_time) : null;
     if (!version.maintainers || !version.maintainers[0]) {
       version.maintainers = pkg.maintainers;
     }
-    if (exists.package &&
-        exists.package.dist.shasum === version.dist.shasum) {
+
+    var abbreviatedMetadata = abbreviatedMetadatas[version.version];
+
+    if (exists.package && exists.package.dist.shasum === version.dist.shasum) {
+      if (!existsModuleAbbreviatedsMap[exists.package.version]) {
+        missingModuleAbbreviateds.push(exists);
+      }
+
       // * shasum make sure equal
       if ((version.publish_time === exists.publish_time) ||
           (!version.publish_time && exists.publish_time)) {
@@ -609,19 +714,29 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
           });
         }
 
-        if (!exists.package.readme && version.readme) {
-          // * make sure readme exists
-          missingReadmes.push({
-            id: exists.id,
-            readme: version.readme
-          });
+        if (config.enableAbbreviatedMetadata) {
+          // remove readme
+          if (exists.package.readme) {
+            missingReadmes.push({
+              id: exists.id,
+              readme: undefined,
+            });
+          }
+        } else {
+          if (!exists.package.readme && version.readme) {
+            // * make sure readme exists
+            missingReadmes.push({
+              id: exists.id,
+              readme: version.readme,
+            });
+          }
         }
 
         if (version.deprecated && version.deprecated !== exists.package.deprecated) {
           // need to sync deprecated field
           missingDeprecateds.push({
             id: exists.id,
-            deprecated: version.deprecated
+            deprecated: version.deprecated,
           });
         }
         if (exists.package.deprecated && !version.deprecated) {
@@ -631,9 +746,28 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
             deprecated: undefined,
           });
         }
+        // find missing abbreviatedMetadata
+        if (abbreviatedMetadata) {
+          for (var key in abbreviatedMetadata) {
+            if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
+              missingAbbreviatedMetadatas.push(Object.assign({
+                id: exists.id,
+                name: exists.package.name,
+                version: exists.package.version,
+              }, abbreviatedMetadata));
+              break;
+            }
+          }
+        }
+
         continue;
       }
     }
+
+    // set abbreviatedMetadata to version package
+    if (abbreviatedMetadata) {
+      Object.assign(version, abbreviatedMetadata);
+    }
     missingVersions.push(version);
   }
 
@@ -729,7 +863,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   // sync missing tags
   function* syncTag() {
     if (deletedTags.length > 0) {
-      yield* packageService.removeModuleTagsByNames(name, deletedTags);
+      yield packageService.removeModuleTagsByNames(name, deletedTags);
       that.log('  [%s] deleted %d tags: %j',
         name, deletedTags.length, deletedTags);
     }
@@ -778,6 +912,70 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     }
   }
 
+  function* syncModuleAbbreviateds() {
+    if (missingModuleAbbreviateds.length === 0) {
+      return;
+    }
+    that.log('  [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length);
+
+    var res = yield gather(missingModuleAbbreviateds.map(function (item) {
+      return packageService.saveModuleAbbreviated(item);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingModuleAbbreviateds[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save moduleAbbreviateds error, module: %s@%s, error: %s', item.name, item.version, r.error.message);
+      } else {
+        that.log('    saved moduleAbbreviateds, module: %s@%s', item.name, item.version);
+      }
+    }
+  }
+
+  function* syncAbbreviatedMetadatas() {
+    if (missingAbbreviatedMetadatas.length === 0) {
+      return;
+    }
+    that.log('  [%s] saving %d abbreviated meta datas', name, missingAbbreviatedMetadatas.length);
+
+    var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
+      return packageService.updateModuleAbbreviatedPackage(item);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingAbbreviatedMetadatas[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, module_abbreviated: %s@%s, error: %s',
+          item.name, item.version, r.error.stack);
+      } else {
+        that.log('    saved, module_abbreviated: %s@%s, %j', item.name, item.version, item);
+      }
+    }
+
+    var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
+      var fields = {};
+      for (var key in item) {
+        if (key === 'id' || key === 'name' || key === 'version') {
+          continue;
+        }
+        fields[key] = item[key];
+      }
+      return packageService.updateModulePackageFields(item.id, fields);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingAbbreviatedMetadatas[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, module id: %s, error: %s', item.id, r.error.stack);
+      } else {
+        that.log('    saved, module id: %s, %j', item.id, item);
+      }
+    }
+  }
+
   function *syncDeprecateds() {
     if (missingDeprecateds.length === 0) {
       return;
@@ -879,15 +1077,26 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     }
   }
 
-  yield [
-    syncDes(),
-    syncTag(),
-    syncReadme(),
-    syncDeprecateds(),
-    syncMissingStarUsers(),
-    syncMissingUsers(),
-    syncNpmPackageMaintainers(),
-  ];
+  if (latestVersionPackageReadme.version && latestVersionPackageReadme.readme) {
+    var existsPackageReadme = yield packageService.getPackageReadme(name, true);
+    if (!existsPackageReadme ||
+        existsPackageReadme.version !== latestVersionPackageReadme.version ||
+        existsPackageReadme.readme !== latestVersionPackageReadme.readme) {
+      var r = yield packageService.savePackageReadme(name, latestVersionPackageReadme.readme, latestVersionPackageReadme.version);
+      that.log('    save packageReadme: %s %s %s', r.id, r.name, r.version);
+    }
+  }
+
+  yield syncDes();
+  yield syncTag();
+  yield syncReadme();
+  yield syncDeprecateds();
+  yield syncMissingStarUsers();
+  yield syncMissingUsers();
+  yield syncNpmPackageMaintainers();
+  yield syncModuleAbbreviateds();
+  yield syncAbbreviatedMetadatas();
+
   return syncedVersionNames;
 };
 
@@ -1070,14 +1279,21 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
 
     mod.package.dist = dist;
     var r = yield packageService.saveModule(mod);
+    var moduleAbbreviatedId = null;
+    if (config.enableAbbreviatedMetadata) {
+      var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod);
+      moduleAbbreviatedId = moduleAbbreviatedResult.id;
+    }
 
     that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, '
-      + 'size: %d, publish_time: %j, publish on cnpm: %s',
+      + 'size: %d, publish_time: %j, publish on cnpm: %s, '
+      + 'moduleAbbreviatedId: %s',
       sourcePackage.name, versionIndex,
       r.id,
       author, mod.version, dataSize,
       new Date(mod.publish_time),
-      that._publish);
+      that._publish,
+      moduleAbbreviatedId);
 
     return r;
   }

controllers/web/package/show.js

@@ -36,7 +36,7 @@ module.exports = function* show(next) {
   var pkg = yield packageService[getPackageMethod].apply(packageService, getPackageArgs);
   if (!pkg || !pkg.package) {
     // check if unpublished
-    var unpublishedInfo = yield* packageService.getUnpublishedModule(name);
+    var unpublishedInfo = yield packageService.getUnpublishedModule(name);
     debug('show unpublished %j', unpublishedInfo);
     if (unpublishedInfo) {
       var data = {
@@ -59,7 +59,7 @@ module.exports = function* show(next) {
       return;
     }
 
-    return yield* next;
+    return yield next;
   }
 
   var r = yield [
@@ -76,6 +76,12 @@ module.exports = function* show(next) {
   pkg.package.fromNow = moment(pkg.publish_time).fromNow();
   pkg = pkg.package;
   pkg.users = users;
+  if (!pkg.readme && config.enableAbbreviatedMetadata) {
+    var packageReadme = yield packageService.getPackageReadme(name);
+    if (packageReadme) {
+      pkg.readme = packageReadme.readme;
+    }
+  }
   if (pkg.readme && typeof pkg.readme !== 'string') {
     pkg.readme = 'readme is not string: ' + JSON.stringify(pkg.readme);
   } else {

docs/db.sql

@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS `user` (
  `roles` varchar(200) NOT NULL DEFAULT '[]',
  `rev` varchar(40) NOT NULL,
  `email` varchar(400) NOT NULL,
- `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+ `json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details',
  `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
  PRIMARY KEY (`id`),
  UNIQUE KEY `name` (`name`),
@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 -- ALTER TABLE `user`
 --   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
 --   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
+-- ALTER TABLE `user` CHANGE `json` `json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details';
 
 CREATE TABLE IF NOT EXISTS `module_keyword` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
@@ -86,6 +87,32 @@ CREATE TABLE IF NOT EXISTS `module` (
 -- show create table module\G
 -- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
+CREATE TABLE IF NOT EXISTS `module_abbreviated` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `version` varchar(30) NOT NULL COMMENT 'module version',
+ `package` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the abbreviated metadata',
+ `publish_time` bigint(20) unsigned,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`name`,`version`),
+ KEY `gmt_modified` (`gmt_modified`),
+ KEY `publish_time` (`publish_time`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module abbreviated info';
+
+CREATE TABLE IF NOT EXISTS `package_readme` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `readme` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the latest version readme',
+ `version` varchar(30) NOT NULL COMMENT 'module version',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='package latest readme';
+
 CREATE TABLE IF NOT EXISTS `module_log` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',

lib/common.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - lib/common.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var crypto = require('crypto');
 var path = require('path');
 var config = require('../config');

models/index.js

@@ -1,24 +1,16 @@
-/**!
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var path = require('path');
+var config = require('../config');
 var sequelize = require('../common/sequelize');
 
 function load(name) {
   return sequelize.import(path.join(__dirname, name));
 }
 
+var _ModuleAbbreviated = config.enableAbbreviatedMetadata ? load('module_abbreviated') : null;
+var _PackageReadme = config.enableAbbreviatedMetadata ? load('package_readme') : null;
+
 module.exports = {
   sequelize: sequelize,
   Module: load('module'),
@@ -44,7 +36,27 @@ module.exports = {
     return data[1];
   },
   queryOne: function* (sql, args) {
-    var rows = yield* this.query(sql, args);
+    var rows = yield this.query(sql, args);
     return rows && rows[0];
-  }
+  },
+
+  get ModuleAbbreviated() {
+    if (!config.enableAbbreviatedMetadata) {
+      return null;
+    }
+    if (!_ModuleAbbreviated) {
+      _ModuleAbbreviated = load('module_abbreviated');
+    }
+    return _ModuleAbbreviated;
+  },
+
+  get PackageReadme() {
+    if (!config.enableAbbreviatedMetadata) {
+      return null;
+    }
+    if (!_PackageReadme) {
+      _PackageReadme = load('package_readme');
+    }
+    return _PackageReadme;
+  },
 };

models/init_script.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - models/init_script.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var config = require('../config');
 
 config.database.logging = console.log;

models/module_abbreviated.js

@@ -0,0 +1,46 @@
+'use strict';
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleAbbreviated', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name'
+    },
+    version: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'module version'
+    },
+    package: {
+      type: DataTypes.LONGTEXT,
+      comment: 'package.json',
+    },
+    publish_time: {
+      type: DataTypes.BIGINT(20),
+      allowNull: true,
+    }
+  }, {
+    tableName: 'module_abbreviated',
+    comment: 'module abbreviated info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name', 'version']
+      },
+      {
+        fields: ['gmt_modified']
+      },
+      {
+        fields: ['publish_time']
+      },
+    ],
+    classMethods: {
+      findByNameAndVersion: function* (name, version) {
+        return yield this.find({
+          where: { name: name, version: version }
+        });
+      }
+    }
+  });
+};

models/package_readme.js

@@ -0,0 +1,39 @@
+'use strict';
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('PackageReadme', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name'
+    },
+    version: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'module latest version'
+    },
+    readme: {
+      type: DataTypes.LONGTEXT,
+      comment: 'latest version readme',
+    },
+  }, {
+    tableName: 'package_readme',
+    comment: 'package latest readme',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name']
+      },
+      {
+        fields: ['gmt_modified']
+      },
+    ],
+    classMethods: {
+      findByName: function* (name) {
+        return yield this.find({
+          where: { name: name },
+        });
+      }
+    }
+  });
+};

package.json

@@ -1,11 +1,12 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.19.2",
+  "version": "3.0.0-alpha.3",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
     "dev": "DEBUG=cnpm* node dispatch.js",
     "test": "make jshint && make test",
+    "test-local": "make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
     "stop": "./bin/nodejsctl stop"
@@ -64,6 +65,7 @@
     "autod": "*",
     "chunkstream": "*",
     "contributors": "*",
+    "intelli-espower-loader": "^1.0.1",
     "istanbul": "*",
     "jshint": "*",
     "mm": "*",
@@ -72,8 +74,8 @@
     "pedding": "*",
     "pg": "5",
     "pg-hstore": "2",
+    "power-assert": "^1.4.2",
     "should": "8",
-    "should-http": "*",
     "sqlite3": "*",
     "supertest": "2",
     "thunk-mocha": "1"

services/npm.js

@@ -11,9 +11,9 @@ function* request(url, options) {
   options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
-  options.headers = {
+  options.headers = Object.assign({
     'user-agent': USER_AGENT
-  };
+  }, options.headers);
   options.gzip = true;
   options.followRedirect = true;
   var registry = options.registry || config.sourceNpmRegistry;
@@ -52,7 +52,7 @@ exports.getUser = function* (name) {
 };
 
 exports.get = function* (name) {
-  var r = yield* request('/' + name);
+  var r = yield request('/' + name);
   var data = r.data;
   if (r.status === 404) {
     data = null;

services/package.js

@@ -3,6 +3,7 @@
 var semver = require('semver');
 var models = require('../models');
 var common = require('./common');
+var config = require('../config');
 var Tag = models.Tag;
 var User = models.User;
 var Module = models.Module;
@@ -49,17 +50,17 @@ exports.getModuleById = function* (id) {
 };
 
 exports.getModule = function* (name, version) {
-  var row = yield* Module.findByNameAndVersion(name, version);
+  var row = yield Module.findByNameAndVersion(name, version);
   parseRow(row);
   return row;
 };
 
 exports.getModuleByTag = function* (name, tag) {
-  var tag = yield* Tag.findByNameAndTag(name, tag);
+  var tag = yield Tag.findByNameAndTag(name, tag);
   if (!tag) {
     return null;
   }
-  return yield* exports.getModule(tag.name, tag.version);
+  return yield exports.getModule(tag.name, tag.version);
 };
 
 exports.getModuleByRange = function* (name, range) {
@@ -82,7 +83,7 @@ exports.getModuleByRange = function* (name, range) {
 };
 
 exports.getLatestModule = function* (name) {
-  return yield* exports.getModuleByTag(name, 'latest');
+  return yield exports.getModuleByTag(name, 'latest');
 };
 
 // module:list
@@ -195,7 +196,7 @@ exports.listPublicModuleNamesByUser = function* (username) {
   });
 
   // find from npm module maintainer table
-  var moduleNames = yield* NpmModuleMaintainer.listModuleNamesByUser(username);
+  var moduleNames = yield NpmModuleMaintainer.listModuleNamesByUser(username);
   moduleNames.forEach(function (name) {
     if (!map[name]) {
       names.push(name);
@@ -275,7 +276,7 @@ exports.saveModule = function* (mod) {
   // dist.shasum = '';
   // dist.size = 0;
   var publish_time = mod.publish_time || Date.now();
-  var item = yield* Module.findByNameAndVersion(mod.name, mod.version);
+  var item = yield Module.findByNameAndVersion(mod.name, mod.version);
   if (!item) {
     item = Module.build({
       name: mod.name,
@@ -316,12 +317,65 @@ exports.saveModule = function* (mod) {
 
   if (words.length > 0) {
     // add keywords
-    yield* exports.addKeywords(mod.name, description, words);
+    yield exports.addKeywords(mod.name, description, words);
   }
 
   return result;
 };
 
+exports.listModuleAbbreviatedsByName = function* (name) {
+  var rows = yield models.ModuleAbbreviated.findAll({
+    where: {
+      name: name
+    },
+    order: [ ['id', 'DESC'] ],
+  });
+
+  for (var row of rows) {
+    row.package = JSON.parse(row.package);
+  }
+  return rows;
+};
+
+// https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+exports.saveModuleAbbreviated = function* (mod) {
+  var pkg = JSON.stringify({
+    name: mod.package.name,
+    version: mod.package.version,
+    dependencies: mod.package.dependencies,
+    optionalDependencies: mod.package.optionalDependencies,
+    devDependencies: mod.package.devDependencies,
+    bundleDependencies: mod.package.bundleDependencies,
+    peerDependencies: mod.package.peerDependencies,
+    bin: mod.package.bin,
+    directories: mod.package.directories,
+    dist: mod.package.dist,
+    engines: mod.package.engines,
+    _hasShrinkwrap: mod.package._hasShrinkwrap,
+    _publish_on_cnpm: mod.package._publish_on_cnpm,
+  });
+  var publish_time = mod.publish_time || Date.now();
+  var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version);
+  if (!item) {
+    item = models.ModuleAbbreviated.build({
+      name: mod.name,
+      version: mod.version,
+    });
+  }
+  item.publish_time = publish_time;
+  item.package = pkg;
+
+  if (item.changed()) {
+    item = yield item.save();
+  }
+  var result = {
+    id: item.id,
+    gmt_modified: item.gmt_modified,
+  };
+
+  return result;
+};
+
 exports.updateModulePackage = function* (id, pkg) {
   var mod = yield Module.findById(Number(id));
   if (!mod) {
@@ -344,18 +398,36 @@ exports.updateModulePackageFields = function* (id, fields) {
   return yield exports.updateModulePackage(id, pkg);
 };
 
+exports.updateModuleAbbreviatedPackage = function* (item) {
+  // item => { id, name, version, _hasShrinkwrap }
+  var mod = yield models.ModuleAbbreviated.findByNameAndVersion(item.name, item.version);
+  if (!mod) {
+    return null;
+  }
+  var pkg = JSON.parse(mod.package);
+  for (var key in item) {
+    if (key === 'name' || key === 'version' || key === 'id') {
+      continue;
+    }
+    pkg[key] = item[key];
+  }
+  mod.package = JSON.stringify(pkg);
+
+  return yield mod.save([ 'package' ]);
+};
+
 exports.updateModuleReadme = function* (id, readme) {
-  var mod = yield* exports.getModuleById(id);
+  var mod = yield exports.getModuleById(id);
   if (!mod) {
     return null;
   }
   var pkg = mod.package || {};
   pkg.readme = readme;
-  return yield* exports.updateModulePackage(id, pkg);
+  return yield exports.updateModulePackage(id, pkg);
 };
 
 exports.updateModuleDescription = function* (id, description) {
-  var mod = yield* exports.getModuleById(id);
+  var mod = yield exports.getModuleById(id);
   if (!mod) {
     return null;
   }
@@ -381,19 +453,62 @@ exports.updateModuleLastModified = function* (name) {
   return yield row.save();
 };
 
+// try to return latest version readme
+exports.getPackageReadme = function* (name, onlyPackageReadme) {
+  if (config.enableAbbreviatedMetadata) {
+    var row = yield models.PackageReadme.findByName(name);
+    if (row) {
+      return {
+        version: row.version,
+        readme: row.readme,
+      };
+    }
+    if (onlyPackageReadme) {
+      return;
+    }
+  }
+
+  var mod = yield exports.getLatestModule(name);
+  if (mod) {
+    return {
+      version: mod.package.version,
+      readme: mod.package.readme,
+    };
+  }
+};
+
+exports.savePackageReadme = function* (name, readme, latestVersion) {
+  var item = yield models.PackageReadme.find({ where: { name: name } });
+  if (!item) {
+    item = models.PackageReadme.build({
+      name: name,
+    });
+  }
+  item.readme = readme;
+  item.version = latestVersion;
+  return yield item.save();
+};
+
 exports.removeModulesByName = function* (name) {
   yield Module.destroy({
     where: {
-      name: name
-    }
+      name: name,
+    },
   });
+  if (config.enableAbbreviatedMetadata) {
+    yield models.ModuleAbbreviated.destroy({
+      where: {
+        name: name,
+      },
+    });
+  }
 };
 
 exports.removeModulesByNameAndVersions = function* (name, versions) {
   yield Module.destroy({
     where: {
       name: name,
-      version: versions
+      version: versions,
     }
   });
 };
@@ -401,12 +516,12 @@ exports.removeModulesByNameAndVersions = function* (name, versions) {
 // tags
 
 exports.addModuleTag = function* (name, tag, version) {
-  var mod = yield* exports.getModule(name, version);
+  var mod = yield exports.getModule(name, version);
   if (!mod) {
     return null;
   }
 
-  var row = yield* Tag.findByNameAndTag(name, tag);
+  var row = yield Tag.findByNameAndTag(name, tag);
   if (!row) {
     row = Tag.build({
       name: name,

sync/index.js

@@ -148,7 +148,8 @@ function sendMailToAdmin(err, result, syncTime) {
   if (err) {
     // ignore 503, 504 error
     // 504: Gateway Time-out
-    if (err.status === 503 || err.status === 504) {
+    // 502 Bad Gateway
+    if (err.status === 503 || err.status === 504 || err.status === 502) {
       return;
     }
     if (err.name === 'JSONResponseFormatError') {

test/controllers/registry/package/list.test.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var should = require('should');
+var assert = require('assert');
 var request = require('supertest');
 var mm = require('mm');
 var pedding = require('pedding');
@@ -57,6 +58,28 @@ describe('test/controllers/registry/package/list.test.js', () => {
     });
   });
 
+  it('should return all versions in abbreviated meta format for private scope package', function(done) {
+    request(app.listen())
+    .get('/@cnpmtest/testmodule-list-1')
+    .set('Accept', 'application/vnd.npm.install-v1+json')
+    .expect(200, function(err, res) {
+      should.not.exist(err);
+      var data = res.body;
+      assert(data.name === '@cnpmtest/testmodule-list-1');
+      assert.deepEqual(Object.keys(data.versions), ['1.0.0', '0.0.1']);
+      assert(data.modified);
+      assert.deepEqual(data['dist-tags'], { latest: '1.0.0' });
+      assert(!data.time);
+
+      // should 304
+      request(app.listen())
+      .get('/@cnpmtest/testmodule-list-1')
+      .set('Accept', 'application/vnd.npm.install-v1+json')
+      .set('If-None-Match', res.headers.etag)
+      .expect(304, done);
+    });
+  });
+
   it('should show star users', function (done) {
     mm(packageService, 'listStarUserNames', function* () {
       return ['fengmk2', 'foouser'];
@@ -153,13 +176,8 @@ describe('test/controllers/registry/package/list.test.js', () => {
       .expect(200, function (err, res) {
         should.not.exist(err);
         var data = res.body;
-        data.name.should.equal('tair');
-        data.maintainers.should.eql([
-          {
-            email: 'kate.sf@taobao.com',
-            name: 'sunfang1cn'
-          }
-        ]);
+        assert(data.name === 'tair');
+        assert(data.maintainers);
         done();
       });
     });
@@ -193,4 +211,87 @@ describe('test/controllers/registry/package/list.test.js', () => {
       });
     });
   });
+
+  describe('list AbbreviatedMeta', () => {
+    before(done => {
+      mm(config, 'sourceNpmRegistry', config.officialNpmRegistry);
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      utils.sync('detect-port', done);
+    });
+
+    it('should return abbreviated meta when Accept: application/vnd.npm.install-v1+json', () => {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      return request(app.listen())
+        .get('/detect-port')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200)
+        .expect(res => {
+          const data = res.body;
+          assert(data.name === 'detect-port');
+          assert(data.modified);
+          assert(data['dist-tags'].latest);
+          assert(Object.keys(data.versions).length > 0);
+          for (const v in data.versions) {
+            assert('_hasShrinkwrap' in data.versions[v]);
+          }
+        });
+    });
+
+    it('should 404 when package not exists', () => {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      return request(app.listen())
+        .get('/@cnpmtest/not-exists-package')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(404)
+        .expect({
+          error: 'not_found',
+          reason: 'document not found'
+        });
+    });
+
+    it('should return full meta when enableAbbreviatedMetadata is false and Accept is application/vnd.npm.install-v1+json', () => {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', false);
+      return request(app.listen())
+        .get('/detect-port')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200)
+        .expect(res => {
+          const data = res.body;
+          assert(data.name === 'detect-port');
+          assert(data.description);
+          assert(data.maintainers);
+          // assert(data.readme);
+          assert(data['dist-tags'].latest);
+          assert(Object.keys(data.versions).length > 0);
+          for (const v in data.versions) {
+            assert('_hasShrinkwrap' in data.versions[v]);
+          }
+        });
+    });
+
+    it('should return full meta when Accept is not application/vnd.npm.install-v1+json', () => {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      return request(app.listen())
+        .get('/detect-port')
+        .set('Accept', 'application/json')
+        .expect(200)
+        .expect(res => {
+          const data = res.body;
+          assert(data.name === 'detect-port');
+          assert(data.description);
+          assert(data.readme);
+          assert(data.maintainers);
+          assert(data['dist-tags'].latest);
+          assert(Object.keys(data.versions).length > 0);
+          for (const v in data.versions) {
+            assert('_hasShrinkwrap' in data.versions[v]);
+          }
+        });
+    });
+  });
 });

test/controllers/sync_module_worker.test.js

@@ -1,5 +1,6 @@
 'use strict';
 
+var assert = require('assert');
 var should = require('should');
 var mm = require('mm');
 var thunkify = require('thunkify-wrap');
@@ -13,10 +14,10 @@ var utils = require('../utils');
 var app = require('../../servers/registry');
 var User = require('../../models').User;
 
-describe('test/controllers/sync_module_worker.test.js', function () {
+describe('test/controllers/sync_module_worker.test.js', () => {
   afterEach(mm.restore);
 
-  beforeEach(function () {
+  beforeEach(() => {
     mm(config, 'syncModel', 'all');
     mm(config, 'sourceNpmRegistryIsCNpm', false);
     mm(config, 'privatePackages', ['google']);
@@ -89,12 +90,12 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     yield checkResult();
 
     var r = yield urllib.request(tgzUrl);
-    console.log(r.status, r.headers);
+    // console.log(r.status, r.headers);
     r.status.should.equal(200);
   });
 
   it('should start a sync worker and dont sync deps', function* () {
-    var log = yield* logService.create({
+    var log = yield logService.create({
       name: 'byte',
       username: 'fengmk2',
     });
@@ -154,7 +155,8 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['mk2testmodule']);
+      assert(names.length >= 1 && names.length <= 2);
+      // names.should.eql(['mk2testmodule', 'mk2testmodule']);
       done();
     });
   });
@@ -181,7 +183,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
 
   it('should sync unpublished info', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['tnpm'],
+      name: ['afp'],
       username: 'fengmk2'
     });
 
@@ -189,7 +191,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['tnpm']);
+      names.should.eql([ 'afp' ]);
       done();
     });
   });

test/controllers/web/package/show.test.js

@@ -36,7 +36,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -52,7 +52,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -181,7 +181,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -213,7 +213,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -245,7 +245,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -277,7 +277,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org\.git/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -309,7 +309,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -341,10 +341,10 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/http:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
     });
   });
-});
+});

test/init.js

@@ -1,9 +1,5 @@
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var config = require('../config');
 
 if (process.env.DB) {

test/init_db.js

@@ -1,9 +1,5 @@
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var crypto = require('crypto');
 var path = require('path');
 var childProcess = require('child_process');

test/sync/sync_all.test.js

@@ -1,19 +1,6 @@
-/*!
- * cnpmjs.org - test/sync/sync_all.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
+var assert = require('assert');
 var mm = require('mm');
 var config = require('../../config');
 var sync = require('../../sync/sync_all');
@@ -21,14 +8,14 @@ var npmSerivce = require('../../services/npm');
 var totalService = require('../../services/total');
 var packageService = require('../../services/package');
 
-describe('test/sync/sync_all.test.js', function () {
-  beforeEach(function () {
+describe('test/sync/sync_all.test.js', () => {
+  beforeEach(() => {
     mm(config, 'syncModel', 'all');
   });
 
   afterEach(mm.restore);
 
-  describe('sync()', function () {
+  describe('sync()', () => {
     it('should sync first time ok', function* () {
       mm.data(npmSerivce, 'getShort', ['mk2testmodule', 'mk2testmodule-not-exists']);
       mm.data(totalService, 'getTotalInfo', {last_sync_time: 0});
@@ -45,7 +32,8 @@ describe('test/sync/sync_all.test.js', function () {
       mm.data(totalService, 'getTotalInfo', {last_sync_time: Date.now()});
       mm.data(packageService, 'listAllPublicModuleNames', [ 'mk2testmodule' ]);
       var data = yield sync;
-      data.successes.should.eql(['mk2testmodule']);
+      assert(data.successes.length >= 1 && data.successes.length <= 2);
+      // data.successes.should.eql(['mk2testmodule']);
       mm.restore();
     });
 
@@ -71,7 +59,8 @@ describe('test/sync/sync_all.test.js', function () {
       mm.data(totalService, 'getTotalInfo', {last_sync_time: Date.now()});
       mm.data(packageService, 'listAllPublicModuleNames', [ 'mk2testmodule' ]);
       var data = yield sync;
-      data.successes.should.eql(['mk2testmodule']);
+      assert(data.successes.length >= 1 && data.successes.length <= 2);
+      // data.successes.should.eql(['mk2testmodule']);
       mm.restore();
     });
   });

test/sync/sync_exist.test.js

@@ -36,8 +36,8 @@ describe('test/sync/sync_exist.test.js', function () {
     it('should sync first time ok', function *() {
       mm.data(npmService, 'getShort', ['byte']);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: 0});
-      var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      var data = yield sync();
+      data.successes[0].should.equal('byte');
     });
 
     it('should sync common ok', function *() {
@@ -47,7 +47,7 @@ describe('test/sync/sync_exist.test.js', function () {
       });
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      data.successes[0].should.equal('byte');
 
       mm.data(npmService, 'getAllSince', []);
       var data = yield* sync();
@@ -62,7 +62,7 @@ describe('test/sync/sync_exist.test.js', function () {
       ]);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      data.successes[0].should.equal('byte');
 
       mm.data(npmService, 'getAllSince', []);
       var data = yield* sync();

test/sync/sync_popular.test.js

@@ -1,36 +1,24 @@
-/**!
- * cnpmjs.org - test/sync/sync_popular.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
+var assert = require('assert');
 var mm = require('mm');
 var config = require('../../config');
 var npmService = require('../../services/npm');
 var syncPopular = require('../../sync/sync_popular');
 
-describe('sync/sync_popular.test.js', function () {
-  beforeEach(function () {
+describe('sync/sync_popular.test.js', () => {
+  beforeEach(() => {
     mm(config, 'syncModel', 'all');
   });
 
   afterEach(mm.restore);
 
-  describe('sync()', function () {
+  describe('sync()', () => {
     it('should sync popular modules ok', function* () {
       mm.data(npmService, 'getPopular', [['mk2testmodule', 1001]]);
-      var data = yield* syncPopular();
-      data.successes.should.eql(['mk2testmodule']);
+      var data = yield syncPopular();
+      assert(data.successes.length >= 1 && data.successes.length <= 2);
+      // data.successes.should.eql(['mk2testmodule']);
       mm.restore();
     });
   });