fix: use koa-bodyparser instead of koa-middlewares

https://snyk.io/vuln/npm:qs:20170213

.travis.yml

@@ -3,6 +3,7 @@ language: node_js
 node_js:
   - '4'
   - '6'
+  - '7'
 addons:
   - postgresql: '9.3'
 script: 'make test-travis-all'

AUTHORS

@@ -7,5 +7,6 @@ alsotang <alsotang@gmail.com> (https://github.com/alsotang)
 afc163 <afc163@gmail.com> (https://github.com/afc163)
 Yuwei Ba <i@xiaoba.me> (https://github.com/ibigbug)
 dickeylth <dickeylth@gmail.com> (https://github.com/dickeylth)
-
+wenbing <wenbing@users.noreply.github.com> (https://github.com/wenbing)
+21paradox <1036339815@qq.com> (https://github.com/21paradox)
 

History.md

@@ -1,11 +1,94 @@
 
+2.19.3 / 2017-02-22
+==================
+
+  * fix: should get package from orginal registry when package is unpublished (#1130)
+
+2.19.2 / 2017-01-05
+==================
+
+  * fix: should auto sync un-deprecate message (#1105)
+
+2.19.1 / 2016-12-29
+==================
+
+  * fix: try to use the best repository url (#1102)
+
+2.19.0 / 2016-12-21
+==================
+
+  * feat: keyword search with limit to support keywords > 100 (#1097)
+
+2.18.0 / 2016-12-05
+==================
+
+  * fix: support downloads total on scope package (#1088)
+  * fix: try to sync from official replicate (#1076)
+  * feat: add change password script (#1070)
+  * test: skip always fail tests
+  * test: add node v7
+  * feat: show more sync info
+
+2.17.2 / 2016-11-13
+==================
+
+  * fix: ignore long package name on unpublished sync (#1067)
+
+2.17.1 / 2016-11-08
+==================
+
+  * fix: add publish_time for private packages (#1061)
+
+2.17.0 / 2016-11-03
+==================
+
+  * feat: make snyk.io url configable (#1058)
+
+2.16.2 / 2016-09-27
+==================
+
+  * fix: try to use config.registryHost first on setDownloadURL (#1044)
+
+2.16.1 / 2016-08-22
+==================
+
+  * refactor: refine publishable's code (#1022)
+
+2.16.0 / 2016-08-22
+==================
+
+  * feat: admin can do everything (#1021)
+
+2.15.0 / 2016-08-22
+==================
+
+  * feat: return dist-tag on package registry ([#1020](https://github.com/cnpm/cnpmjs.org/issues/1020))
+  * chore(package): update supertest to version 2.0.0 ([#1004](https://github.com/cnpm/cnpmjs.org/issues/1004))
+
+2.14.0 / 2016-08-04
+==================
+
+  * feat: password may contains ":" ([#999](https://github.com/cnpm/cnpmjs.org/issues/999))
+  * fix: limit sync fails email notice ([#1006](https://github.com/cnpm/cnpmjs.org/issues/1006))
+
+2.13.0 / 2016-07-26
+==================
+
+  * feat: enable maxrequests middleware ([#1003](https://github.com/cnpm/cnpmjs.org/issues/1003))
+
+2.12.2 / 2016-07-11
+==================
+
+  * fix: getModuleByRange don't list all packages ([#990](https://github.com/cnpm/cnpmjs.org/issues/990))
+  * fix: should show new version package count ([#984](https://github.com/cnpm/cnpmjs.org/issues/984))
+
 2.12.1 / 2016-07-01
 ==================
 
-  * fix: make sure chagnes stream destroy (#982)
-  * chore(package): update semver to version 5.2.0 (#978)
-  * deps: use ^ instead of ~ (#976)
-  * chore(package): update mini-logger to version 1.1.1 (#973)
+  * fix: make sure chagnes stream destroy ([#982](https://github.com/cnpm/cnpmjs.org/issues/982))
+  * chore(package): update semver to version 5.2.0 ([#978](https://github.com/cnpm/cnpmjs.org/issues/978))
+  * deps: use ^ instead of ~ ([#976](https://github.com/cnpm/cnpmjs.org/issues/976))
+  * chore(package): update mini-logger to version 1.1.1 ([#973](https://github.com/cnpm/cnpmjs.org/issues/973))
 
 2.12.0 / 2016-06-26
 ==================

Makefile

@@ -4,14 +4,7 @@ TIMEOUT = 30000
 MOCHA_OPTS =
 DB = sqlite
 
-install:
-	@npm install --build-from-source --registry=https://registry.npm.taobao.org \
-		--disturl=https://npm.taobao.org/mirrors/node
-
-install-production production:
-	@NODE_ENV=production $(MAKE) install
-
-jshint: install
+jshint:
 	@node_modules/.bin/jshint .
 
 init-database:
@@ -25,12 +18,11 @@ init-pg:
 	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;'
 	@psql -c 'CREATE DATABASE cnpmjs_test;'
 
-test: install init-database
+test: init-database
 	@NODE_ENV=test DB=${DB} node_modules/.bin/mocha \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -47,7 +39,7 @@ test-pg: init-pg
 
 test-all: test-sqlite test-mysql
 
-test-cov cov: install init-database
+test-cov cov: init-database
 	@NODE_ENV=test DB=${DB} node \
 		node_modules/.bin/istanbul cover \
 		node_modules/.bin/_mocha \
@@ -55,7 +47,6 @@ test-cov cov: install init-database
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -67,7 +58,7 @@ test-cov-sqlite:
 test-cov-mysql: init-mysql
 	@$(MAKE) test-cov DB=mysql
 
-test-travis: install init-database
+test-travis: init-database
 	@NODE_ENV=test DB=${DB} CNPM_SOURCE_NPM=https://registry.npmjs.com CNPM_SOURCE_NPM_ISCNPM=false \
 		node \
 		node_modules/.bin/istanbul cover \
@@ -76,7 +67,6 @@ test-travis: install init-database
 		--reporter dot \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -98,15 +88,14 @@ test-travis-all: jshint test-travis-sqlite test-travis-mysql test-travis-pg
 dev:
 	@NODE_ENV=development node node_modules/.bin/node-dev dispatch.js
 
-contributors: install
+contributors:
 	@node_modules/.bin/contributors -f plain -o AUTHORS
 
-autod: install
+autod:
 	@node_modules/.bin/autod -w \
 		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
 		--dep mysql \
-		--keep should,supertest,should-http,chunkstream,mm,pedding
-	@$(MAKE) install
+		--keep should,supertest,chunkstream,mm,pedding
 
 .PHONY: test

bin/change_password.js

@@ -0,0 +1,24 @@
+"use strict";
+// Only support for ./services/DefaultUserService. If you use custom user service, ignore this file.
+// call with:
+// $ node ./bin/change_password.js 'username' 'new_password'
+
+var UserModel = require('../models').User;
+var co = require('co');
+var utility = require('utility');
+
+var username = process.argv[2];
+var newPassword = process.argv[3];
+
+co(function * () {
+  var user = yield UserModel.find({where: {name: username}});
+  var salt = user.salt;
+  console.log(`user original password_sha: ${user.password_sha}`);
+  var newPasswordSha = utility.sha1(newPassword + salt);
+  user.password_sha = newPasswordSha;
+  user = yield user.save();
+  console.log(`change user password successful!! user new password_sha: ${user.password_sha}`);
+  process.exit(0);
+}).catch(function (e) {
+  console.log(e);
+});

config/index.js

@@ -163,6 +163,7 @@ var config = {
   // but sometimes will request it for some package infomations
   // please don't change it if not necessary
   officialNpmRegistry: 'https://registry.npmjs.com',
+  officialNpmReplicate: 'https://replicate.npmjs.com',
 
   // sync source, upstream registry
   // If you want to directly sync from official npm's registry
@@ -217,6 +218,9 @@ var config = {
   // if you're behind firewall, need to request through http proxy, please set this
   // e.g.: `httpProxy: 'http://proxy.mycompany.com:8080'`
   httpProxy: null,
+
+  // snyk.io root url
+  snykUrl: 'https://snyk.io',
 };
 
 if (process.env.NODE_ENV !== 'test') {

controllers/registry/package/dist_tag.js

@@ -1,17 +1,5 @@
-/**!
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var packageService = require('../../../services/package');
 
 function ok() {

controllers/registry/package/download_total.js

@@ -1,25 +1,11 @@
-/*!
- * cnpmjs.org - controllers/registry/package/download_total.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var DownloadTotal = require('../../../services/download_total');
 var DATE_REG = /^\d{4}-\d{2}-\d{2}$/;
 
-module.exports = function* downloadTotal () {
-  var range = this.params.range;
-  var name = this.params.name;
+module.exports = function* downloadTotal() {
+  var range = this.params.range || this.params[0] || '';
+  var name = this.params.name || this.params[1];
 
   range = range.split(':');
   if (range.length !== 2

controllers/registry/package/list.js

@@ -126,6 +126,8 @@ module.exports = function* list() {
     }
     common.setDownloadURL(pkg, this);
     pkg._cnpm_publish_time = row.publish_time;
+    pkg.publish_time = pkg.publish_time || row.publish_time;
+
     versions[pkg.version] = pkg;
 
     var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;

controllers/registry/package/list_since.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - controllers/registry/package/list_since.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var packageService = require('../../../services/package');
 
 var A_WEEK_MS = 3600000 * 24 * 7;
@@ -21,7 +7,7 @@ var A_WEEK_MS = 3600000 * 24 * 7;
 // GET /-/all/since?stale=update_after&startkey={key}
 // List packages names since startkey
 // https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L89
-module.exports = function* () {
+module.exports = function* listSince() {
   var query = this.query;
   if (query.stale !== 'update_after') {
     this.status = 400;
@@ -49,7 +35,7 @@ module.exports = function* () {
       Date(), query, this.ip);
   }
 
-  var names = yield* packageService.listPublicModuleNamesSince(startkey);
+  var names = yield packageService.listPublicModuleNamesSince(startkey);
   var result = { _updated: updated };
   names.forEach(function (name) {
     result[name] = true;

controllers/registry/package/show.js

@@ -1,17 +1,5 @@
-/**!
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:show');
 var semver = require('semver');
 var packageService = require('../../../services/package');
@@ -47,10 +35,23 @@ module.exports = function* show() {
   if (mod) {
     setDownloadURL(mod.package, this);
     mod.package._cnpm_publish_time = mod.publish_time;
-    var maintainers = yield* packageService.listMaintainers(name);
+    mod.package.publish_time = mod.package.publish_time || mod.publish_time;
+    var rs = yield [
+      packageService.listMaintainers(name),
+      packageService.listModuleTags(name),
+    ];
+    var maintainers = rs[0];
     if (maintainers.length > 0) {
       mod.package.maintainers = maintainers;
     }
+    var tags = rs[1];
+    var distTags = {};
+    for (var i = 0; i < tags.length; i++) {
+      var t = tags[i];
+      distTags[t.tag] = t.version;
+    }
+    // show tags for npminstall faster download
+    mod.package['dist-tags'] = distTags;
     this.jsonp = mod.package;
     return;
   }

controllers/sync_module_worker.js

@@ -59,6 +59,7 @@ function SyncModuleWorker(options) {
 
   this.successes = [];
   this.fails = [];
+  this.updates = [];
 }
 
 util.inherits(SyncModuleWorker, EventEmitter);
@@ -127,7 +128,7 @@ SyncModuleWorker.prototype.start = function () {
     // sync upstream
     if (that.syncUpstreamFirst) {
       try {
-        yield* that.syncUpstream(that.startName);
+        yield that.syncUpstream(that.startName);
       } catch (err) {
         logger.error(err);
       }
@@ -180,14 +181,16 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
   var that = this;
   // relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
   defer.setImmediate(function* () {
-    yield* that.next(concurrencyId);
+    yield that.next(concurrencyId);
   });
 };
 
 SyncModuleWorker.prototype.syncUpstream = function* (name) {
   if (config.sourceNpmRegistry.indexOf('registry.npmjs.org') >= 0 ||
-      config.sourceNpmRegistry.indexOf('registry.npmjs.com') >= 0) {
-    this.log('----------------- upstream is npm registry: %s, ignore it -------------------', config.sourceNpmRegistry);
+      config.sourceNpmRegistry.indexOf('registry.npmjs.com') >= 0 ||
+      config.sourceNpmRegistry.indexOf('replicate.npmjs.com') >= 0) {
+    this.log('----------------- upstream is npm registry: %s, ignore it -------------------',
+      config.sourceNpmRegistry);
     return;
   }
   var syncname = name;
@@ -300,72 +303,116 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   if (common.isPrivateScopedPackage(name)) {
     this.log('[c#%d] [%s] ignore sync private scoped %j package',
       concurrencyId, name, config.scopes);
-    yield* this._doneOne(concurrencyId, name, true);
+    yield this._doneOne(concurrencyId, name, true);
     return;
   }
 
   // get from npm
+  const packageUrl = '/' + name.replace('/', '%2f');
+  // try to sync from official replicate when source npm registry is not cnpmjs.org
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
   try {
-    var result = yield* npmSerivce.request('/' + name.replace('/', '%2f'));
+    var result = yield npmSerivce.request(packageUrl, { registry: registry });
     pkg = result.data;
     status = result.status;
   } catch (err) {
     // if 404
     if (!err.res || err.res.statusCode !== 404) {
       var errMessage = err.name + ': ' + err.message;
-      that.log('[c#%s] [error] [%s] get package error: %s, status: %s',
-        concurrencyId, name, errMessage, status);
-      yield *that._doneOne(concurrencyId, name, false);
+      that.log('[c#%s] [error] [%s] get package(%s%s) error: %s, status: %s',
+        concurrencyId, name, registry, packageUrl, errMessage, status);
+      yield that._doneOne(concurrencyId, name, false);
       return;
     }
   }
 
+  if (status === 404 && pkg && pkg.reason === 'deleted' && registry === config.officialNpmReplicate) {
+    // unpublished package on replicate.npmjs.com
+    // 404 { error: 'not_found', reason: 'deleted' }
+    // try to read from registry.npmjs.com and get the whole unpublished info
+    try {
+      var result = yield npmSerivce.request(packageUrl, { registry: config.sourceNpmRegistry });
+      pkg = result.data;
+      status = result.status;
+    } catch (err) {
+      // if 404
+      if (!err.res || err.res.statusCode !== 404) {
+        var errMessage = err.name + ': ' + err.message;
+        that.log('[c#%s] [error] [%s] get package(%s%s) error: %s, status: %s',
+          concurrencyId, name, config.sourceNpmRegistry, packageUrl, errMessage, status);
+        yield that._doneOne(concurrencyId, name, false);
+        return;
+      }
+    }
+  }
+
   var unpublishedInfo = null;
   if (status === 404) {
     // check if it's unpublished
-    if (pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+    // ignore too long package name, see https://github.com/cnpm/cnpmjs.org/issues/1066
+    if (name.length < 80 && pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
       unpublishedInfo = pkg.time.unpublished;
     } else {
       pkg = null;
     }
+  } else {
+    // unpublished package status become to 200
+    if (name.length < 80 && pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+      unpublishedInfo = pkg.time.unpublished;
+    }
   }
 
   if (!pkg) {
-    that.log('[c#%s] [error] [%s] get package error: package not exists, status: %s',
-      concurrencyId, name, status);
-    yield* that._doneOne(concurrencyId, name, true);
+    that.log('[c#%s] [error] [%s] get package(%s%s) error: package not exists, status: %s',
+      concurrencyId, name, registry, packageUrl, status);
+    yield that._doneOne(concurrencyId, name, true);
     return;
   }
 
-  that.log('[c#%d] [%s] pkg status: %d, start...', concurrencyId, name, status);
+  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, unpublished: %j, start...',
+    concurrencyId, name, registry, packageUrl, status,
+    pkg['dist-tags'], pkg.time && pkg.time.modified,
+    unpublishedInfo);
 
   if (unpublishedInfo) {
     try {
-      yield* that._unpublished(name, unpublishedInfo);
+      yield that._unpublished(name, unpublishedInfo);
     } catch (err) {
       that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
-      yield* that._doneOne(concurrencyId, name, false);
+      yield that._doneOne(concurrencyId, name, false);
       return;
     }
-    return yield* that._doneOne(concurrencyId, name, true);
+    return yield that._doneOne(concurrencyId, name, true);
   }
 
   var versions;
   try {
-    versions = yield* that._sync(name, pkg);
+    versions = yield that._sync(name, pkg);
   } catch (err) {
     that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
-    yield* that._doneOne(concurrencyId, name, false);
+    yield that._doneOne(concurrencyId, name, false);
     return;
   }
 
+  // if (versions.length === 0 && registry === config.officialNpmReplicate) {
+  //   // TODO:
+  //   // need to sync sourceNpmRegistry also
+  //   // make sure package data be update event replicate down.
+  //   // https://github.com/npm/registry/issues/129
+  // }
+
+  // has new version
+  if (versions.length > 0) {
+    that.updates.push(name);
+  }
+
   this.log('[c#%d] [%s] synced success, %d versions: %s',
     concurrencyId, name, versions.length, versions.join(', '));
-  yield* this._doneOne(concurrencyId, name, true);
+  yield this._doneOne(concurrencyId, name, true);
 };
 
 function* _listStarUsers(modName) {
-  var users = yield* packageService.listStarUserNames(modName);
+  var users = yield packageService.listStarUserNames(modName);
   var userMap = {};
   users.forEach(function (user) {
     userMap[user] = true;
@@ -611,6 +658,13 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
             deprecated: version.deprecated
           });
         }
+        if (exists.package.deprecated && !version.deprecated) {
+          // remove deprecated info
+          missingDeprecateds.push({
+            id: exists.id,
+            deprecated: undefined,
+          });
+        }
         continue;
       }
     }
@@ -663,7 +717,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
       continue;
     }
     try {
-      yield* that._syncOneVersion(index, syncModule);
+      yield that._syncOneVersion(index, syncModule);
       syncedVersionNames.push(syncModule.version);
     } catch (err) {
       that.log('    [%s:%d] sync error, version: %s, %s: %s',
@@ -671,7 +725,6 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     }
   }
 
-
   if (deletedVersionNames.length === 0) {
     that.log('  [%s] no versions need to deleted', name);
   } else {
@@ -777,7 +830,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
       if (r.error) {
         that.log('    save error, id: %s, error: %s', item.id, r.error.message);
       } else {
-        that.log('    saved, id: %s', item.id);
+        that.log('    saved, id: %s, deprecated: %j', item.id, item.deprecated);
       }
     }
   }
@@ -873,7 +926,10 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
 };
 
 SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
-  logger.syncInfo('[sync_module_worker] start sync %s@%s', sourcePackage.name, sourcePackage.version);
+  var delay = Date.now() - sourcePackage.publish_time;
+  logger.syncInfo('[sync_module_worker] delay: %s ms, publish_time: %s, start sync %s@%s',
+    delay, utility.logDate(new Date(sourcePackage.publish_time)),
+    sourcePackage.name, sourcePackage.version);
   var that = this;
   var username = this.username;
   var downurl = sourcePackage.dist.tarball;
@@ -897,9 +953,11 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     devDependencies = Object.keys(sourcePackage.devDependencies || {});
   }
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, ' +
+  that.log('    [%s:%d] syncing, delay: %s ms, version: %s, dist: %j, no deps: %s, ' +
    'publish on cnpm: %s, dependencies: %d, devDependencies: %d, syncDevDependencies: %s',
-    sourcePackage.name, versionIndex, sourcePackage.version,
+    sourcePackage.name, versionIndex,
+    delay,
+    sourcePackage.version,
     sourcePackage.dist, that.noDep, that._publish,
     dependencies.length,
     devDependencies.length, this.syncDevDependencies);
@@ -923,7 +981,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   }
 
   // add module dependence
-  yield* packageService.addDependencies(sourcePackage.name, dependencies);
+  yield packageService.addDependencies(sourcePackage.name, dependencies);
 
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
@@ -931,7 +989,14 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   try {
     // get tarball
     logger.syncInfo('[sync_module_worker] downloading %j to %j', downurl, filepath);
-    var r = yield urllib.request(downurl, options);
+    var r;
+    try {
+      r = yield urllib.request(downurl, options);
+    } catch (err) {
+      logger.syncInfo('[sync_module_worker] download %j to %j error: %s', downurl, filepath, err);
+      throw err;
+    }
+
     var statusCode = r.status || -1;
     // https://github.com/cnpm/cnpmjs.org/issues/325
     // if (statusCode === 404) {
@@ -945,6 +1010,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -961,6 +1027,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       var err = new Error('Download ' + downurl + ' file size is zero');
       err.name = 'DownloadTarballSizeZeroError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -971,6 +1038,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
         ' not match ' + sourcePackage.dist.shasum);
       err.name = 'DownloadTarballShasumError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -989,8 +1057,9 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       throw err;
     }
     logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result);
-    var r = yield *afterUpload(result);
-    logger.syncInfo('[sync_module_worker] sync %s@%s done!', sourcePackage.name, sourcePackage.version);
+    var r = yield afterUpload(result);
+    logger.syncInfo('[sync_module_worker] sync %s@%s done!',
+      sourcePackage.name, sourcePackage.version);
     return r;
   } finally {
     // remove tmp file whatever
@@ -1034,7 +1103,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     }
 
     mod.package.dist = dist;
-    var r = yield* packageService.saveModule(mod);
+    var r = yield packageService.saveModule(mod);
 
     that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, '
       + 'size: %d, publish_time: %j, publish on cnpm: %s',

controllers/web/package/search.js

@@ -21,8 +21,16 @@ var packageService = require('../../../services/package');
 module.exports = function* search() {
   var params = this.params;
   var word = params.word || params[0];
+  var limit = Number(this.query.limit) || 100;
+
+  if (limit > 10000) {
+    limit = 10000;
+  }
+
   debug('search %j', word);
-  var result = yield* packageService.search(word);
+  var result = yield* packageService.search(word, {
+    limit: limit
+  });
 
   var match = null;
   for (var i = 0; i < result.searchMatchs.length; i++) {

controllers/web/package/show.js

@@ -110,7 +110,7 @@ module.exports = function* show(next) {
     pkg.repository = null;
   }
   if (pkg.repository && pkg.repository.url) {
-    pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
+    pkg.repository.weburl = /^https?:\/\//.test(pkg.repository.url) ? pkg.repository.url : (giturl.parse(pkg.repository.url) || pkg.repository.url);
   }
   if (!pkg.bugs) {
     pkg.bugs = {};
@@ -176,8 +176,8 @@ module.exports = function* show(next) {
     pkg.isPrivate = false;
     // add security check badge
     pkg.snyk = {
-      badge: `https://snyk.io/test/npm/${pkg.name}/badge.svg?style=flat-square`,
-      url: `https://snyk.io/test/npm/${pkg.name}`,
+      badge: `${config.snykUrl}/test/npm/${pkg.name}/badge.svg?style=flat-square`,
+      url: `${config.snykUrl}/test/npm/${pkg.name}`,
     };
   }
 

docs/registry-api.md

@@ -9,6 +9,8 @@
 * [User](/docs/registry-api.md#user)
 * [Search](/docs/registry-api.md#search)
 
+[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/f6c8cb46358039bcd689#?env%5BRegistry%5D=W3sia2V5IjoicmVnaXN0cnkiLCJ0eXBlIjoidGV4dCIsInZhbHVlIjoiaHR0cHM6Ly9yZWdpc3RyeS5ucG0udGFvYmFvLm9yZyIsImVuYWJsZWQiOnRydWV9LHsia2V5IjoicGFja2FnZSIsInZhbHVlIjoiY25wbSIsInR5cGUiOiJ0ZXh0IiwiZW5hYmxlZCI6dHJ1ZX1d)
+
 ## Schema
 
 All API access is over HTTPS or HTTP,

docs/web/readme.md

@@ -67,9 +67,9 @@ Default style is `flat-square`.
 
 Badge URL: `https://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-version-badge](//cnpmjs.org/badge/v/cnpmjs.org.svg)
 
-* `<0.1.0 & >=0.0.0`: ![red-badge](https://dn-img-shields-io.qbox.me/badge/cnpm-0.0.1-red.svg?style=flat-square)
-* `<1.0.0 & >=0.1.0`: ![red-badge](https://dn-img-shields-io.qbox.me/badge/cnpm-0.1.0-green.svg?style=flat-square)
-* `>=1.0.0`: ![red-badge](https://dn-img-shields-io.qbox.me/badge/cnpm-1.0.0-blue.svg?style=flat-square)
+* `<0.1.0 & >=0.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.0.1-red.svg?style=flat-square)
+* `<1.0.0 & >=0.1.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.1.0-green.svg?style=flat-square)
+* `>=1.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-1.0.0-blue.svg?style=flat-square)
 
 ### Downloads
 
@@ -147,7 +147,7 @@ Release [History](/history).
 
 ## npmjs.org, cnpmjs.org and npm.taobao.org relation
 
-[![npm&cnpm](https://dn-cnpm.qbox.me/network.png)](https://dn-cnpm.qbox.me/network.png)
+![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png)
 
 ## Sponsors
 

lib/common.js

@@ -37,7 +37,7 @@ exports.getCDNKey = function (name, filename) {
 
 exports.setDownloadURL = function (pkg, ctx, host) {
   if (pkg.dist) {
-    host = host || ctx.host;
+    host = host || config.registryHost || ctx.host;
     pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
       ctx.protocol,
       host, pkg.name, pkg.name, pkg.version);

middleware/auth.js

@@ -34,13 +34,14 @@ module.exports = function () {
       return yield* unauthorized.call(this, next);
     }
 
-    authorization = new Buffer(authorization, 'base64').toString().split(':');
-    if (authorization.length !== 2) {
-      return yield* unauthorized.call(this, next);
+    authorization = new Buffer(authorization, 'base64').toString();
+    var pos = authorization.indexOf(':');
+    if (pos === -1) {
+       return yield* unauthorized.call(this, next);
     }
 
-    var username = authorization[0];
-    var password = authorization[1];
+    var username = authorization.slice(0, pos);
+    var password = authorization.slice(pos + 1);
 
     var row;
     try {

middleware/publishable.js

@@ -19,8 +19,13 @@ var config = require('../config');
 var debug = require('debug')('cnpmjs.org:middlewares/publishable');
 
 module.exports = function *publishable(next) {
-  // private mode, only admin user can publish
-  if (config.enablePrivate && !this.user.isAdmin) {
+  // admins always can publish and unpublish
+  if (this.user.isAdmin) {
+    return yield next;
+  }
+
+  // private mode, normal user can't publish and unpublish
+  if (config.enablePrivate) {
     this.status = 403;
     this.body = {
       error: 'no_perms',
@@ -29,28 +34,26 @@ module.exports = function *publishable(next) {
     return;
   }
 
-  // public mode, all user have permission to publish `scoped package`
+  // public mode, normal user have permission to publish `scoped package`
   // and only can publish with scopes in `ctx.user.scopes`, default is `config.scopes`
 
   var name = this.params.name || this.params[0];
 
   // check if is private package list in config
   if (config.privatePackages && config.privatePackages.indexOf(name) !== -1) {
-    return yield* next;
+    return yield next;
   }
 
-  // scope
+  // scoped module
   if (name[0] === '@') {
     if (checkScope(name, this)) {
-      return yield* next;
+      return yield next;
     }
     return;
   }
 
   // none-scope
-  if (checkNoneScope(name, this)) {
-    return yield* next;
-  }
+  assertNoneScope(name, this);
 };
 
 /**
@@ -81,12 +84,7 @@ function checkScope(name, ctx) {
  * check if user have permission to publish without scope
  */
 
-function checkNoneScope(name, ctx) {
-  // admins unpublished everything
-  if (ctx.user.isAdmin && ctx.method === 'DELETE') {
-    return true;
-  }
-
+function assertNoneScope(name, ctx) {
   ctx.status = 403;
   if (ctx.user.scopes.length === 0) {
     ctx.body = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.12.1",
+  "version": "2.19.3",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -36,8 +36,10 @@
     "is-type-of": "^1.0.0",
     "kcors": "^1.2.1",
     "koa": "^1.2.0",
+    "koa-bodyparser": "^2.4.0",
     "koa-limit": "^1.0.2",
     "koa-markdown": "^2.0.1",
+    "koa-maxrequests": "^1.0.0",
     "koa-middlewares": "^2.1.0",
     "koa-mock": "^1.6.1",
     "koa-safe-jsonp": "^0.3.1",
@@ -49,6 +51,7 @@
     "mysql": "^2.10.2",
     "mz": "^2.4.0",
     "nodemailer": "^1.3.0",
+    "normalize-registry-metadata": "^1.1.2",
     "semver": "^5.2.0",
     "sequelize": "^3.23.4",
     "thunkify-wrap": "^1.0.4",
@@ -71,10 +74,8 @@
     "pg": "5",
     "pg-hstore": "2",
     "should": "8",
-    "should-http": "*",
     "sqlite3": "*",
-    "supertest": "1",
-    "supertest-as-promised": "3",
+    "supertest": "2",
     "thunk-mocha": "1"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",

routes/registry.js

@@ -1,17 +1,4 @@
-/**!
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
+'use strict';
 
 var limit = require('../middleware/limit');
 var login = require('../middleware/login');
@@ -118,6 +105,7 @@ function routes(app) {
 
   // download times
   app.get('/downloads/range/:range/:name', downloadTotal);
+  app.get(/^\/downloads\/range\/([^\/]+)\/(@[\w\-\.]+\/[\w\-\.]+)$/, downloadTotal);
   app.get('/downloads/range/:range', downloadTotal);
 
   // GET /-/package/:pkg/dependents

routes/web.js

@@ -1,19 +1,4 @@
-/**!
- * cnpmjs.org - routes/web.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
+'use strict';
 
 var showPackage = require('../controllers/web/package/show');
 var searchPackage = require('../controllers/web/package/search');

servers/registry.js

@@ -1,19 +1,4 @@
-/**!
- * cnpmjs.org - servers/registry.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
+'use strict';
 
 var koa = require('koa');
 var app = module.exports = koa();
@@ -28,25 +13,28 @@ var staticCache = require('../middleware/static');
 var notFound = require('../middleware/registry_not_found');
 var cors = require('kcors');
 var proxyToNpm = require('../middleware/proxy_to_npm');
+var maxrequests = require('koa-maxrequests');
+var bodyParser = require('koa-bodyparser');
 
+app.use(maxrequests());
 app.use(block());
 middlewares.jsonp(app);
-app.use(middlewares.rt({headerName: 'X-ReadTime'}));
+app.use(middlewares.rt({ headerName: 'X-ReadTime' }));
 app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
 app.use(staticCache);
 
 app.keys = ['todokey', config.sessionSecret];
 app.proxy = true;
-app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
+app.use(bodyParser({ jsonLimit: config.jsonLimit }));
 app.use(cors({
-  allowMethods: 'GET,HEAD'
+  allowMethods: 'GET,HEAD',
 }));
 app.use(auth());
 app.use(proxyToNpm());
 app.use(notFound);
 
 if (config.enableCompress) {
-  app.use(middlewares.compress({threshold: 150}));
+  app.use(middlewares.compress({ threshold: 150 }));
 }
 app.use(middlewares.conditional());
 app.use(middlewares.etag());

servers/web.js

@@ -1,19 +1,4 @@
-/**!
- * cnpmjs.org - servers/web.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
+'use strict';
 
 var opensearch = require('../middleware/opensearch');
 var notFound = require('../middleware/web_not_found');
@@ -32,6 +17,8 @@ var path = require('path');
 var http = require('http');
 var koa = require('koa');
 var fs = require('fs');
+var maxrequests = require('koa-maxrequests');
+var bodyParser = require('koa-bodyparser');
 
 var app = koa();
 
@@ -39,6 +26,7 @@ jsonp(app);
 
 var rootdir = path.dirname(__dirname);
 
+app.use(maxrequests());
 app.use(block());
 app.use(middlewares.rt({headerName: 'X-ReadTime'}));
 app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
@@ -56,7 +44,7 @@ app.proxy = true;
 app.use(proxyToNpm({
   isWeb: true
 }));
-app.use(middlewares.bodyParser());
+app.use(bodyParser({ jsonLimit: config.jsonLimit }));
 app.use(auth());
 app.use(notFound);
 

services/npm.js

@@ -1,21 +1,7 @@
-/**!
- * cnpmjs.org - services/npm.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- *   dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var ms = require('humanize-ms');
+var cleanNpmMetadata = require('normalize-registry-metadata');
 var urllib = require('../common/urllib');
 var config = require('../config');
 
@@ -34,7 +20,11 @@ function* request(url, options) {
   url = registry + url;
   var r;
   try {
-    r = yield urllib.requestThunk(url, options);
+    r = yield urllib.request(url, options);
+    // https://github.com/npm/registry/issues/87#issuecomment-261450090
+    if (options.dataType === 'json' && r.data && config.officialNpmReplicate === registry) {
+      cleanNpmMetadata(r.data);
+    }
   } catch (err) {
     var statusCode = err.status || -1;
     var data = err.data || '[empty]';

services/package.js

@@ -1,17 +1,5 @@
-/**
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var semver = require('semver');
 var models = require('../models');
 var common = require('./common');
@@ -75,7 +63,7 @@ exports.getModuleByTag = function* (name, tag) {
 };
 
 exports.getModuleByRange = function* (name, range) {
-  var rows = yield exports.listModulesByName(name);
+  var rows = yield exports.listModulesByName(name, [ 'id', 'version' ]);
   var versionMap = {};
   var versions = rows.map(function(row) {
     versionMap[row.version] = row;
@@ -85,7 +73,12 @@ exports.getModuleByRange = function* (name, range) {
   });
 
   var version = semver.maxSatisfying(versions, range);
-  return versionMap[version];
+  if (!versionMap[version]) {
+    return null;
+  }
+
+  var id = versionMap[version].id;
+  return yield exports.getModuleById(id);
 };
 
 exports.getLatestModule = function* (name) {
@@ -212,7 +205,7 @@ exports.listPublicModuleNamesByUser = function* (username) {
 };
 
 // start must be a date or timestamp
-exports.listPublicModuleNamesSince = function* (start) {
+exports.listPublicModuleNamesSince = function* listPublicModuleNamesSince(start) {
   if (!(start instanceof Date)) {
     start = new Date(Number(start));
   }
@@ -241,12 +234,13 @@ exports.listAllPublicModuleNames = function* () {
   });
 };
 
-exports.listModulesByName = function* (moduleName) {
+exports.listModulesByName = function* (moduleName, attributes) {
   var mods = yield Module.findAll({
     where: {
       name: moduleName
     },
-    order: [ ['id', 'DESC'] ]
+    order: [ ['id', 'DESC'] ],
+    attributes,
   });
 
   for (var mod of mods) {
@@ -339,7 +333,7 @@ exports.updateModulePackage = function* (id, pkg) {
 };
 
 exports.updateModulePackageFields = function* (id, fields) {
-  var mod = yield* exports.getModuleById(id);
+  var mod = yield exports.getModuleById(id);
   if (!mod) {
     return null;
   }
@@ -347,7 +341,7 @@ exports.updateModulePackageFields = function* (id, fields) {
   for (var k in fields) {
     pkg[k] = fields[k];
   }
-  return yield* exports.updateModulePackage(id, pkg);
+  return yield exports.updateModulePackage(id, pkg);
 };
 
 exports.updateModuleReadme = function* (id, readme) {

sync/changes_stream_syncer.js

@@ -8,13 +8,14 @@ const streamAwait = require('await-event');
 const logger = require('../common/logger');
 const config = require('../config');
 
-const db = 'https://replicate.npmjs.com';
+const db = config.officialNpmReplicate;
 const lastSeqFile = path.join(config.dataDir, '.cnpmjs.org.last_seq.txt');
 let _STREAM_ID = 0;
 
 module.exports = function* sync() {
   const since = yield getLastSequence();
   const streamId = _STREAM_ID++;
+  let changesCount = 0;
   logger.syncInfo('start changes stream#%d, since: %s', streamId, since);
   const changes = new ChangesStream({
     db,
@@ -23,7 +24,8 @@ module.exports = function* sync() {
   });
   changes.await = streamAwait;
   changes.on('data', change => {
-    logger.syncInfo('stream#%d get change: %j', streamId, change);
+    changesCount++;
+    logger.syncInfo('stream#%d get change#%d: %j', streamId, changesCount, change);
     syncPackage(change);
   });
 
@@ -32,7 +34,7 @@ module.exports = function* sync() {
   } catch (err) {
     // make sure changes steam is destroy
     changes.destroy();
-    err.message += `, stream#${streamId}`;
+    err.message += `, stream#${streamId}, changesCount#${changesCount}`;
     throw err;
   }
 };

sync/index.js

@@ -146,8 +146,12 @@ function sendMailToAdmin(err, result, syncTime) {
   let type;
   let html;
   if (err) {
-    // ignore 503 error
-    if (err.status === 503) {
+    // ignore 503, 504 error
+    // 504: Gateway Time-out
+    if (err.status === 503 || err.status === 504) {
+      return;
+    }
+    if (err.name === 'JSONResponseFormatError') {
       return;
     }
     subject = 'Sync Error';
@@ -161,6 +165,10 @@ function sendMailToAdmin(err, result, syncTime) {
       'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
       syncTime, result.fails.length, result.fails.slice(0, 10),
       result.successes.length, result.successes.slice(0, 10));
+    // skip email notice when fails items small then 3
+    if (result.fails.length < 3) {
+      type = 'log';
+    }
   } else if (result.successes && result.successes.length) {
     subject = 'Sync Finished';
     type = 'log';

sync/status.js

@@ -22,7 +22,7 @@ Status.prototype.log = function (syncDone) {
     lastSyncModule: this.lastSyncModule,
   };
   co(function* () {
-    yield* Total.updateSyncNum(params);
+    yield Total.updateSyncNum(params);
   }).catch(function () {});
 };
 

sync/sync_all.js

@@ -57,19 +57,22 @@ module.exports = function* sync() {
     concurrency: config.syncConcurrency,
     syncUpstreamFirst: false,
   });
-  Status.init({need: packages.length}, worker);
+  Status.init({
+    need: packages.length,
+  }, worker);
   worker.start();
   var end = thunkify.event(worker);
   yield end();
 
-  logger.syncInfo('All packages sync done, successes %d, fails %d',
-      worker.successes.length, worker.fails.length);
+  logger.syncInfo('All packages sync done, successes %d, fails %d, updates %d',
+      worker.successes.length, worker.fails.length, worker.updates.length);
   //only when all succss, set last sync time
   if (!worker.fails.length) {
-    yield* totalService.setLastSyncTime(syncTime);
+    yield totalService.setLastSyncTime(syncTime);
   }
   return {
     successes: worker.successes,
-    fails: worker.fails
+    fails: worker.fails,
+    updates: worker.updates,
   };
 };

test/controllers/registry/package/download_total.test.js

@@ -1,38 +1,34 @@
-/**!
- * cnpmjs.org - test/controllers/registry/package/download_total.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+const request = require('supertest');
+const mm = require('mm');
+const DownloadTotal = require('../../../../services/download_total');
+const app = require('../../../../servers/registry');
+const utils = require('../../../utils');
 
-var request = require('supertest');
-var mm = require('mm');
-var DownloadTotal = require('../../../../services/download_total');
-var app = require('../../../../servers/registry');
-
-describe('controllers/registry/package/download_total.test.js', function () {
+describe('test/controllers/registry/package/download_total.test.js', () => {
   afterEach(mm.restore);
 
-  it('should error when range error', function (done) {
-    request(app.listen())
+  before(() => {
+    const pkg2 = utils.getPackage('@cnpmtest/download_total_test_module', '1.0.1', utils.otherUser);
+    return request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201);
+  });
+
+  it('should error when range error', () => {
+    return request(app.listen())
     .get('/downloads/range/2014-10-10:xxxx/koa')
     .expect(400)
     .expect({
       error: 'range_error',
       reason: 'range must be YYYY-MM-DD:YYYY-MM-DD style'
-    }, done);
+    });
   });
 
-  it('should get package downloads ok', function (done) {
+  it('should get package downloads ok', () => {
     mm.data(DownloadTotal, 'getModuleTotal', [{
       id: 1,
       count: 10,
@@ -50,7 +46,7 @@ describe('controllers/registry/package/download_total.test.js', function () {
       name: 'koa'
     }]);
 
-    request(app.listen())
+    return request(app.listen())
     .get('/downloads/range/2014-12-01:2014-12-03/koa')
     .expect(200)
     .expect({
@@ -67,10 +63,10 @@ describe('controllers/registry/package/download_total.test.js', function () {
         day: '2014-12-03',
         downloads: 10
       }]
-    }, done);
+    });
   });
 
-  it('should get total downloads ok', function (done) {
+  it('should get total downloads ok', () => {
     mm.data(DownloadTotal, 'getTotal', [{
       count: 20,
       date: '2014-12-03',
@@ -82,7 +78,7 @@ describe('controllers/registry/package/download_total.test.js', function () {
       date: '2014-12-02',
     }]);
 
-    request(app.listen())
+    return request(app.listen())
     .get('/downloads/range/2014-12-01:2014-12-03')
     .expect(200)
     .expect({
@@ -98,6 +94,44 @@ describe('controllers/registry/package/download_total.test.js', function () {
         day: '2014-12-03',
         downloads: 20
       }]
-    }, done);
+    });
+  });
+
+  it('should get scope package downloads ok', () => {
+    mm.data(DownloadTotal, 'getModuleTotal', [{
+      id: 1,
+      count: 10,
+      date: '2014-12-03',
+      name: '@cnpmtest/download_total_test_module'
+    }, {
+      id: 1,
+      count: 8,
+      date: '2014-12-01',
+      name: '@cnpmtest/download_total_test_module'
+    }, {
+      id: 1,
+      count: 5,
+      date: '2014-12-02',
+      name: '@cnpmtest/download_total_test_module'
+    }]);
+
+    return request(app.listen())
+    .get('/downloads/range/2014-12-01:2014-12-03/@cnpmtest/download_total_test_module')
+    .expect(200)
+    .expect({
+      start: '2014-12-01',
+      end: '2014-12-03',
+      package: '@cnpmtest/download_total_test_module',
+      downloads: [{
+        day: '2014-12-01',
+        downloads: 8
+      }, {
+        day: '2014-12-02',
+        downloads: 5
+      }, {
+        day: '2014-12-03',
+        downloads: 10
+      }]
+    });
   });
 });

test/controllers/registry/package/list.test.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - test/controllers/registry/package/list.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -23,7 +9,7 @@ var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 var config = require('../../../../config');
 
-describe('controllers/registry/package/list.test.js', function () {
+describe('test/controllers/registry/package/list.test.js', () => {
   afterEach(mm.restore);
 
   before(function (done) {
@@ -85,6 +71,7 @@ describe('controllers/registry/package/list.test.js', function () {
         fengmk2: true,
         foouser: true
       });
+      data.versions['0.0.1'].publish_time.should.equal(data.versions['0.0.1']._cnpm_publish_time);
       done();
     });
   });
@@ -135,12 +122,13 @@ describe('controllers/registry/package/list.test.js', function () {
     }, done);
   });
 
-  describe('unpublished', function () {
-    before(function (done) {
+  describe.skip('unpublished', () => {
+    before(done => {
+      mm(config, 'syncModel', 'all');
       utils.sync('moduletest1', done);
     });
 
-    it('should show unpublished info', function (done) {
+    it('should show unpublished info', done => {
       mm(config, 'syncModel', 'all');
       request(app.listen())
       .get('/moduletest1')

test/controllers/registry/package/show.test.js

@@ -1,17 +1,5 @@
-/**
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -55,7 +43,11 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('0.0.1');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       data.dist.tarball.should.containEql('/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-0.0.1.tgz');
+      data._cnpm_publish_time.should.equal(data.publish_time);
       done();
     });
   });
@@ -68,6 +60,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('1.1.0');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       data.dist.tarball.should.containEql('/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-1.1.0.tgz');
       done();
     });
@@ -81,6 +76,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('0.0.1');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       data.dist.tarball.should.containEql('/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-0.0.1.tgz');
       done();
     });
@@ -94,6 +92,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('1.1.0');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       data.dist.tarball.should.containEql('/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-1.1.0.tgz');
       done();
     });
@@ -107,6 +108,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-only-beta');
       data.version.should.equal('1.0.0-beta.1');
+      data['dist-tags'].should.eql({
+        latest: '1.0.0-beta.1',
+      });
       data.dist.tarball.should.containEql('/@cnpmtest/testmodule-only-beta/download/@cnpmtest/testmodule-only-beta-1.0.0-beta.1.tgz');
       done();
     });
@@ -127,6 +131,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('1.1.0');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       done();
     });
   });
@@ -145,6 +152,9 @@ describe('test/controllers/registry/package/show.test.js', function () {
       var data = res.body;
       data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('0.0.1');
+      data['dist-tags'].should.eql({
+        latest: '1.1.0',
+      });
       done();
     });
   });

test/controllers/sync_module_worker.test.js

@@ -1,9 +1,5 @@
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
 var mm = require('mm');
 var thunkify = require('thunkify-wrap');
@@ -57,6 +53,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
   });
 
   it('should sync public scoped package', function* () {
+    mm(config, 'registryHost', '');
     mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.org');
     var worker = new SyncModuleWorker({
       name: '@sindresorhus/df',
@@ -92,6 +89,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     yield checkResult();
 
     var r = yield urllib.request(tgzUrl);
+    console.log(r.status, r.headers);
     r.status.should.equal(200);
   });
 
@@ -183,7 +181,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
 
   it('should sync unpublished info', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['tnpm'],
+      name: ['afp'],
       username: 'fengmk2'
     });
 
@@ -191,7 +189,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['tnpm']);
+      names.should.eql([ 'afp' ]);
       done();
     });
   });
@@ -281,6 +279,44 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     });
   });
 
+  describe('sync deprecated info', () => {
+    before(function* () {
+      mm(config, 'syncModel', 'all');
+      const worker = new SyncModuleWorker({
+        name: 'ms',
+        username: 'fengmk2',
+      });
+      worker.start();
+      const end = thunkify.event(worker, 'end');
+      yield end();
+    });
+
+    it('should sync support un-deprecate action', function* () {
+      const listModulesByName = packageService.listModulesByName;
+      mm(packageService, 'listModulesByName', function* (name) {
+        const mods = yield listModulesByName.call(packageService, name);
+        mods.forEach(function (mod) {
+          mod.package.deprecated = 'mock deprecated';
+        });
+        return mods;
+      });
+
+      var worker = new SyncModuleWorker({
+        name: 'ms',
+        username: 'fengmk2',
+      });
+      worker.start();
+      const end = thunkify.event(worker, 'end');
+      yield end();
+      mm.restore();
+      // check deprecated
+      const mods = yield packageService.listModulesByName('ms');
+      for (const mod of mods) {
+        should.ok(mod.package.deprecated === undefined);
+      }
+    });
+  });
+
   describe('sync user', function () {
     it('should sync fengmk2', function* () {
       var worker = new SyncModuleWorker({

test/controllers/web/package/search.test.js

@@ -22,6 +22,9 @@ var registry = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
 describe('controllers/web/package/search.test.js', function () {
+
+  var app_reg;
+
   before(function (done) {
     var pkg = utils.getPackage('@cnpmtest/testmodule-web-search', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
@@ -29,7 +32,10 @@ describe('controllers/web/package/search.test.js', function () {
       mocha: '~1.0.0',
       'testmodule-web-show': '0.0.1'
     };
-    request(registry.listen())
+
+    app_reg = registry.listen();
+    
+    request(app_reg)
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
@@ -73,5 +79,51 @@ describe('controllers/web/package/search.test.js', function () {
       .expect(200)
       .expect(/Can not found package match notexistpackage/, done);
     });
+
+
+    describe('GET /browse/keyword/:word searchlist', function () {
+
+      before(function (done) {
+        var pkg = utils.getPackage('@cnpmtest/testmodule-web-search_a', '0.0.1', utils.admin);
+        pkg.versions['0.0.1'].dependencies = {
+          bytetest: '~0.0.1',
+          mocha: '~1.0.0',
+          'testmodule-web-show': '0.0.1'
+        };
+        request(app_reg)
+          .put('/' + pkg.name)
+          .set('authorization', utils.adminAuth)
+          .send(pkg)
+          .expect(201, done);
+      });
+
+      it('should list by keyword with json(default limit=100)', function (done) {
+          request(app)
+          .get('/browse/keyword/@cnpmtest/testmodule-web-searc?type=json')
+          .expect(200)
+          .expect({
+            keyword: '@cnpmtest/testmodule-web-searc',
+            match: null,
+            packages: [ { name: '@cnpmtest/testmodule-web-search', description: '' },
+            { name: '@cnpmtest/testmodule-web-search_a', description: '' }],
+            keywords: []
+          })
+          .expect('content-type', 'application/json; charset=utf-8', done);
+      });
+
+      it('should list by keyword with json(use limit)', function (done) {
+          request(app)
+          .get('/browse/keyword/@cnpmtest/testmodule-web-searc?type=json&limit=1')
+          .expect(200)
+          .expect({
+            keyword: '@cnpmtest/testmodule-web-searc',
+            match: null,
+            packages: [ { name: '@cnpmtest/testmodule-web-search', description: '' }],
+            keywords: []
+          })
+          .expect('content-type', 'application/json; charset=utf-8', done);
+      });
+
+    });
   });
 });

test/controllers/web/package/show.test.js

@@ -1,14 +1,14 @@
 'use strict';
 
 var should = require('should');
-var request = require('supertest-as-promised');
+var request = require('supertest');
 var mm = require('mm');
 var config = require('../../../../config');
 var app = require('../../../../servers/web');
 var registry = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
-describe('controllers/web/package/show.test.js', () => {
+describe('test/controllers/web/package/show.test.js', () => {
   before(function (done) {
     var pkg = utils.getPackage('@cnpmtest/testmodule-web-show', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
@@ -36,7 +36,7 @@ describe('controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -52,7 +52,7 @@ describe('controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -99,8 +99,9 @@ describe('controllers/web/package/show.test.js', () => {
     });
   });
 
-  describe('unpublished package', () => {
+  describe.skip('unpublished package', () => {
     before(done => {
+      mm(config, 'syncModel', 'all');
       utils.sync('mk2testmodule', done);
     });
 
@@ -115,7 +116,8 @@ describe('controllers/web/package/show.test.js', () => {
 
   describe('xss filter', function () {
     before(function (done) {
-      var pkg = utils.getPackage('@cnpmtest/xss-test-ut', '0.0.1', utils.admin, null, '[xss link](javascript:alert(2)) \n\nfoo<script>alert(1)</script>/xss\'"&#');
+      var pkg = utils.getPackage('@cnpmtest/xss-test-ut', '0.0.1', utils.admin,
+        null, '[xss link](javascript:alert(2)) \n\nfoo<script>alert(1)</script>/xss\'"&#');
       request(registry.listen())
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -153,4 +155,196 @@ describe('controllers/web/package/show.test.js', () => {
         .expect(/pedding/);
     });
   });
+
+  describe('show repository url in git syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-git', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'git://github.com/cnpm/cnpmjs.org.git'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-git')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-git/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
+
+   describe('show repository url in ssh syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-ssh', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'git@github.com:cnpm/cnpmjs.org.git'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-ssh')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-ssh/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
+
+  describe('show repository url in raw ssh syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-raw-ssh', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'ssh://git@github.com/cnpm/cnpmjs.org.git'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-raw-ssh')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-raw-ssh/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
+
+  describe('show repository url in https syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-https', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'https://github.com/cnpm/cnpmjs.org.git'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-https')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-https/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org\.git/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
+
+   describe('show repository url in short https syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-short-https', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'https://github.com/cnpm/cnpmjs.org'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-short-https')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-short-https/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
+
+  describe('show repository url in short http syntax', () => {
+    before(function (done) {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-repo-short-http', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].repository = {
+        type: 'git',
+        url: 'http://github.com/cnpm/cnpmjs.org.git'
+      }
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should get 200', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/testmodule-repo-short-http')
+      .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
+      .expect(/testmodule-repo-short-http/)
+      .expect(/Maintainers/)
+      .expect(/Dependencies/)
+      .expect(/http:\/\/github\.com\/cnpm\/cnpmjs\.org/)
+      .expect(/Downloads/, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers.etag);
+        res.text.should.containEql('<meta charset="utf-8">');
+        done();
+      });
+    });
+  });
 });

test/init_db.js

@@ -25,6 +25,7 @@ var usernames = [
   'cnpmjstest101',
   'cnpmjstest102',
   'cnpmjstest103',
+  ['cnpmjstest104', 'cnpmjs:test104'],
   'cnpmjstest10', // admin
   'cnpmjstestAdmin2', // other admin
   'cnpmjstestAdmin3', // other admin
@@ -33,6 +34,13 @@ var usernames = [
 
 var count = usernames.length;
 usernames.forEach(function (name) {
+  var pass;
+  if (Array.isArray(name)) {
+    name = name[0];
+    pass = name[1];
+  } else {
+    pass = name;
+  }
   var user = User.build({
     name: name,
     email: 'fengmk2@gmail.com',
@@ -40,7 +48,7 @@ usernames.forEach(function (name) {
     rev: '1',
   });
   user.salt = crypto.randomBytes(30).toString('hex');
-  user.password_sha = User.createPasswordSha(name, user.salt);
+  user.password_sha = User.createPasswordSha(pass, user.salt);
   user.save().then(function () {
     count--;
     if (count === 0) {

test/middleware/auth.test.js

@@ -53,6 +53,13 @@ describe('middleware/auth.test.js', function () {
       .expect(200, done);
     });
 
+    it('should pass with authorization (password contains ":") and check ok', function (done) {
+      request(app)
+      .get('/-/user/org.couchdb.user:cnpmjstest104')
+      .set('authorization', 'basic ' + new Buffer('cnpmjstest104:cnpmjs:test104').toString('base64'))
+      .expect(200, done);
+    });
+
     describe('config.customUserService = true', function () {
       beforeEach(function () {
         mm(config, 'customUserService', true);

test/middleware/publishable.test.js

@@ -0,0 +1,116 @@
+'use strict';
+
+var app = require('../../servers/registry');
+var config = require('../../config');
+var request = require('supertest');
+var utils = require('../utils');
+var mm = require('mm');
+
+describe('middleware/publishable.test.js', function () {
+  afterEach(mm.restore);
+
+  it('should not 403 when admin put to public package', function (done) {
+    mm(config, 'enablePrivate', false);
+    request(app)
+    .put('/koa/-rev/1')
+    .set('authorization', utils.adminAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(404, done);
+  });
+
+  it('should not 403 when admin put to unsupported scope package', function (done) {
+    mm(config, 'enablePrivate', false);
+    request(app)
+    .put('/@unsupported/koa/-rev/1')
+    .set('authorization', utils.adminAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(404, done);
+  });
+
+  it('should not 403 when normal user put to white list package', function (done) {
+    mm(config, 'enablePrivate', false);
+    mm(config, 'privatePackages', ['koa'])
+    request(app)
+    .put('/koa/-rev/1')
+    .set('authorization', utils.otherUserAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(404, done);
+  });
+
+  it('should not 403 when normal user put to supported scope', function (done) {
+    mm(config, 'enablePrivate', false);
+    mm(config, 'scopes', ['@test'])
+    request(app)
+    .put('/@test/koa/-rev/1')
+    .set('authorization', utils.otherUserAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(404, done);
+  });
+
+  it('should 400 when normal user put to unsupported scope', function (done) {
+    mm(config, 'enablePrivate', false);
+    mm(config, 'scopes', ['@test'])
+    request(app)
+    .put('/@test1/koa/-rev/1')
+    .set('authorization', utils.otherUserAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(400, done);
+  });
+
+  it('should 403 when common user put to public package', function (done) {
+    mm(config, 'enablePrivate', false);
+    request(app)
+    .put('/koa/-rev/1')
+    .set('authorization', utils.otherUserAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect(403, done);
+  });
+
+  it('should 403 when common user put in private mode', function (done) {
+    mm(config, 'enablePrivate', true);
+    request(app)
+    .put('/koa/-rev/1')
+    .set('authorization', utils.otherUserAuth)
+    .send({
+      versions: {
+        '0.0.1': {},
+        '0.0.2': {}
+      }
+    })
+    .expect({
+      error: 'no_perms',
+      reason: 'Private mode enable, only admin can publish this module'
+    })
+    .expect(403, done);
+  });
+});

test/services/npm.test.js

@@ -1,45 +1,32 @@
-/**!
- * cnpmjs.org - test/services/npm.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
 var mm = require('mm');
 var fs = require('fs');
 var path = require('path');
 var ChunkStream = require('chunkstream');
+var config = require('../../config');
 var npm = require('../../services/npm');
 
 var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 
-describe('services/npm.test.js', function () {
+describe('services/npm.test.js', () => {
   afterEach(mm.restore);
 
   it('should return a module info from source npm', function* () {
-    var data = yield* npm.get('pedding');
+    var data = yield npm.get('pedding');
     data.name.should.equal('pedding');
   });
 
   it('should return null when module not exist', function *() {
-    var data = yield* npm.get('pedding-not-exists');
+    var data = yield npm.get('pedding-not-exists');
     should.not.exist(data);
   });
 
   it.skip('should return error when http error', function* () {
     mm.http.request(/\//, new ChunkStream(['{']));
     try {
-      yield* npm.get('pedding-not-exists');
+      yield npm.get('pedding-not-exists');
       throw new Error('should not run this');
     } catch (err) {
       err.name.should.equal('JSONResponseFormatError');
@@ -51,7 +38,7 @@ describe('services/npm.test.js', function () {
     mm.http.request(/\//, content, { statusCode: 500 });
     // http://registry.npmjs.org/octopie
     try {
-      yield* npm.get('octopie');
+      yield npm.get('octopie');
       throw new Error('should not run this');
     } catch (err) {
       err.name.should.equal('NPMServerError');
@@ -60,7 +47,31 @@ describe('services/npm.test.js', function () {
     }
   });
 
-  describe('getPopular()', function () {
+  describe('request()', () => {
+    it('should request from replicate and clean meta data', function* () {
+      const result = yield npm.request('/shelljs', {
+        registry: config.officialNpmReplicate,
+      });
+      const pkg = result.data;
+      pkg.name.should.equal('shelljs');
+      pkg.time['0.0.1-alpha1'].should.equal('2012-03-02T21:46:14.725Z');
+      pkg.versions['0.0.1-alpha1'].version.should.equal('0.0.1-alpha1');
+      pkg.versions['0.0.1-alpha1'].dist.shasum.should.equal('cfa9394e29c3eb0fe58998f5bf5bda79aa7d3e2e');
+      pkg.versions['0.0.1-alpha1'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.1alpha1.tgz');
+
+      pkg.time['0.7.5'].should.equal('2016-10-27T05:50:21.479Z');
+      pkg.versions['0.7.5'].version.should.equal('0.7.5');
+      pkg.versions['0.7.5'].dist.shasum.should.equal('2eef7a50a21e1ccf37da00df767ec69e30ad0675');
+      pkg.versions['0.7.5'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.7.5.tgz');
+
+      pkg.time['0.0.6-pre2'].should.equal('2012-05-25T18:14:25.441Z');
+      pkg.versions['0.0.6-pre2'].version.should.equal('0.0.6-pre2');
+      pkg.versions['0.0.6-pre2'].dist.shasum.should.equal('8c3eecaddba6f425bd5cae001f80a4d224750911');
+      pkg.versions['0.0.6-pre2'].dist.tarball.should.equal('http://registry.npmjs.org/shelljs/-/shelljs-0.0.6pre2.tgz');
+    });
+  });
+
+  describe('getPopular()', () => {
     it('should return popular modules', function* () {
       mm.http.request(/\//, JSON.stringify({
         rows: [
@@ -82,7 +93,7 @@ describe('services/npm.test.js', function () {
           { key: ['foo15'], value: 1 },
         ]
       }));
-      var rows = yield* npm.getPopular(10);
+      var rows = yield npm.getPopular(10);
       rows.should.length(2);
       rows[0][0].should.equal('underscore');
     });

tools/resync_npm_issue_87.js

@@ -0,0 +1,43 @@
+// try to fix https://github.com/npm/registry/issues/87
+
+const urllib = require('urllib');
+const co = require('co');
+
+const sleep = ms => cb => setTimeout(cb, ms);
+
+// node resync_npm.js [registry]
+const registry = process.argv[2] || require('../config').sourceNpmRegistry;
+const url = 'https://os.alipayobjects.com/rmsportal/eDMScnlBhNhHaGXMJWxjvCjfxMHhYwEx.html';
+
+co(function* () {
+  const result = yield urllib.request(url);
+  const items = result.data.toString().split('\n');
+  let count = 0;
+  for (let item of items) {
+    item = item.trim().split(',');
+    let name = item[0];
+    name = name.substring(1, name.length - 1);
+    if (!name) {
+      continue;
+    }
+
+    const r = yield urllib.request(`${registry}/${name}/sync`, {
+      method: 'PUT',
+      dataType: 'json',
+    });
+
+    r.data = r.data || {};
+
+    count++;
+    console.log('#%d %s %s, log: %s',
+      count, name, r.status, `${registry}/${name}/sync/log/${r.data.logId}`);
+    if (count % 50 === 0) {
+      yield sleep(10000);
+    }
+  }
+  console.log('All %d packages sync done', items.length);
+  process.exit(0);
+}).catch(err => {
+  console.error(err.stack);
+  process.exit(1);
+});

worker.js

@@ -1,18 +1,5 @@
-/**!
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var graceful = require('graceful');
 var registry = require('./servers/registry');
 var web = require('./servers/web');