Release 0.3.10

Only /_session request send the authSession. fixed #223

.gitignore

@@ -21,3 +21,5 @@ backup/*.gz
 docs/web/history.md
 view/web/_layout.html
 bin/mysql.js
+bin/test.sql
+coverage/

.jshintignore

@@ -0,0 +1,4 @@
+node_modules/
+coverage/
+.tmp/
+.git/

.jshintrc

@@ -0,0 +1,95 @@
+{
+    // JSHint Default Configuration File (as on JSHint website)
+    // See http://jshint.com/docs/ for more details
+
+    "maxerr"        : 50,       // {int} Maximum error before stopping
+
+    // Enforcing
+    "bitwise"       : true,     // true: Prohibit bitwise operators (&, |, ^, etc.)
+    "camelcase"     : false,    // true: Identifiers must be in camelCase
+    "curly"         : true,     // true: Require {} for every new block or scope
+    "eqeqeq"        : true,     // true: Require triple equals (===) for comparison
+    "forin"         : false,    // true: Require filtering for..in loops with obj.hasOwnProperty()
+    "immed"         : false,    // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
+    "indent"        : false,    // {int} Number of spaces to use for indentation
+    "latedef"       : false,    // true: Require variables/functions to be defined before being used
+    "newcap"        : false,    // true: Require capitalization of all constructor functions e.g. `new F()`
+    "noarg"         : true,     // true: Prohibit use of `arguments.caller` and `arguments.callee`
+    "noempty"       : true,     // true: Prohibit use of empty blocks
+    "nonew"         : false,    // true: Prohibit use of constructors for side-effects (without assignment)
+    "plusplus"      : false,    // true: Prohibit use of `++` & `--`
+    "quotmark"      : false,    // Quotation mark consistency:
+                                //   false    : do nothing (default)
+                                //   true     : ensure whatever is used is consistent
+                                //   "single" : require single quotes
+                                //   "double" : require double quotes
+    "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
+    "unused"        : false,    // true: Require all defined variables be used
+    "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
+    "trailing"      : false,    // true: Prohibit trailing whitespaces
+    "maxparams"     : false,    // {int} Max number of formal params allowed per function
+    "maxdepth"      : false,    // {int} Max depth of nested blocks (within functions)
+    "maxstatements" : false,    // {int} Max number statements per function
+    "maxcomplexity" : false,    // {int} Max cyclomatic complexity per function
+    "maxlen"        : false,    // {int} Max number of characters per line
+
+    // Relaxing
+    "asi"           : false,     // true: Tolerate Automatic Semicolon Insertion (no semicolons)
+    "boss"          : true,      // true: Tolerate assignments where comparisons would be expected
+    "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
+    "eqnull"        : false,     // true: Tolerate use of `== null`
+    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
+    "esnext"        : true,      // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
+                                 // (ex: `for each`, multiple try/catch, function expression…)
+    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`
+    "expr"          : true,      // true: Tolerate `ExpressionStatement` as Programs
+    "funcscope"     : false,     // true: Tolerate defining variables inside control statements"
+    "globalstrict"  : false,     // true: Allow global "use strict" (also enables 'strict')
+    "iterator"      : false,     // true: Tolerate using the `__iterator__` property
+    "lastsemic"     : false,     // true: Tolerate omitting a semicolon for the last statement of a 1-line block
+    "laxbreak"      : true,      // true: Tolerate possibly unsafe line breakings
+    "laxcomma"      : false,     // true: Tolerate comma-first style coding
+    "loopfunc"      : false,     // true: Tolerate functions being defined in loops
+    "multistr"      : true,      // true: Tolerate multi-line strings
+    "proto"         : false,     // true: Tolerate using the `__proto__` property
+    "scripturl"     : false,     // true: Tolerate script-targeted URLs
+    "smarttabs"     : false,     // true: Tolerate mixed tabs/spaces when used for alignment
+    "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
+    "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
+    "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
+    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+
+    // Environments
+    "browser"       : true,     // Web Browser (window, document, etc)
+    "couch"         : false,    // CouchDB
+    "devel"         : true,     // Development/debugging (alert, confirm, etc)
+    "dojo"          : false,    // Dojo Toolkit
+    "jquery"        : false,    // jQuery
+    "mootools"      : false,    // MooTools
+    "node"          : true,     // Node.js
+    "nonstandard"   : false,    // Widely adopted globals (escape, unescape, etc)
+    "prototypejs"   : false,    // Prototype and Scriptaculous
+    "rhino"         : false,    // Rhino
+    "worker"        : false,    // Web Workers
+    "wsh"           : false,    // Windows Scripting Host
+    "yui"           : false,    // Yahoo User Interface
+    "noyield"       : true,     // allow generators without a yield
+
+    // Legacy
+    "nomen"         : false,    // true: Prohibit dangling `_` in variables
+    "onevar"        : false,    // true: Allow only one `var` statement per function
+    "passfail"      : false,    // true: Stop on first error
+    "white"         : false,    // true: Check against strict whitespace and indentation rules
+
+    // Custom Globals
+    "globals"       : {         // additional predefined global variables
+      // mocha
+      "describe": true,
+      "it": true,
+      "before": true,
+      "afterEach": true,
+      "beforeEach": true,
+      "after": true
+    }
+}

.npmignore

@@ -10,3 +10,7 @@ backup/*.json
 backup/*.gz
 view/web/_layout.html
 bin/mysql.js
+bin/test.sql
+coverage/
+.jshintrc
+.jshintignore

.travis.yml

@@ -1,5 +1,3 @@
 language: node_js
 node_js:
-  - '0.10'
-install: make install
-script: make test-coveralls
+  - '0.11'

AUTHORS

@@ -1,5 +1,5 @@
 # Ordered by date of first contribution.
-# Auto-generated by 'contributors' on Fri, 24 Jan 2014 08:35:59 GMT.
+# Auto-generated by 'contributors' on Mon, 03 Mar 2014 13:01:28 GMT.
 # https://github.com/xingrz/node-contributors
 
 fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)

History.md

@@ -1,4 +1,148 @@
 
+0.3.10 / 2014-03-16 
+==================
+
+  * Only /_session request send the authSession. fixed #223
+  * sync npm user info when maintainers and contributors not exists. fixed #82
+  * save npm user to mysql
+  * password salt always be randoms
+  * remove session access in /name and /name/version, fixed #274
+  * fix update maintainer session error
+  * update koa-middlewares
+  * fix test, fix sync_by_install
+  * use defer session
+  * Support npm owner|author add [name] [pkg]. fixed #271
+
+0.3.9 / 2014-03-14 
+==================
+
+  * custom user-agent
+  * use co-urllib instead of thunkify urllib; fix mock http.request test cases
+  * request limit custom message
+  * add config.redis check
+  * add koa-limit, fixed #267
+
+0.3.8 / 2014-03-11 
+==================
+
+  * update middlewares, fixed missing charset bug #264
+
+0.3.7 / 2014-03-11 
+==================
+
+  * show worker die date time
+  * update to koa@0.5.1
+  * hotfix for star user
+  * fix yield gather, sync missing deps even no missing versions
+  * fix return versions
+  * fix makefile, remove eventproxy
+  * refactor sync_module_worker
+  * add make test-dev, fixed #259
+  * change npm.js to generator
+  * update urllib, proxy/npm.js use generator
+  * sync_all and sync_exist to generator
+  * change function to generator
+  * need node >= v0.11.9
+
+0.3.6 / 2014-03-06 
+==================
+
+  * install missing package should sync it from source npm. fixed #252
+  * npm publish dont contains .jshint*
+  * npm test run jshint
+  * Add jshint check: $ make jshint
+  * use `yield* next` instead of `yield next`
+  * replace dist.u.qiniudn.com with cnpmjs.org/dist
+
+0.3.5 / 2014-03-05 
+==================
+
+  * redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
+  * redirect /name to /package/name when /name is 404. fixed #245
+  * Add missing properies and sync missing star users. fixed #235
+
+0.3.4 / 2014-03-04 
+==================
+
+  * add cov
+  * use istanbul run test coverage
+  * gzip support. fix #241
+  * readme spelling patch (@stanzheng)
+  * default readme to null, fixed #233
+  * remove readme in versions
+
+0.3.3 / 2014-02-28 
+==================
+
+  * Merge pull request #232 from cnpm/host-hotfix
+  * get request host from request.headers
+  * Merge pull request #231 from cnpm/bug-fix
+  * fix deps display bug#230 and nsf.url TypeError#229
+
+0.3.2 / 2014-02-28 
+==================
+
+  * update koa-sess and koa-redis
+  * fix sync all test
+  * remove nfs.downloadStream first, fix tmppath error
+  * fix fengmk2/giturl#1 bug
+
+0.3.1 / 2014-02-27 
+==================
+
+  * add etag fixed #224
+  * travis ci install on source npm
+
+0.3.0 / 2014-02-27 
+==================
+
+  * fix typo and dont sync not exists pkgs
+  * use koa-middlewares
+  * fix signed cookie not work on npm@1.3.25; node --harmony-generators
+  * fix opensearch test case
+  * update koa bodyparser
+  * logger.error(err) should send err stack email notice
+  * json body parse limit and bug fix.
+  * fix sync 404 reason not clear
+  * all controllers to koa
+  * controller/web/user.js to koa
+  * change web connect to koa
+  * use outputError
+  * use yield exports.addPackageAndDist.call(this, next);
+  * add end() when ws write end
+  * fix yield coWrite
+  * fix all the test of registry module.test.js
+  * convert registry/module.js to koa type
+  * fix auth middleware
+  * finish registry user controller koa and update mm to support thunkify. fixed #196
+  * change controllers/user.js to koa
+  * thunkify all proxy
+  * convert all middlewares to koa type
+  * change regsitry sync to koa
+  * addd koa-jsonp, koa-bodyparser, fix / controller
+  * first koa run registry home page /
+  * Merge pull request #212 from cnpm/fix-sync-404
+  * return friendly 404 reason
+  * Merge pull request #211 from cnpm/bug-fix
+  * override json limit to default 10mb. fixed #209
+  * fix #210 addPackageAndDist package version detect bug
+
+0.2.27 / 2014-02-19 
+==================
+
+  * support json result in search, fixed #189
+
+0.2.26 / 2014-02-19 
+==================
+
+  * npm publish also need to add deps
+
+0.2.25 / 2014-02-19 
+==================
+
+  * max handle number of package.json `dependencies` property
+  * Dependents support. fixed #190
+
 0.2.24 / 2014-02-13 
 ==================
 

Makefile

@@ -4,35 +4,47 @@ TIMEOUT = 30000
 MOCHA_OPTS =
 
 install:
-	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm --disturl=http://dist.u.qiniudn.com
+	@npm install --registry=http://r.cnpmjs.org \
+		--disturl=http://dist.cnpmjs.org
 
-test: install
-	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
+jshint:
+	@-./node_modules/.bin/jshint ./
+
+test:
+	@NODE_ENV=test ./node_modules/.bin/mocha \
+		--harmony-generators \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
+
 test-cov:
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
+	@NODE_ENV=test node --harmony \
+		node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
+		-- -u exports \
+		--reporter $(REPORTER) \
+		--timeout $(TIMEOUT) \
+		$(MOCHA_OPTS) \
+		$(TESTS)
+	@-$(MAKE) check-coverage
 
-test-cov-html:
-	@rm -f coverage.html
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
-	@ls -lh coverage.html
+check-coverage:
+	@./node_modules/.bin/istanbul check-coverage \
+		--statements 100 \
+		--functions 100 \
+		--branches 100 \
+		--lines 100
 
-test-coveralls: test
-	@echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
-	@-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
+cov:
+	@./node_modules/.bin/cov coverage
 
-test-all: test test-cov
+contributors:
+	@./node_modules/.bin/contributors -f plain -o AUTHORS
 
-contributors: install
-	@./node_modules/contributors/bin/contributors -f plain -o AUTHORS
-
-autod: install
-	@./node_modules/.bin/autod -w -e public,view,docs,backup
+autod:
+	@./node_modules/.bin/autod -w -e public,view,docs,backup,coverage
 	@$(MAKE) install
 
 .PHONY: test

README.md

@@ -1,7 +1,7 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
@@ -9,9 +9,10 @@ cnpmjs.org
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
+Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
+* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
@@ -19,15 +20,17 @@ Private npm registry and web for Enterprise, base on MySQL and [Simple Store Ser
 ## Install
 
 ```bash
-$ npm install
+$ npm install --registry=http://r.cnpmjs.org --disturl=http://cnpmjs.org/dist
 ```
 
 ## Usage
 
 ```js
-$ node dispatch.js
+$ node --harmony-generators dispatch.js
 ```
 
+**Notice**: need node version >=0.11.9
+
 ## Guide
 
 * [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
@@ -39,14 +42,14 @@ $ node dispatch.js
 $ git summary
 
  project  : cnpmjs.org
- repo age : 7 weeks
- active   : 132 days
- commits  : 315
- files    : 88
+ repo age : 3 months
+ active   : 145 days
+ commits  : 366
+ files    : 94
  authors  :
-   190  fengmk2                 60.3%
-   122  dead_horse              38.7%
-     2  4simple                 0.6%
+   217  fengmk2                 59.3%
+   146  dead_horse              39.9%
+     2  4simple                 0.5%
      1  Alsotang                0.3%
 ```
 

bin/nodejsctl

@@ -7,7 +7,7 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS=node
+NODEJS='node --harmony-generators'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
 STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\

bin/restore_module_deps.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - bin/restore_module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var Module = require('../proxy/module');
+var ModuleDeps = require('../proxy/module_deps');
+
+var addCount = 0;
+
+function restore(id, callback) {
+  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
+  mysql.query(sql, [id], function (err, rows) {
+    if (err) {
+      return callback(err);
+    }
+    if (rows.length === 0) {
+      return callback(null, []);
+    }
+
+    console.log('[%s] got %d rows', id, rows.length);
+
+    rows.forEach(function (r) {
+      Module.parseRow(r);
+      if (!r.package) {
+        return;
+      }
+      var deps = Object.keys(r.package.dependencies || {});
+      if (!Array.isArray(deps) || !deps.length) {
+        return;
+      }
+      deps.forEach(function (dep) {
+        ModuleDeps.add(dep, r.name, function (err) {
+          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
+        });
+      });
+      addCount += deps.length;
+    });
+    setTimeout(function () {
+      console.log('[%s] add %d relations', id, addCount);
+      callback(null, rows);
+    }, 1000);
+  });
+}
+
+var id = 0;
+function run() {
+  restore(id, function (err, rows) {
+    if (err) {
+      throw err;
+    }
+    if (rows.length === 0) {
+      console.log('finished, last id: %s, exit.', id);
+      process.exit(0);
+    }
+
+    id = rows[rows.length - 1].id;
+    run();
+  });
+}
+
+run();

common/logger.js

@@ -1,7 +1,12 @@
-/*!
+/**!
  * cnpmjs.org - common/logger.js
- * Copyright(c) 2013 
- * Author: dead_horse <undefined>
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
  */
 
 'use strict';
@@ -10,11 +15,12 @@
  * Module dependencies.
  */
 
-var config = require('../config');
 var util = require('util');
 var moment = require('moment');
 var logstream = require('logfilestream');
 var ms = require('ms');
+var config = require('../config');
+var mail = require('./mail');
 
 var isTEST = process.env.NODE_ENV === 'test';
 var ONE_DAY = ms('1d');
@@ -29,7 +35,9 @@ levels.forEach(function (catetory) {
   var stream = logstream(options);
   function write(msg) {
     var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
+    var subject = null;
     if (msg instanceof Error) {
+      subject = msg.name;
       var err = {
         name: msg.name,
         code: msg.code,
@@ -57,12 +65,21 @@ levels.forEach(function (catetory) {
     } else {
       msg = time + ' ' + util.format.apply(util, arguments) + '\n';
     }
+
     if (!isTEST) {
+      stream.write(msg);
       if (config.debug) {
         var level = catetory;
         console.log('[' + level + '] ' + msg);
       } else {
-        stream.write(msg);
+        if (catetory === 'error' && subject) {
+          // send error email
+          var to = [];
+          for (var name in config.admins) {
+            to.push(config.admins[name]);
+          }
+          mail.error(to, subject, msg);
+        }
       }
     }
   }

common/mail.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - common/mail.js
  *
  * Copyright(c) cnpmjs.org and other contributors.

common/mysql.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var ready = require('ready');
 var mysql = require('mysql');
 var config = require('../config');
@@ -35,7 +36,13 @@ var pool = mysql.createPool({
 exports.pool = pool;
 
 exports.query = function (sql, values, cb) {
-  pool.query(sql, values, cb);
+  if (typeof values === 'function') {
+    cb = values;
+    values = null;
+  }
+  pool.query(sql, values, function (err, rows) {
+    cb(err, rows);
+  });
 };
 
 exports.queryOne = function (sql, values, cb) {
@@ -57,6 +64,8 @@ exports.escape = function (val) {
 
 ready(exports);
 
+thunkify(exports);
+
 function init() {
   exports.query('show tables', function (err, rows) {
     if (err) {

common/qnfs.js

@@ -14,9 +14,9 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var qn = require('qn');
 var config = require('../config');
-
 var client = qn.create(config.qn);
 
 exports._client = client;
@@ -61,3 +61,5 @@ exports.url = function (key) {
 exports.remove = function (key, callback) {
   client.delete(key, callback);
 };
+
+thunkify(exports);

common/redis.js

@@ -0,0 +1,38 @@
+/**!
+ * cnpmjs.org - common/redis.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+
+// close redis by set config.redis to `null` or `{}`
+if (!config.redis || !config.redis.host || !config.redis.port) {
+
+  var redis = require('redis');
+  var wrapper = require('co-redis');
+  var logger = require('./logger');
+
+  var _client = redis.createClient(config.redis);
+
+  _client.on('error', function (err) {
+    logger.error(err);
+  });
+
+  module.exports = wrapper(_client);
+
+} else {
+  console.warn('[%s] [worker:%s:common/redis.js] Redis config can not found',
+    Date(), process.pid);
+  module.exports = null;
+}

common/session.js

@@ -15,27 +15,20 @@
  * Module dependencies.
  */
 
-var connect = require('connect');
-var RedisStore = require('connect-redis')(connect);
+var middlewares = require('koa-middlewares');
 var config = require('../config');
 
-var session;
 var key = 'AuthSession';
-var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 30 };
+var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 365, signed: false };
+var options = {
+  key: key,
+  cookie: cookie,
+  defer: true,
+  rolling: false
+};
 
-if (config.debug) {
-  session = connect.cookieSession({
-    secret: config.sessionSecret,
-    key: key,
-    cookie: cookie
-  });
-} else {
-  session = connect.session({
-    key: key,
-    secret: config.sessionSecret,
-    store: config.sessionStore || new RedisStore(config.redis),
-    cookie: cookie,
-  });
+if (!config.debug) {
+  options.store = config.sessionStore || middlewares.RedisStore(config.redis);
 }
 
-module.exports = session;
+module.exports = middlewares.session(options);

config/index.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - config/index.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,18 +17,20 @@
 
 var path = require('path');
 var fs = require('fs');
+var os = require('os');
 var mkdirp = require('mkdirp');
 
 fs.existsSync = fs.existsSync || path.existsSync;
-var pkg = require('../package.json');
+var version = require('../package.json').version;
 
 var root = path.dirname(__dirname);
 
 var config = {
-  version: pkg.version,
+  version: version,
   registryPort: 7001,
   webPort: 7002,
   enableCluster: false,
+  numCPUs: os.cpus().length,
   debug: true, // if debug
   logdir: path.join(root, '.tmp', 'logs'),
   viewCache: false,
@@ -51,6 +53,7 @@ var config = {
     port: 19533,
     pass: 'cnpmjs_dev'
   },
+  jsonLimit: '10mb', // max request json body size
   uploadDir: path.join(root, 'public', 'dist'),
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
   qn: {
@@ -71,6 +74,7 @@ var config = {
     debug: false
   },
 
+  disturl: 'http://dist.u.qiniudn.com',
   logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
   registryHost: 'r.cnpmjs.org',
   customFooter: '', // you can add copyright and site total script html here
@@ -79,8 +83,8 @@ var config = {
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
-    admin: 'admin@cnpmjs.org',
     fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
     dead_horse: 'dead_horse@qq.com',
     cnpmjstest10: 'cnpmjstest10@cnpmjs.org',
   },
@@ -88,6 +92,17 @@ var config = {
   backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
   syncModel: 'none', // 'none', 'all', 'exist'
   syncConcurrency: 1,
+  maxDependencies: 200, // max handle number of package.json `dependencies` property
+
+  limit: {
+    enable: false,
+    token: 'koa-limit:download',
+    limit: 1000,
+    interval: 1000 * 60 * 60 * 24,
+    whiteList: [],
+    blackList: [],
+    message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
+  }
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js

controllers/download.js

@@ -14,10 +14,15 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var moment = require('moment');
 var DownloadTotal = require('../proxy/download');
 
 exports.total = function (name, callback) {
+  if (typeof name === 'function') {
+    callback = name;
+    name = null;
+  }
   var end = moment();
   var start = end.clone().subtract('months', 1).startOf('month');
   var lastday = end.clone().subtract('days', 1).format('YYYY-MM-DD');
@@ -74,3 +79,5 @@ exports.total = function (name, callback) {
 
   DownloadTotal[method].apply(DownloadTotal, args);
 };
+
+thunkify(exports);

controllers/registry/user.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - controllers/registry/user.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,147 +16,198 @@
  */
 
 var debug = require('debug')('cnpmjs.org:controllers:registry');
-var logger = require('../../common/logger');
+var utility = require('utility');
+var crypto = require('crypto');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
+var config = require('../../config');
 
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  User.get(name, function (err, row) {
-    if (err) {
-      return next(err);
-    }
-    if (!row) {
-      return next();
-    }
+exports.show = function *(next) {
+  var name = this.params.name;
+  var user = yield User.get(name);
+  if (!user) {
+    return yield *next;
+  }
 
-    res.setHeader('etag', '"' + row.rev + '"');
-    var data = {
-      _id: 'org.couchdb.user:' + row.name,
-      _rev: row.rev,
-      name: row.name,
-      email: row.email,
+  var data = user.json;
+  if (!data) {
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
       type: 'user',
       roles: [],
-      date: row.gmt_modified,
+      date: user.gmt_modified,
     };
-    res.json(data);
-  });
+  }
+  data._cnpm_meta = {
+    id: user.id,
+    npm_user: user.npm_user,
+    gmt_create: user.gmt_create,
+    gmt_modified: user.gmt_modified,
+    admin: !!config.admins[user.name],
+  };
+
+  this.body = data;
 };
 
+function ensurePasswordSalt(user, body) {
+  if (!user.password_sha && body.password) {
+    // create password_sha on server
+    user.salt = crypto.randomBytes(30).toString('hex');
+    user.password_sha = utility.sha1(body.password + user.salt);
+  }
+}
+
+// npm 1.4.4
+// add new user first
+// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'accept-encoding': 'gzip',
+//   'user-agent': 'node/v0.11.12 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '150',
+//   connection: 'close' } { name: 'mk2',
+//   password: '123456',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:33:19.465Z' }
+
+// old npm flow
 // json:
-//  { name: 'fengmk2',
-//    salt: 'xxxx',
-//    password_sha: 'xxxxxx',
-//    email: 'fengmk2@gmail.com',
-//    _id: 'org.couchdb.user:fengmk2',
-//    type: 'user',
-//    roles: [],
-//    date: '2013-12-04T12:56:13.714Z' } }
-exports.add = function (req, res, next) {
-  var name = req.params.name;
-  var body = req.body || {};
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'user-agent': 'node/v0.8.26 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '258',
+//   connection: 'keep-alive' }
+// { name: 'mk2',
+//   salt: '18d8d51936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
+//   password_sha: '8f4408912a6db1d96b132a90856d99db029cef3d',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:39:25.696Z' }
+exports.add = function *() {
+  var name = this.params.name;
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     // roles: body.roles || [],
   };
+
+  ensurePasswordSalt(user, body);
+
   if (!user.name || !user.salt || !user.password_sha || !user.email) {
-    return res.json(422, {
+    this.status = 422;
+    this.body = {
       error: 'paramError',
-      reason: 'params missing'
-    });
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
   }
   debug('add user: %j', user);
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  User.get(name, ep.doneLater(function (row) {
-    if (row) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    User.add(user, ep.done('add'));
-  }));
+  var existUser = yield User.get(name);
+  if (existUser) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'User ' + name + ' already exists.'
+    };
+    return;
+  }
 
-  ep.once('add', function (result) {
-    res.setHeader('etag', '"' + result.rev + '"');
-    // location: 'http://registry.npmjs.org/_users/org.couchdb.user:cnpmjstest1',
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.add(user);
+  this.etag = '"' + result.rev + '"';
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
 };
 
-exports.authSession = function (req, res, next) {
+exports.authSession = function *() {
   // body: {"name":"foo","password":"****"}
-  var body = req.body || {};
+  var body = this.request.body || {};
   var name = body.name;
   var password = body.password;
-  User.auth(name, password, function (err, user) {
-    debug('authSession %s: %j', name, user);
-    if (err) {
-      return next(err);
-    }
-    if (!user) {
-      return res.json(401, {ok: false, name: null, roles: []});
-    }
+  var user = yield User.auth(name, password);
+  debug('authSession %s: %j', name, user);
 
-    req.session.name = user.name;
-    res.json(200, {ok: true, name: user.name, roles: []});
-  });
+  if (!user) {
+    this.status = 401;
+    this.body = {ok: false, name: null, roles: []};
+    return;
+  }
+  var session = yield *this.session;
+  session.name = user.name;
+  this.body = {ok: true, name: user.name, roles: []};
 };
 
-exports.update = function (req, res, next) {
-  var name = req.params.name;
-  var rev = req.params.rev;
+exports.update = function *(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
   if (!name || !rev) {
-    return next();
+    return yield* next;
   }
+  debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
 
-  debug('update: %s, rev: %s, session.name: %s', name, rev, req.session.name);
-
-  if (name !== req.session.name) {
-    // must authSession first
-    res.statusCode = 401;
-    return res.json({
+  if (name !== this.user.name) {
+    // must auth user first
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Name is incorrect.'
-    });
+    };
+    return;
   }
 
-  var body = req.body || {};
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     rev: body.rev || body._rev,
     // roles: body.roles || [],
   };
-  User.update(user, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    //check rev error
-    if (!result) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + user.name,
-      rev: result.rev
-    });
-  });
+
+  ensurePasswordSalt(user, body);
+
+  if (!user.name || !user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    this.body = {
+      error: 'paramError',
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
+  }
+
+  var result = yield User.update(user);
+  if (!result) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
 };

controllers/sync.js

@@ -17,48 +17,59 @@
 var Log = require('../proxy/module_log');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 
-exports.sync = function (req, res, next) {
-  var username = req.session.name || 'anonymous';
-  var name = req.params.name;
-  var publish = req.query.publish === 'true';
-  var noDep = req.query.nodeps === 'true';
-  if (publish && !req.session.isAdmin) {
-    return res.json(403, {
+exports.sync = function *() {
+  var username = this.user.name || 'anonymous';
+  var name = this.params.name;
+  var publish = this.query.publish === 'true';
+  var noDep = this.query.nodeps === 'true';
+  if (publish && !this.user.isAdmin) {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Only admin can publish'
-    });
+    };
+    return;
   }
 
   var options = {
     publish: publish,
     noDep: noDep,
   };
-  SyncModuleWorker.sync(name, username, options, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    if (!result.ok) {
-      return res.json(result.statusCode, result.pkg);
-    }
-    res.json(201, {
-      ok: true,
-      logId: result.logId
-    });
-  });
+
+  var result = yield SyncModuleWorker.sync(name, username, options);
+
+  // friendly 404 reason info
+  if (result.staticCache === 404) {
+    this.status = 404;
+    this.body = {
+      ok: false,
+      reason: 'can not found ' + name + ' in the source registry'
+    };
+    return;
+  }
+  if (!result.ok) {
+    this.status = result.statusCode;
+    this.body = result.pkg;
+    return;
+  }
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logId: result.logId
+  };
 };
 
-exports.getSyncLog = function (req, res, next) {
-  var logId = req.params.id;
-  var name = req.params.name;
-  var offset = Number(req.query.offset) || 0;
-  Log.get(logId, function (err, row) {
-    if (err || !row) {
-      return next(err);
-    }
-    var log = row.log.trim();
-    if (offset > 0) {
-      log = log.split('\n').slice(offset).join('\n');
-    }
-    res.json(200, {ok: true, log: log});
-  });
+exports.getSyncLog = function *(next) {
+  var logId = this.params.id;
+  var offset = Number(this.query.offset) || 0;
+  var row = yield Log.get(logId);
+  if (!row) {
+    return yield* next;
+  }
+
+  var log = row.log.trim();
+  if (offset > 0) {
+    log = log.split('\n').slice(offset).join('\n');
+  }
+  this.body = {ok: true, log: log};
 };

controllers/total.js

@@ -16,31 +16,25 @@
  */
 
 var microtime = require('microtime');
-var eventproxy = require('eventproxy');
 var Total = require('../proxy/total');
-var down = require('./download');
+var Download = require('./download');
 var version = require('../package.json').version;
 var config = require('../config');
 
 var startTime = '' + microtime.now();
 
-exports.show = function (req, res, next) {
-  var ep = eventproxy.create();
-  ep.fail(next);
+exports.show = function *() {
+  var r = yield [Total.get(), Download.total()];
+  var total = r[0];
+  var download = r[1];
 
-  Total.get(ep.done('total'));
-  down.total(null, ep.done('download'));
-  ep.all('total', 'download', function (total, download) {
-    total.download = download;
-    total.db_name = 'registry';
-    total.instance_start_time = startTime;
-    total.node_version = process.version;
-    total.app_version = version;
-    total.donate = 'https://me.alipay.com/imk2';
-    total.sync_model = config.syncModel;
-    if (req.query.callback) {
-      return res.jsonp(total, req.query.callback);
-    }
-    res.json(total);
-  });
+  total.download = download;
+  total.db_name = 'registry';
+  total.instance_start_time = startTime;
+  total.node_version = process.version;
+  total.app_version = version;
+  total.donate = 'https://me.alipay.com/imk2';
+  total.sync_model = config.syncModel;
+
+  this.body = total;
 };

controllers/web/dist.js

@@ -0,0 +1,23 @@
+/**!
+ * cnpmjs.org - controllers/web/dist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../../config');
+
+exports.redirect = function *(next) {
+  var params = this.params;
+  var url = config.disturl + (params[0] || '/');
+  this.redirect(url);
+};

controllers/web/package.js

@@ -26,136 +26,152 @@ var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var ModuleDeps = require('../../proxy/module_deps');
 var setDownloadURL = require('../../lib/common').setDownloadURL;
+var ModuleStar = require('../../proxy/module_star');
 
-exports.display = function (req, res, next) {
-  var params = req.params;
+exports.display = function *(next) {
+  var params = this.params;
   var name = params.name;
   var tag = params.version;
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  if (tag) {
-    var version = semver.valid(tag);
-    if (version) {
-      Module.get(name, version, ep.done('pkg'));
-    } else {
-      Module.getByTag(name, tag, ep.done('pkg'));
-    }
+  var getPackageMethod;
+  var getPackageArgs;
+  var version = semver.valid(tag || '');
+  if (version) {
+    getPackageMethod = 'get';
+    getPackageArgs = [name, version];
   } else {
-    Module.getByTag(name, 'latest', ep.done('pkg'));
+    getPackageMethod = 'getByTag';
+    getPackageArgs = [name, tag || 'latest'];
+  }
+  var r = yield [
+    Module[getPackageMethod].apply(Module, getPackageArgs),
+    down.total(name),
+    ModuleDeps.list(name),
+    ModuleStar.listUsers(name),
+  ];
+  var pkg = r[0];
+  var download = r[1];
+  var dependents = (r[2] || []).map(function (item) {
+    return item.deps;
+  });
+  var users = r[3];
+
+  if (!pkg || !pkg.package) {
+    return yield* next;
   }
 
-  down.total(name, ep.done('download'));
+  pkg.package.fromNow = moment(pkg.publish_time).fromNow();
+  pkg = pkg.package;
+  pkg.users = users;
+  pkg.readme = marked(pkg.readme || '');
+  if (!pkg.readme) {
+    pkg.readme = pkg.description || '';
+  }
 
-  ep.all('pkg', 'download', function (pkg, download) {
-    if (!pkg || !pkg.package) {
-      return next();
-    }
-
-    pkg.package.fromNow = moment(pkg.publish_time).fromNow();
-    pkg = pkg.package;
-    pkg.readme = marked(pkg.readme || '');
-    if (pkg.maintainers) {
-      for (var i = 0; i < pkg.maintainers.length; i++) {
-        var maintainer = pkg.maintainers[i];
-        if (maintainer.email) {
-          maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
-        }
+  if (pkg.maintainers) {
+    for (var i = 0; i < pkg.maintainers.length; i++) {
+      var maintainer = pkg.maintainers[i];
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
       }
     }
+  }
 
-    if (pkg.contributors) {
-      // registry.cnpmjs.org/compressible
-      if (!Array.isArray(pkg.contributors)) {
-        pkg.contributors = [pkg.contributors];
+  if (pkg.contributors) {
+    // registry.cnpmjs.org/compressible
+    if (!Array.isArray(pkg.contributors)) {
+      pkg.contributors = [pkg.contributors];
+    }
+    for (var i = 0; i < pkg.contributors.length; i++) {
+      var contributor = pkg.contributors[i];
+      if (contributor.email) {
+        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
       }
-      for (var i = 0; i < pkg.contributors.length; i++) {
-        var contributor = pkg.contributors[i];
-        if (contributor.email) {
-          contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
-        }
-        if (config.packagePageContributorSearch || !contributor.url) {
-          contributor.url = '/~' + encodeURIComponent(contributor.name);
-        }
+      if (config.packagePageContributorSearch || !contributor.url) {
+        contributor.url = '/~' + encodeURIComponent(contributor.name);
       }
     }
+  }
 
-    if (pkg.repository && pkg.repository.url) {
-      pkg.repository.weburl = giturl.parse(pkg.repository.url);
-    }
+  if (pkg.repository && pkg.repository.url) {
+    pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
+  }
 
-    setLicense(pkg);
+  setLicense(pkg);
 
-    for (var k in download) {
-      download[k] = humanize(download[k]);
-    }
+  for (var k in download) {
+    download[k] = humanize(download[k]);
+  }
+  setDownloadURL(pkg, this, config.registryHost);
 
-    setDownloadURL(pkg, req, config.registryHost);
+  pkg.dependents = dependents;
 
-    res.render('package', {
-      title: 'Package - ' + pkg.name,
-      package: pkg,
-      download: download
-    });
+  yield this.render('package', {
+    title: 'Package - ' + pkg.name,
+    package: pkg,
+    download: download
   });
 };
 
-exports.search = function (req, res, next) {
-  var params = req.params;
-  var word = req.params.word;
-  Module.search(word, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-
-    res.render('search', {
-      title: 'Keyword - ' + word,
+exports.search = function *(next) {
+  var params = this.params;
+  var word = params.word;
+  var result = yield Module.search(word);
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.body = {
       keyword: word,
       packages: result.searchMatchs,
-      keywords: result.keywordMatchs,
-    });
+      keywords: result.keywordMatchs
+    };
+    this.type = 'application/json; charset=utf-8';
+    return;
+  }
+
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
   });
 };
 
-exports.rangeSearch = function (req, res, next) {
-  var startKey = req.query.startkey || '';
+exports.rangeSearch = function *(next) {
+  var startKey = this.query.startkey || '';
   if (startKey[0] === '"') {
     startKey = startKey.substring(1);
   }
   if (startKey[startKey.length - 1] === '"') {
     startKey = startKey.substring(0, startKey.length - 1);
   }
-  var limit = Number(req.query.limit) || 20;
-  Module.search(startKey, {limit: limit}, function (err, result) {
-    if (err) {
-      return next(err);
-    }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield Module.search(startKey, {limit: limit});
 
-    var packages = result.searchMatchs.concat(result.keywordMatchs);
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
 
-    var rows = [];
-    for (var i = 0; i < packages.length; i++) {
-      var p = packages[i];
-      var row = {
-        key: p.name,
-        count: 1,
-        value: {
-          name: p.name,
-          description: p.description,
-        }
-      };
-      rows.push(row);
-    }
-    res.json({
-      rows: rows
-    });
-  });
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
 };
 
-exports.displaySync = function (req, res, next) {
-  var name = req.params.name || req.query.name;
-  res.render('sync', {
+exports.displaySync = function *(next) {
+  var name = this.params.name || this.query.name;
+  yield this.render('sync', {
     name: name,
     title: 'Sync - ' + name
   });

controllers/web/user.js

@@ -15,31 +15,24 @@
  */
 var Module = require('../../proxy/module');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.display = function (req, res, next) {
-  var name = req.params.name;
+exports.display = function *(next) {
+  var name = this.params.name;
 
-  var ep = eventproxy.create();
-  ep.fail(next);
-  Module.listByAuthor(name, ep.done('packages'));
-  User.get(name, ep.done('user'));
+  var r = yield [Module.listByAuthor(name), User.get(name)];
+  var packages = r[0];
+  var user = r[1];
+  if (!user && !packages.length) {
+    return yield* next;
+  }
+  user = {
+    name: name,
+    email: user && user.email
+  };
 
-  ep.all('packages', 'user', function (packages, user) {
-    //because of sync, maybe no this user in database,
-    //but his packages in this registry
-    if (!user && !packages.length) {
-      return next();
-    }
-    user = {
-      name: name,
-      email: user && user.email
-    };
-
-    return res.render('profile', {
-      title: 'User - ' + name,
-      packages: packages || [],
-      user: user
-    });
+  yield this.render('profile', {
+    title: 'User - ' + name,
+    packages: packages || [],
+    user: user
   });
 };

dispatch.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - dispatch.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,7 +17,6 @@
 
 var path = require('path');
 var util = require('util');
-var fs = require('fs');
 var cluster = require('cluster');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
@@ -30,25 +29,25 @@ if (config.enableCluster) {
   });
 
   cluster.on('fork', function (worker) {
-    console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
+    console.log('[%s] [worker:%d] new worker start', Date(), worker.process.pid);
   });
 
   cluster.on('disconnect', function (worker) {
     var w = cluster.fork();
-    console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
-      new Date(), process.pid, worker.process.pid, w.process.pid);
+    console.error('[%s] [master:%s] wroker:%s disconnect, suicide: %s, state: %s. New worker:%s fork',
+      Date(), process.pid, worker.process.pid, worker.suicide, worker.state, w.process.pid);
   });
 
   cluster.on('exit', function (worker, code, signal) {
     var exitCode = worker.process.exitCode;
-    var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
+    var err = new Error(util.format('worker %s died (code: %s, signal: %s, suicide: %s, state: %s)',
+      worker.process.pid, exitCode, signal, worker.suicide, worker.state));
     err.name = 'WorkerDiedError';
-    console.error(err);
+    console.error('[%s] [master:%s] wroker exit: %s', Date(), process.pid, err.stack);
   });
 
-  var numCPUs = require('os').cpus().length;
   // Fork workers.
-  for (var i = 0; i < numCPUs; i++) {
+  for (var i = 0; i < config.numCPUs; i++) {
     cluster.fork();
   }
 

docs/db.sql

@@ -9,10 +9,15 @@ CREATE TABLE `user` (
  `roles` varchar(200) NOT NULL DEFAULT '[]',
  `rev` varchar(40) NOT NULL,
  `email` varchar(400) NOT NULL,
+ `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+ `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
  PRIMARY KEY (`id`),
  UNIQUE KEY `name` (`name`),
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
+-- ALTER TABLE `user`
+--   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+--   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
 
 CREATE TABLE `module_keyword` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
@@ -25,6 +30,16 @@ CREATE TABLE `module_keyword` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
 
+CREATE TABLE `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
+
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
@@ -112,3 +127,13 @@ CREATE TABLE `download_total` (
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
 -- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

docs/publish.md

@@ -1,5 +1,24 @@
 # npm publish flow
 
+## Flows
+
+1. try to put package.json and tgz, maybe base64 tgz body
+2. if new version not exists, publish success
+3. if new version exists, 409, try to get full package info with ?write=true
+4. if new version had publish, show: "Update the 'version' field in package.json and try again."
+
+```bash
+$ cnpm publish
+npm http PUT http://r.cnpmjs.org/cnpmjs.org
+npm http 409 http://r.cnpmjs.org/cnpmjs.org
+npm http GET http://r.cnpmjs.org/cnpmjs.org?write=true
+npm http 200 http://r.cnpmjs.org/cnpmjs.org?write=true
+npm ERR! publish fail Cannot publish over existing version.
+npm ERR! publish fail Update the 'version' field in package.json and try again.
+```
+
+## Details
+
 code: https://github.com/isaacs/npm-registry-client/blob/master/lib/publish.js
 
 * couch login if token not exists: [couch-login](https://github.com/isaacs/couch-login)

docs/web/install.md

@@ -103,13 +103,13 @@ npm install -g cnpm
 alias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
- --disturl=http://dist.u.qiniudn.com\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'
 
  #or put this in .zshrc or .bashrc
  echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
- --disturl=http://dist.u.qiniudn.com\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
 ```

docs/web/readme.md

@@ -1,14 +1,15 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
 ## What is this?
 
-> Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
+> Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
+* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ## Install your private npm registry
 
@@ -121,13 +122,13 @@ alias it:
 ```bash
 alias cnpm="npm --registry=http://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=http://dist.u.qiniudn.com \
+--disturl=http://cnpmjs.org/dist \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
 $ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=http://dist.u.qiniudn.com \
+  --disturl=http://cnpmjs.org/dist \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -139,7 +140,7 @@ $ npm install cnpm -g
 
 ### install
 
-Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
 ```
 $ cnpm install [name]

index.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - index.js
  *
  * Copyright(c) cnpmjs.org and other contributors.

lib/common.js

@@ -29,11 +29,11 @@ exports.getCDNKey = function (name, filename) {
   return '/' + name + '/-/' + filename;
 };
 
-exports.setDownloadURL = function (pkg, req, host) {
+exports.setDownloadURL = function (pkg, ctx, host) {
   if (pkg.dist) {
-    host = host || req.headers.host;
+    host = host || ctx.host;
     pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
-      req.connection.encrypted ? 'https' : 'http',
+      ctx.protocol,
       host, pkg.name, pkg.name, pkg.version);
   }
 };
@@ -42,12 +42,12 @@ exports.isAdmin = function (username) {
   return typeof config.admins[username] === 'string';
 };
 
-exports.isMaintainer = function (req, maintainers) {
-  if (req.session.isAdmin) {
+exports.isMaintainer = function (user, maintainers) {
+  if (user.isAdmin) {
     return true;
   }
 
-  var username = req.session.name;
+  var username = user.name;
   maintainers = maintainers || [];
   var match = maintainers.filter(function (item) {
     return item.name === username;

middleware/auth.js

@@ -20,50 +20,41 @@ var config = require('../config');
 var common = require('../lib/common');
 
 module.exports = function (options) {
-  return function auth(req, res, next) {
-    if (!req.session) {
-      // redis crash
-      req.session = {};
-      return next();
+  return function *auth(next) {
+    var session = yield *this.session;
+    debug('%s, %s, %j', this.url, this.sessionId, session);
+    this.user = {};
+
+    if (session.name) {
+      this.user.name = session.name;
+      this.user.isAdmin = common.isAdmin(session.name);
+      debug('auth exists user: %j, headers: %j', this.user, this.header);
+      return yield *next;
     }
-    req.session.onlySync = config.enablePrivate ? true : false;
-    if (req.session.name) {
-      req.session.isAdmin = common.isAdmin(req.session.name);
-      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
-        req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
-      return next();
-    }
-    var authorization = (req.headers.authorization || '').split(' ')[1] || '';
+
+    var authorization = (this.get('authorization') || '').split(' ')[1] || '';
     authorization = authorization.trim();
     if (!authorization) {
-      return next();
+      return yield *next;
     }
 
     authorization = new Buffer(authorization, 'base64').toString().split(':');
     if (authorization.length !== 2) {
-      return next();
+      return yield *next;
     }
 
     var username = authorization[0];
     var password = authorization[1];
 
-    User.auth(username, password, function (err, row) {
-      if (err) {
-        return next(err);
-      }
+    var row = yield User.auth(username, password);
+    if (!row) {
+      debug('auth fail user: %j, headers: %j', row, this.header);
+      return yield *next;
+    }
 
-      if (!row) {
-        debug('auth fail user: %j, headers: %j', row, req.headers);
-        req.session.name = null;
-        req.session.isAdmin = false;
-        return next();
-      }
-
-      req.session.name = row.name;
-      req.session.isAdmin = common.isAdmin(req.session.name);
-      debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
-        row, req.session.onlySync, req.session.isAdmin, req.headers);
-      next();
-    });
+    this.user.name = row.name;
+    this.user.isAdmin = common.isAdmin(row.name);
+    debug('auth pass user: %j, headers: %j', this.user, this.header);
+    yield *next;
   };
 };

middleware/limit.js

@@ -0,0 +1,34 @@
+/**!
+ * cnpmjs.org - middleware/limit.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+var limit = require('koa-limit');
+var store = require('../common/redis');
+
+var limitConfig = config.limit;
+
+if (!limitConfig.enable) {
+  module.exports = function *ignoreLimit(next) {
+    yield *next;
+  };
+} else {
+
+  if (!config.debug) {
+    limitConfig.store = store;
+  }
+
+  module.exports = limit(limitConfig);
+}

middleware/login.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
-module.exports = function login(req, res, next) {
-  if (!req.session.name) {
-    return res.json(401, {
+module.exports = function *login(next) {
+  if (!this.user.name) {
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Login first.'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/opensearch.js

@@ -24,8 +24,11 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
 
 var lastModifyDate = new Date();
 
-module.exports = function publishable(req, res, next) {
-  res.charset = res.charset || 'utf-8';
-  res.setHeader('Content-Type', 'text/xml');
-  res.send(template.replace('${host}', req.headers.host));
+module.exports = function *publishable(next) {
+  if (this.path === '/opensearch.xml') {
+    this.type = 'text/xml';
+    this.charset = 'utf-8';
+    this.body = template.replace('${host}', this.host);
+  }
+  yield *next;
 };

middleware/publishable.js

@@ -14,13 +14,17 @@
  * Module dependencies.
  */
 
-module.exports = function publishable(req, res, next) {
-  if (req.session.onlySync && !req.session.isAdmin) {
+var config = require('../config');
+
+module.exports = function *publishable(next) {
+  if (config.enablePrivate && !this.user.isAdmin) {
     // private mode, only admin user can publish
-    return res.json(403, {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Private mode enable, only admin can publish this module'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/registry_not_found.js

@@ -0,0 +1,28 @@
+/**!
+ * cnpmjs.org - middleware/registry_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+  this.status = 404;
+  this.body = {
+    error: 'not_found',
+    reason: 'document not found'
+  };
+};

middleware/sync_by_install.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - middleware/sync_by_install.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -14,32 +14,27 @@
  * Module dependencies.
  */
 
-var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
 var config = require('../config');
 
 /**
- * req.session.allowSync  -  allow sync triggle by cnpm install
+ * this.allowSync  -  allow sync triggle by cnpm install
  */
 
-module.exports = function (req, res, next) {
+module.exports = function *syncByInstall(next) {
   if (!config.syncByInstall || !config.enablePrivate) {
     // only config.enablePrivate should enable sync on install
-    return next();
+    return yield *next;
   }
   // request not by node, consider it request from web
-  if (req.headers['user-agent'] && req.headers['user-agent'].indexOf('node') !== 0) {
-    return next();
+  var ua = this.get('user-agent');
+  if (!ua || ua.indexOf('node') < 0) {
+    return yield *next;
   }
 
-  req.session.allowSync = true;
-  if (req.session.isAdmin) {
-    // if current user is admin, should not enable auto sync on install, because it would be unpublish
-    req.session.allowSync = false;
+  if (this.query.write) {
+    return yield *next;
   }
 
-  // TODO: allow sync will let publish sync package...
-  req.session.allowSync = false;
-
-  debug('%s allowSync: %s', req.session.name, req.session.allowSync);
-  next();
+  this.allowSync = true;
+  yield *next;
 };

middleware/web_not_found.js

@@ -0,0 +1,33 @@
+/**!
+ * cnpmjs.org - middleware/web_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+
+  var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  if (m) {
+    return this.redirect('/package/' + m[1]);
+  }
+
+  this.status = 404;
+  this.type = 'text/html';
+  this.charset = 'utf-8';
+  this.body = 'Cannot GET ' + this.path;
+};

package.json

@@ -1,69 +1,64 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.2.24",
+  "version": "0.3.10",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
-    "test": "make test-all",
+    "test": "make install && make jshint && make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
     "stop": "./bin/nodejsctl stop"
   },
   "config": {
-    "blanket": {
-      "pattern": "//^((?!(node_modules|test|common)).)*$/",
-      "data-cover-flags": {
-        "debug": false
-      }
-    },
-    "travis-cov": {
-      "threshold": 90
+    "cov": {
+      "threshold": 83
     }
   },
   "dependencies": {
-    "bagpipe": "0.3.5",
-    "connect": "2.12.0",
-    "connect-markdown": "0.0.3",
-    "connect-redis": "1.4.6",
-    "connect-render": "0.3.2",
-    "connect-rt": "0.0.2",
+    "co": "3.0.4",
+    "co-gather": "0.0.1",
+    "co-read": "0.0.1",
+    "co-redis": "1.1.0",
+    "co-urllib": "0.1.3",
+    "co-write": "0.3.0",
     "debug": "0.7.4",
-    "eventproxy": "0.2.6",
-    "forward": "0.0.4",
-    "giturl": "0.0.1",
-    "graceful": "0.0.5",
+    "eventproxy": "0.3.0",
+    "giturl": "0.0.2",
+    "graceful": "0.0.6",
     "gravatar": "1.0.6",
     "humanize-number": "0.0.2",
+    "koa": "0.5.1",
+    "koa-limit": "1.0.0",
+    "koa-markdown": "0.0.3",
+    "koa-middlewares": "0.0.11",
     "logfilestream": "0.1.0",
-    "marked": "0.3.0",
+    "marked": "0.3.2",
     "microtime": "0.5.1",
     "mime": "1.2.11",
     "mkdirp": "0.3.5",
     "moment": "2.5.1",
     "ms": "0.6.2",
-    "mysql": "2.0.1",
-    "nodemailer": "0.6.0",
-    "qn": "0.2.0",
+    "mysql": "2.1.1",
+    "nodemailer": "0.6.1",
+    "qn": "0.2.1",
     "ready": "0.1.1",
-    "response-cookie": "0.0.2",
-    "response-patch": "0.1.1",
+    "redis": "0.10.1",
     "semver": "2.2.1",
-    "urllib": "0.5.5",
-    "urlrouter": "0.5.4",
-    "utility": "0.1.10"
+    "thunkify-wrap": "0.0.5",
+    "utility": "0.1.11"
   },
   "devDependencies": {
     "autod": ">=0.0.13",
-    "blanket": "*",
+    "chunkstream": "0.0.1",
     "contributors": "*",
-    "coveralls": "*",
-    "mm": "0.1.8",
+    "cov": "*",
+    "istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",
+    "jshint": "*",
+    "mm": "0.2.1",
     "mocha": "*",
-    "mocha-lcov-reporter": "*",
     "pedding": "0.0.3",
-    "should": "3.0.1",
-    "supertest": "0.9.0",
-    "travis-cov": "*"
+    "should": "3.1.3",
+    "supertest": "0.9.0"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
@@ -79,7 +74,7 @@
     "cnpmjs.org", "npm", "npmjs", "npmjs.org", "registry"
   ],
   "engines": {
-    "node": ">= 0.8.0"
+    "node": ">= 0.11.9"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

proxy/download.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var mysql = require('../common/mysql');
 
@@ -41,3 +42,5 @@ var SELECT_ALL_TOTAL_SQL = 'SELECT date, sum(count) AS count \
 exports.getTotal = function (start, end, callback) {
   mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
 };
+
+thunkify(exports);

proxy/module.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
 
-var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, version, description, package, dist_tarball, dist_shasum, dist_size';
+var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, \
+  version, description, package, dist_tarball, dist_shasum, dist_size';
 
 var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
   publish_time, author, name, version, package, dist_tarball, dist_shasum, dist_size, description) \
@@ -175,6 +177,7 @@ function parseRow(row) {
     }
   }
 }
+exports.parseRow = parseRow;
 
 function stringifyPackage(pkg) {
   return encodeURIComponent(JSON.stringify(pkg));
@@ -448,3 +451,12 @@ exports.search = function (word, options, callback) {
     }));
   });
 };
+
+thunkify(exports);
+
+exports.updateMaintainers = function *(id, maintainers) {
+  var mod = yield exports.getById(id);
+  mod.package.maintainers = maintainers;
+  var pkg = stringifyPackage(mod.package);
+  return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
+};

proxy/module_deps.js

@@ -0,0 +1,44 @@
+/**!
+ * cnpmjs.org - proxy/module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var thunkify = require('thunkify-wrap');
+var mysql = require('../common/mysql');
+
+var LIST_DEPS_SQL = 'SELECT deps FROM module_deps WHERE name=?;';
+
+exports.list = function (name, callback) {
+  mysql.query(LIST_DEPS_SQL, [name], callback);
+};
+
+var INSERT_DEPS_SQL = 'INSERT INTO module_deps(gmt_create, name, deps) \
+  VALUES(now(), ?, ?);';
+
+exports.add = function (name, deps, callback) {
+  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
+    if (err && err.code === 'ER_DUP_ENTRY') {
+      err = null;
+    }
+    callback(err);
+  });
+};
+
+var DELETE_DEPS_SQL = 'DELETE FROM module_deps WHERE name=? AND deps=?;';
+
+exports.remove = function (name, deps, callback) {
+  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
+};
+
+thunkify(exports);

proxy/module_log.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var mysql = require('../common/mysql');
 
 var INSERT_LOG_SQL = 'INSERT INTO module_log(gmt_create, gmt_modified, name, username, log) \
@@ -40,3 +41,5 @@ var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?;';
 exports.get = function (id, callback) {
   mysql.queryOne(SELECT_SQL, [id], callback);
 };
+
+thunkify(exports);

proxy/module_star.js

@@ -0,0 +1,48 @@
+/**!
+ * cnpmjs.org - proxy/module_star.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+
+exports.add = function *add(name, user) {
+  var sql = 'INSERT INTO module_star(name, user) VALUES(?, ?);';
+  try {
+    yield mysql.query(sql, [name, user]);
+  } catch (err) {
+    if (err.code !== 'ER_DUP_ENTRY') {
+      throw err;
+    }
+  }
+};
+
+exports.remove = function *(name, user) {
+  var sql = 'DELETE FROM module_star WHERE name = ? AND user = ?;';
+  return yield mysql.query(sql, [name, user]);
+};
+
+exports.listUsers = function *(name) {
+  var sql = 'SELECT user FROM module_star WHERE name = ?;';
+  var rows = yield mysql.query(sql, [name]);
+  return rows.map(function (r) {
+    return r.user;
+  });
+};
+
+exports.listUserModules = function *(user) {
+  var sql = 'SELECT name FROM module_star WHERE user = ?;';
+  return (yield mysql.query(sql, [user])).map(function (r) {
+    return r.name;
+  });
+};

proxy/npm.js

@@ -14,53 +14,63 @@
  * Module dependencies.
  */
 
-var urllib = require('urllib');
+var urllib = require('co-urllib');
 var config = require('../config');
 
-function request(url, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = null;
-  }
+var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+
+function *request(url, options) {
   options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
+  options.headers = {
+    'user-agent': USER_AGENT
+  };
   url = config.sourceNpmRegistry + url;
-  urllib.request(url, options, function (err, data, res) {
-    if (err) {
-      var statusCode = res && res.statusCode || -1;
-      if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
-        err.name = 'NPMServerError';
-        err.message = 'Status ' + statusCode + ', ' + (data && data.toString() || 'empty body');
-      }
+  var r;
+  try {
+    r = yield *urllib.request(url, options);
+  } catch (err) {
+    var statusCode = err.status || -1;
+    var data = err.data || '[empty]';
+    if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
+      err.name = 'NPMServerError';
+      err.message = 'Status ' + statusCode + ', ' + data.toString();
     }
-    callback(err, data, res);
-  });
+    throw err;
+  }
+  return r;
 }
 
-exports.get = function (name, callback) {
-  request('/' + name, function (err, data, res) {
-    if (err) {
-      return callback(err);
-    }
-    res = res || {};
-    if (res.statusCode === 404) {
-      data = null;
-    }
-    callback(null, data, res);
+exports.getUser = function *(name) {
+  var url = '/-/user/org.couchdb.user:' + name;
+  var r = yield *request(url);
+  var data = r.data;
+  if (data && !data.name) {
+    data = null;
+  }
+  return data;
+};
+
+exports.get = function *(name) {
+  var r = yield *request('/' + name);
+  var data = r.data;
+  if (r.status === 404) {
+    data = null;
+  }
+  return data;
+};
+
+exports.getAllSince = function *(startkey) {
+  var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, {
+    timeout: 300000
   });
+  return r.data;
 };
 
-exports.getAllSince = function (startkey, callback) {
-  request('/-/all/since?stale=update_after&startkey=' + startkey, {
-    dataType: 'json',
+exports.getShort = function *() {
+  var r = yield *request('/-/short', {
     timeout: 300000
-  }, callback);
-};
-
-exports.getShort = function (callback) {
-  request('/-/short', {
-    dataType: 'json',
-    timeout: 300000
-  }, callback);
+  });
+  return r.data;
 };

proxy/sync_module_worker.js

@@ -15,21 +15,29 @@
  * Module dependencies.
  */
 
+var co = require('co');
+var gather = require('co-gather');
+var thunkify = require('thunkify-wrap');
 var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
 var util = require('util');
 var fs = require('fs');
 var path = require('path');
 var crypto = require('crypto');
-var eventproxy = require('eventproxy');
-var urllib = require('urllib');
+var urllib = require('co-urllib');
 var utility = require('utility');
+var ms = require('ms');
 var nfs = require('../common/nfs');
 var npm = require('./npm');
 var common = require('../lib/common');
 var Module = require('./module');
+var ModuleDeps = require('./module_deps');
 var Log = require('./module_log');
-var ms = require('ms');
+var config = require('../config');
+var ModuleStar = require('./module_star');
+var User = require('./user');
+
+var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -86,9 +94,24 @@ SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
 SyncModuleWorker.prototype.start = function () {
   this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
     this.username, this.names[0], this.concurrency, this.noDep, this._publish);
-  for (var i = 0; i < this.concurrency; i++) {
-    this.next(i);
-  }
+  var self = this;
+  co(function *() {
+    var arr = [];
+    for (var i = 0; i < self.concurrency; i++) {
+      arr.push(self.next(i));
+    }
+    yield arr;
+  })();
+};
+
+SyncModuleWorker.prototype.pushSuccess = function (name) {
+  this.successes.push(name);
+  this.emit('success', name);
+};
+
+SyncModuleWorker.prototype.pushFail = function (name) {
+  this.fails.push(name);
+  this.emit('fail', name);
 };
 
 SyncModuleWorker.prototype.add = function (name) {
@@ -97,10 +120,11 @@ SyncModuleWorker.prototype.add = function (name) {
   }
   this.nameMap[name] = true;
   this.names.push(name);
+  this.emit('add', name);
   this.log('    add dependencies: %s', name);
 };
 
-SyncModuleWorker.prototype.next = function (concurrencyId) {
+SyncModuleWorker.prototype.next = function *(concurrencyId) {
   var name = this.names.shift();
   if (!name) {
     return process.nextTick(this.finish.bind(this));
@@ -108,369 +132,513 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
 
   var that = this;
   that.syncingNames[name] = true;
-  npm.get(name, function (err, pkg, response) {
-    var statusCode = response && response.statusCode || -1;
-    if (!err && !pkg) {
-      err = new Error('Module ' + name + ' not exists, http status ' + statusCode);
-      err.name = 'NpmModuleNotExsitsError';
-    }
-    if (err) {
-      if (statusCode === 404) {
-        that.successes.push(name);
-      } else {
-        that.fails.push(name);
-      }
-      that.log('[error] [%s] get package error: %s', name, err.stack);
-      delete that.syncingNames[name];
-      return that.next(concurrencyId);
+  var pkg;
+  // get from npm
+  try {
+    pkg = yield npm.get(name);
+  } catch (err) {
+    // if 404
+    if (err.res && err.res.statusCode === 404) {
+      that.pushSuccess(name);
+    } else {
+      that.pushFail(name);
     }
+    that.log('[error] [%s] get package error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  if (!pkg) {
+    that.log('[error] [%s] get package error: package not exist', name);
+    delete that.syncingNames[name];
+    yield that.next(concurrencyId);
+    return;
+  }
 
-    that.log('[c#%d] [%s] start...', concurrencyId, name);
-    that._sync(name, pkg, function (err, versions) {
-      delete that.syncingNames[name];
-      if (err) {
-        that.fails.push(name);
-        that.log('[error] [%s] sync error: %s', name, err.stack);
-        return that.next(concurrencyId);
-      }
-      that.log('[%s] synced success, %d versions: %s',
-        name, versions.length, versions.join(', '));
-      that.successes.push(name);
-      that.emit('success', name);
-      that.next(concurrencyId);
-    });
-  });
+  that.log('[c#%d] [%s] start...', concurrencyId, name);
+  var versions;
+  try {
+    versions = yield that._sync(name, pkg);
+  } catch (err) {
+    that.pushFail(name);
+    that.log('[error] [%s] sync error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  that.log('[%s] synced success, %d versions: %s',
+    name, versions.length, versions.join(', '));
+  that.pushSuccess(name);
+  delete that.syncingNames[name];
+  yield that.next(concurrencyId);
 };
 
-SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
+function *_listStarUsers(modName) {
+  var users = yield ModuleStar.listUsers(modName);
+  var userMap = {};
+  users.forEach(function (user) {
+    userMap[user] = true;
+  });
+  return userMap;
+}
+
+function *_addStar(modName, username) {
+  yield ModuleStar.add(modName, username);
+}
+
+function *_saveNpmUser(username) {
+  var user = yield *npm.getUser(username);
+  if (!user) {
+    return;
+  }
+  yield User.saveNpmUser(user);
+}
+
+SyncModuleWorker.prototype._sync = function *(name, pkg) {
   var username = this.username;
   var that = this;
-  var ep = eventproxy.create();
-  ep.fail(callback);
 
   var hasModules = false;
-  Module.listByName(name, ep.done(function (rows) {
-    hasModules = rows.length > 0;
-    var map = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.package || !r.package.dist) {
-        // package json parse error
-        continue;
-      }
+  var result = yield [
+    Module.listByName(name),
+    Module.listTags(name),
+    _listStarUsers(name)
+  ];
+  var moduleRows = result[0];
+  var tagRows = result[1];
+  var existsStarUsers = result[2];
 
-      if (r.package && r.package._publish_on_cnpm) {
-        // publish on cnpm, dont sync this version package
-        that.log('  [%s] publish on local cnpm, don\'t sync', name);
-        ep.unbind();
-        callback(null, []);
-        return;
-      }
-
-      if (r.version === 'next') {
-        continue;
-      }
-      if (!map.latest) {
-        map.latest = r;
-      }
-      map[r.version] = r;
+  hasModules = moduleRows.length > 0;
+  var map = {};
+  for (var i = 0; i < moduleRows.length; i++) {
+    var r = moduleRows[i];
+    if (!r.package || !r.package.dist) {
+      // package json parse error
+      continue;
     }
-    ep.emit('existsMap', map);
-  }));
 
-  Module.listTags(name, ep.done(function (rows) {
-    var tags = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.module_id) {
-        // no module_id, need to sync tags
-        continue;
-      }
-      tags[r.tag] = r.version;
+    if (r.package && r.package._publish_on_cnpm) {
+      // publish on cnpm, dont sync this version package
+      that.log('  [%s] publish on local cnpm, don\'t sync', name);
+      return [];
     }
-    ep.emit('existsTags', tags);
-  }));
+
+    if (r.version === 'next') {
+      continue;
+    }
+    if (!map.latest) {
+      map.latest = r;
+    }
+    map[r.version] = r;
+  }
+
+  var tags = {};
+  for (var i = 0; i < tagRows.length; i++) {
+    var r = tagRows[i];
+    if (!r.module_id) {
+      // no module_id, need to sync tags
+      continue;
+    }
+    tags[r.tag] = r.version;
+  }
 
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
   var missingReadmes = [];
+  var missingStarUsers = [];
+  var npmUsernames = {};
 
-  ep.all('existsMap', 'existsTags', function (map, tags) {
-    var times = pkg.time || {};
-    pkg.versions = pkg.versions || {};
-    var versionNames = Object.keys(times);
-    if (versionNames.length === 0) {
-      versionNames = Object.keys(pkg.versions);
+  // find out all user names
+  for (var v in pkg.versions) {
+    var p = pkg.versions[v];
+    var contributors = p.contributors || [];
+    if (contributors && !Array.isArray(contributors)) {
+      contributors = [contributors];
     }
-    if (versionNames.length === 0) {
-      that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
-      if (!hasModules) {
-        // save a next module
-        var maintainer = pkg.maintainers && pkg.maintainers[0];
-        if (maintainer && maintainer.name) {
-          maintainer = maintainer.name;
-        }
-        if (!maintainer) {
-          maintainer = '-';
-        }
-        var nextMod = {
+
+    var maintainers = p.maintainers || [];
+    if (maintainers && !Array.isArray(maintainers)) {
+      maintainers = [maintainers];
+    }
+
+    maintainers.concat(contributors).forEach(function (m) {
+      npmUsernames[m.name.toLowerCase()] = 1;
+    });
+  }
+
+  // get the missing star users
+  var starUsers = pkg.users || {};
+  for (var k in starUsers) {
+    if (!existsStarUsers[k]) {
+      missingStarUsers.push(k);
+    }
+    npmUsernames[k.toLowerCase()] = 1;
+  }
+  that.log('  [%s] found %d missing star users', name, missingStarUsers.length);
+
+  var times = pkg.time || {};
+  pkg.versions = pkg.versions || {};
+  var versionNames = Object.keys(times);
+  if (versionNames.length === 0) {
+    versionNames = Object.keys(pkg.versions);
+  }
+  if (versionNames.length === 0) {
+    that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
+    if (!hasModules) {
+      // save a next module
+      var maintainer = pkg.maintainers && pkg.maintainers[0];
+      if (maintainer && maintainer.name) {
+        maintainer = maintainer.name;
+      }
+      if (!maintainer) {
+        maintainer = '-';
+      }
+      var nextMod = {
+        name: name,
+        version: 'next',
+        author: maintainer,
+        package: {
           name: name,
           version: 'next',
-          author: maintainer,
-          package: {
-            name: name,
-            version: 'next',
-            description: pkg.description || '',
-            readme: pkg.readme || '',
-            maintainers: pkg.maintainers || {
-              name: maintainer
-            },
+          description: pkg.description || '',
+          readme: pkg.readme || '',
+          maintainers: pkg.maintainers || {
+            name: maintainer
           },
-        };
-        Module.add(nextMod, function (err, result) {
-          that.log('  [%s] save next module, %j, error: %s', name, result, err);
-        });
+        },
+      };
+      try {
+        var result = yield Module.add(nextMod);
+        that.log('  [%s] save next module, %j', name, result);
+      } catch (err) {
+        that.log('  [%s] save next module error %s', err.message);
       }
     }
-    var versions = [];
-    for (var i = 0; i < versionNames.length; i++) {
-      var v = versionNames[i];
-      var exists = map[v] || {};
-      var version = pkg.versions[v];
-      if (!version || !version.dist) {
+  }
+
+  var versions = [];
+  for (var i = 0; i < versionNames.length; i++) {
+    var v = versionNames[i];
+    var exists = map[v] || {};
+    var version = pkg.versions[v];
+    if (!version || !version.dist || !version.dist.tarball) {
+      continue;
+    }
+    //patch for readme
+    if (!version.readme) {
+      version.readme = pkg.readme;
+    }
+    var publish_time = times[v];
+    version.publish_time = publish_time ? Date.parse(publish_time) : null;
+    if (!version.maintainers || !version.maintainers[0]) {
+      version.maintainers = pkg.maintainers;
+    }
+    var sourceAuthor = version.maintainers && version.maintainers[0] &&
+      version.maintainers[0].name || exists.author;
+
+    if (exists.package && exists.package.dist.shasum === version.dist.shasum &&
+      exists.author === sourceAuthor) {
+      // * author make sure equal
+      // * shasum make sure equal
+      if ((version.publish_time === exists.publish_time) ||
+          (!version.publish_time && exists.publish_time)) {
+        // debug('  [%s] %s publish_time equal: %s, %s',
+        //   name, version.version, version.publish_time, exists.publish_time);
+        // * publish_time make sure equal
+        if (exists.description === null && version.description) {
+          // * make sure description exists
+          missingDescriptions.push({
+            id: exists.id,
+            description: version.description
+          });
+        }
+
+        if (!exists.package.readme && version.readme) {
+          // * make sure readme exists
+          missingReadmes.push({
+            id: exists.id,
+            readme: version.readme
+          });
+        }
         continue;
       }
-      //patch for readme
-      if (!version.readme) {
-        version.readme = pkg.readme;
-      }
-      var publish_time = times[v];
-      version.publish_time = publish_time ? Date.parse(publish_time) : null;
-      if (!version.maintainers || !version.maintainers[0]) {
-        version.maintainers = pkg.maintainers;
-      }
-      var sourceAuthor = version.maintainers && version.maintainers[0] && version.maintainers[0].name || exists.author;
-
-      if (exists.package && exists.package.dist.shasum === version.dist.shasum && exists.author === sourceAuthor) {
-        // * author make sure equal
-        // * shasum make sure equal
-        if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
-          debug('  [%s] %s publish_time equal: %s, %s',
-            name, version.version, version.publish_time, exists.publish_time);
-          // * publish_time make sure equal
-          if (exists.description === null && version.description) {
-            // * make sure description exists
-            missingDescriptions.push({
-              id: exists.id,
-              description: version.description
-            });
-          }
-
-          if (!exists.package.readme && version.readme) {
-            // * make sure readme exists
-            missingReadmes.push({
-              id: exists.id,
-              readme: version.readme
-            });
-          }
-          continue;
-        }
-      }
-      versions.push(version);
     }
+    versions.push(version);
+  }
 
-    var sourceTags = pkg['dist-tags'] || {};
-    for (var t in sourceTags) {
-      var sourceTagVersion = sourceTags[t];
-      if (sourceTagVersion && tags[t] !== sourceTagVersion) {
-        missingTags.push([t, sourceTagVersion]);
-      }
-    }
-
-    if (versions.length === 0) {
-      that.log('  [%s] all versions are exists', name);
-      return ep.emit('syncDone');
+  var sourceTags = pkg['dist-tags'] || {};
+  for (var t in sourceTags) {
+    var sourceTagVersion = sourceTags[t];
+    if (sourceTagVersion && tags[t] !== sourceTagVersion) {
+      missingTags.push([t, sourceTagVersion]);
     }
+  }
 
+  if (versions.length === 0) {
+    that.log('  [%s] all versions are exists', name);
+  } else {
     versions.sort(function (a, b) {
       return a.publish_time - b.publish_time;
     });
-    missingVersions = versions;
-    that.log('  [%s] %d versions', name, versions.length);
-    ep.emit('syncModule', missingVersions.shift());
-  });
+    that.log('  [%s] %d versions need to sync', name, versions.length);
+  }
 
+  missingVersions = versions;
   var versionNames = [];
   var syncIndex = 0;
-  ep.on('syncModule', function (syncModule) {
+
+  // sync missing versions
+  while (missingVersions.length) {
     var index = syncIndex++;
-    that._syncOneVersion(index, syncModule, function (err, result) {
-      if (err) {
-        that.log('    [%s:%d] error, version: %s, %s: %s',
-          syncModule.name, index, syncModule.version, err.name, err.message);
-      } else {
-        versionNames.push(syncModule.version);
-      }
+    var syncModule = missingVersions.shift();
+    if (!syncModule.dist.tarball) {
+      continue;
+    }
+    try {
+      var result = yield that._syncOneVersion(index, syncModule);
+      versionNames.push(syncModule.version);
+    } catch (err) {
+      that.log('    [%s:%d] error, version: %s, %s: %s',
+        syncModule.name, index, syncModule.version, err.name, err.message);
+    }
+  }
 
-      var nextVersion = missingVersions.shift();
-      if (!nextVersion) {
-        return ep.emit('syncDone', result);
-      }
-      ep.emit('syncModule', nextVersion);
-    });
-  });
-
-  ep.on('syncDone', function () {
+  // sync missing descriptions
+  function *syncDes() {
     if (missingDescriptions.length === 0) {
-      return ep.emit('descriptionDone');
+      return;
     }
-
     that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
-    missingDescriptions.forEach(function (item) {
-      Module.updateDescription(item.id, item.description, function (err, result) {
-        if (err) {
-          that.log('    save error, id： %s, description: %s, error: %s', item.id, item.description, err);
-        } else {
-          that.log('    saved, id: %s, description length: %d', item.id, item.description.length);
-        }
-        ep.emitLater('saveDescription');
-      });
-    });
+    var res = yield gather(missingDescriptions.map(function (item) {
+      return Module.updateDescription(item.id, item.description);
+    }));
 
-    ep.after('saveDescription', missingDescriptions.length, function () {
-      ep.emit('descriptionDone');
-    });
-  });
-
-  ep.on('syncDone', function () {
-    if (missingTags.length === 0) {
-      return ep.emit('tagDone');
+    for (var i = 0; i < res.length; i++) {
+      var item = missingDescriptions[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, description: %s, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    saved, id: %s, description length: %d',
+          item.id, item.description.length);
+      }
     }
+  }
 
+  // sync missing tags
+  function *syncTag() {
+    if (missingTags.length === 0) {
+      return;
+    }
     that.log('  [%s] adding %d tags', name, missingTags.length);
     // sync tags
-    missingTags.forEach(function (item) {
-      Module.addTag(name, item[0], item[1], ep.done(function (result) {
-        that.log('    added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
-        ep.emitLater('addTag');
-      }));
-    });
 
-    ep.after('addTag', missingTags.length, function () {
-      ep.emit('tagDone');
-    });
-  });
+    var res = yield gather(missingTags.map(function (item) {
+      return Module.addTag(name, item[0], item[1]);
+    }));
 
-  ep.once('syncDone', function () {
+    for (var i = 0; i < res.length; i++) {
+      var item = missingTags[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    add tag %s:%s error, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    added tag %s:%s, module_id: %s',
+          item[0], item[1], r.value && r.value.module_id);
+      }
+    }
+  }
+
+  // sycn missing readme
+  function *syncReadme() {
     if (missingReadmes.length === 0) {
-      return ep.emit('readmeDone');
+      return;
+    }
+    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
+
+    var res = yield gather(missingReadmes.map(function (item) {
+      return Module.updateReadme(item.id, item.readme);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingReadmes[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, error: %s', item.id, r.error.message);
+      } else {
+        that.log('    saved, id: %s', item.id);
+      }
+    }
+  }
+
+  function *syncMissingUsers() {
+    var missingUsers = [];
+    var names = Object.keys(npmUsernames);
+    if (names.length === 0) {
+      return;
+    }
+    var rows = yield *User.listByNames(names);
+    var map = {};
+    rows.forEach(function (r) {
+      map[r.name] = r;
+    });
+    names.forEach(function (username) {
+      var r = map[username];
+      if (!r || !r.json) {
+        missingUsers.push(username);
+      }
+    });
+
+    if (missingUsers.length === 0) {
+      that.log('  [%s] all %d npm users exists', name, names.length);
+      return;
     }
 
-    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
-    missingReadmes.forEach(function (item) {
-      Module.updateReadme(item.id, item.readme, function (err, result) {
-        if (err) {
-          that.log('    save error, id: %s, error: %s', item.id, err);
-        } else {
-          that.log('    saved, id: %s', item.id);
-        }
-        ep.emitLater('saveReadme');
-      });
-    });
+    that.log('  [%s] saving %d/%d missing npm users: %j',
+      name, missingUsers.length, names.length, missingUsers);
+    var res = yield gather(missingUsers.map(function (username) {
+      return _saveNpmUser(username);
+    }));
 
-    ep.after('saveReadme', missingReadmes.length, function () {
-      ep.emit('readmeDone');
-    });
-  });
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    save npm user error, %s', r.error.message);
+      }
+    }
+  }
 
-  ep.all('tagDone', 'descriptionDone', 'readmeDone', function () {
-    // TODO: set latest version
-    callback(null, versionNames);
-  });
+  // sync missing star users
+  function *syncMissingStarUsers() {
+    if (missingStarUsers.length === 0) {
+      return;
+    }
+
+    that.log('  [%s] saving %d star users', name, missingStarUsers.length);
+    var res = yield gather(missingStarUsers.map(function (username) {
+      return _addStar(name, username);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    add star user error, %s', r.error.message);
+      }
+    }
+  }
+
+  yield [syncDes(), syncTag(), syncReadme(), syncMissingStarUsers(), syncMissingUsers()];
+  return versionNames;
 };
 
-SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePackage, callback) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
   var that = this;
   var username = this.username;
   var downurl = sourcePackage.dist.tarball;
   var filename = path.basename(downurl);
   var filepath = common.getTarballFilepath(filename);
   var ws = fs.createWriteStream(filepath);
+
   var options = {
     writeStream: ws,
     followRedirect: true,
     timeout: 600000, // 10 minutes download
+    headers: {
+      'user-agent': USER_AGENT
+    }
   };
-  var ep = eventproxy.create();
-  ep.fail(function (err) {
-    // remove tmp file whatever
-    fs.unlink(filepath, utility.noop);
-    callback(err);
-  });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s',
+  var dependencies = Object.keys(sourcePackage.dependencies || {});
+  var devDependencies = Object.keys(sourcePackage.devDependencies || {});
+
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, ' +
+   'publish on cnpm: %s, dependencies: %d, devDependencies: %d',
     sourcePackage.name, versionIndex, sourcePackage.version,
-    sourcePackage.dist, that.noDep, that._publish);
+    sourcePackage.dist, that.noDep, that._publish,
+    dependencies.length, devDependencies.length);
+
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
+  }
+
+  if (devDependencies.length > config.maxDependencies) {
+    devDependencies = devDependencies.slice(0, config.maxDependencies);
+  }
+
   if (!that.noDep) {
-    for (var k in sourcePackage.dependencies) {
-      that.add(k);
+    for (var i = 0; i < dependencies.length; i++) {
+      that.add(dependencies[i]);
     }
 
-    for (var k in sourcePackage.devDependencies) {
-      that.add(k);
+    for (var i = 0; i < devDependencies.length; i++) {
+      that.add(devDependencies[i]);
     }
   }
 
+  // add module dependence
+  try {
+    yield dependencies.map(function (depName) {
+      return ModuleDeps.add(depName, sourcePackage.name);
+    });
+  } catch (err) {
+    // ignore
+  }
+
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
-  urllib.request(downurl, options, ep.done(function (_, response) {
-    var statusCode = response && response.statusCode || -1;
+
+  try {
+    // get tarball
+    var r = yield *urllib.request(downurl, options);
+    var statusCode = r.status || -1;
     if (statusCode === 404) {
-      // just copy source dist
       shasum = sourcePackage.dist.shasum;
-      return ep.emit('uploadResult', {
+      return yield afterUpload({
         url: downurl
       });
     }
-
     if (statusCode !== 200) {
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
       err.data = sourcePackage;
-      return ep.emit('error', err);
+      throw err;
     }
 
+    // read and check
     var rs = fs.createReadStream(filepath);
-    rs.once('error', function (err) {
-      ep.emit('error', err);
-    });
     rs.on('data', function (data) {
       shasum.update(data);
       dataSize += data.length;
     });
-    rs.on('end', function () {
-      shasum = shasum.digest('hex');
-      if (shasum !== sourcePackage.dist.shasum) {
-        var err = new Error('Download ' + downurl + ' shasum:' + shasum + ' not match ' + sourcePackage.dist.shasum);
-        err.name = 'DownloadTarballShasumError';
-        err.data = sourcePackage;
-        return ep.emit('error', err);
-      }
+    yield thunkify(rs)(); // after end event emit
 
-      var options = {
-        key: common.getCDNKey(sourcePackage.name, filename),
-        size: dataSize,
-        shasum: shasum
-      };
-      nfs.upload(filepath, options, ep.done('uploadResult'));
-    });
-  }));
+    // check shasum
+    shasum = shasum.digest('hex');
+    if (shasum !== sourcePackage.dist.shasum) {
+      var err = new Error('Download ' + downurl + ' shasum:' + shasum +
+        ' not match ' + sourcePackage.dist.shasum);
+      err.name = 'DownloadTarballShasumError';
+      err.data = sourcePackage;
+      throw err;
+    }
 
-  ep.on('uploadResult', function (result) {
+    options = {
+      key: common.getCDNKey(sourcePackage.name, filename),
+      size: dataSize,
+      shasum: shasum
+    };
+    // upload to NFS
+    var result = yield nfs.upload(filepath, options);
+    return yield afterUpload(result);
+  } finally {
     // remove tmp file whatever
     fs.unlink(filepath, utility.noop);
+  }
 
+  function *afterUpload(result) {
     //make sure sync module have the correct author info
     //only if can not get maintainers, use the username
     var author = username;
@@ -505,52 +673,41 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     }
 
     mod.package.dist = dist;
-    Module.add(mod, ep.done(function (result) {
-      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j, publish on cnpm: %s',
-        sourcePackage.name, versionIndex,
-        result.id,
-        author, mod.version, dataSize,
-        new Date(mod.publish_time),
-        that._publish);
-      callback(null, result);
-    }));
-  });
+    var r = yield Module.add(mod);
+
+    that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, ' +
+    'size: %d, publish_time: %j, publish on cnpm: %s',
+    sourcePackage.name, versionIndex,
+    r.id,
+    author, mod.version, dataSize,
+    new Date(mod.publish_time),
+    that._publish);
+
+    return r;
+  }
 };
 
-SyncModuleWorker.sync = function (name, username, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = null;
-  }
+SyncModuleWorker.sync = function *(name, username, options) {
   options = options || {};
-  npm.get(name, function (err, pkg, response) {
-    if (err) {
-      return callback(err);
-    }
-    if (!pkg || !pkg._rev) {
-      return callback(null, {
-        ok: false,
-        statusCode: response.statusCode,
-        pkg: pkg
-      });
-    }
-    Log.create({name: name, username: username}, function (err, result) {
-      if (err) {
-        return callback(err);
-      }
-      var worker = new SyncModuleWorker({
-        logId: result.id,
-        name: name,
-        username: username,
-        noDep: options.noDep,
-        publish: options.publish,
-      });
-      worker.start();
-      callback(null, {
-        ok: true,
-        logId: result.id,
-        pkg: pkg
-      });
-    });
+  var pkg = yield npm.get(name);
+  if (!pkg || !pkg._rev) {
+    return {
+      ok: false,
+      pkg: pkg
+    };
+  }
+  var result = yield Log.create({name: name, username: username});
+  var worker = new SyncModuleWorker({
+    logId: result.id,
+    name: name,
+    username: username,
+    noDep: options.noDep,
+    publish: options.publish,
   });
+  worker.start();
+  return {
+    ok: true,
+    logId: result.id,
+    pkg: pkg
+  };
 };

proxy/total.js

@@ -14,9 +14,10 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
+var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var eventproxy = require('eventproxy');
 
 var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
   FROM information_schema.tables \
@@ -118,3 +119,5 @@ exports.updateSyncNum = function (params, callback) {
   ];
   mysql.query(UPDATE_SYNC_NUM_SQL, query, callback);
 };
+
+thunkify(exports);

proxy/user.js

@@ -14,31 +14,32 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var crypto = require('crypto');
 
-var COLUMNS = 'id, rev, name, email, salt, password_sha, ip, roles, gmt_create, gmt_modified';
+var COLUMNS = 'id, rev, name, email, salt, password_sha, ip, roles, json, \
+  npm_user, gmt_create, gmt_modified';
 var SELECT_USER_SQL = 'SELECT ' + COLUMNS + ' FROM user WHERE name=?;';
 
-function sha1(s) {
-  return crypto.createHash("sha1").update(s).digest("hex");
-}
-
 function passwordSha(password, salt) {
-  return sha1(password + salt);
+  return utility.sha1(password + salt);
 }
-exports.passwordSha = passwordSha;
 
 exports.get = function (name, callback) {
   mysql.queryOne(SELECT_USER_SQL, [name], function (err, row) {
     if (row) {
       try {
-        row.roles = JSON.parse(row.roles);
+        row.roles = row.roles ? JSON.parse(row.roles) : [];
       } catch (e) {
         row.roles = [];
       }
+      try {
+        row.json = row.json ? JSON.parse(row.json) : null;
+      } catch (e) {
+        row.json = null;
+      }
     }
     callback(err, row);
   });
@@ -105,3 +106,28 @@ exports.update = function (user, callback) {
   });
 };
 
+thunkify(exports);
+
+exports.passwordSha = passwordSha;
+
+exports.saveNpmUser = function *(user) {
+  var sql = 'SELECT id, json FROM user WHERE name=?;';
+  var row = yield mysql.queryOne(sql, [user.name]);
+  if (!row) {
+    sql = 'INSERT INTO user(npm_user, json, rev, name, email, salt, password_sha, ip, gmt_create, gmt_modified) \
+      VALUES(1, ?, ?, ?, ?, "0", "0", "0", now(), now());';
+    yield mysql.query(sql, [JSON.stringify(user), user._rev, user.name, user.email]);
+  } else {
+    sql = 'UPDATE user SET json=?, rev=? WHERE id=?;';
+    yield mysql.query(sql, [JSON.stringify(user), user._rev, row.id]);
+  }
+};
+
+exports.listByNames = function *(names) {
+  if (names.length === 0) {
+    return [];
+  }
+
+  var sql = 'SELECT id, name, email, json FROM user where name in (?);';
+  return yield mysql.query(sql, [names]);
+};

routes/registry.js

@@ -15,6 +15,8 @@
  * Module dependencies.
  */
 
+var middlewares = require('koa-middlewares');
+var limit = require('../middleware/limit');
 var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
@@ -24,7 +26,7 @@ var user = require('../controllers/registry/user');
 var sync = require('../controllers/sync');
 
 function routes(app) {
-  app.get('/', total.show);
+  app.get('/', middlewares.jsonp(), total.show);
 
   //before /:name/:version
   //get all modules, for npm search
@@ -34,36 +36,36 @@ function routes(app) {
   app.get('/-/short', mod.listAllModuleNames);
 
   // module
-  app.get('/:name', [syncByInstall], mod.show);
-  app.get('/:name/:version', [syncByInstall], mod.get);
+  app.get('/:name', syncByInstall, mod.show);
+  app.get('/:name/:version', syncByInstall, mod.get);
   // try to add module
-  app.put('/:name', [login, publishable], mod.add);
+  app.put('/:name', login, publishable, mod.add);
 
   // sync from source npm
   app.put('/:name/sync', sync.sync);
   app.get('/:name/sync/log/:id', sync.getSyncLog);
 
-  app.get('/:name/download/:filename', mod.download);
+  // need limit by ip
+  app.get('/:name/download/:filename', limit, mod.download);
 
   // put tarball
   // https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
-  app.put('/:name/-/:filename/-rev/:rev', [login, publishable], mod.upload);
+  app.put('/:name/-/:filename/-rev/:rev', login, publishable, mod.upload);
   // delete tarball
-  app.delete('/:name/download/:filename/-rev/:rev', [login, publishable], mod.removeTar);
+  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
 
   // put package.json to module
-  app.put('/:name/:version/-tag/latest', [login, publishable], mod.updateLatest);
+  app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
 
   // update module, unpublish will PUT this
-  app.put('/:name/-rev/:rev', [login, publishable], mod.removeWithVersions);
-  app.delete('/:name/-rev/:rev', [login, publishable], mod.removeAll);
+  app.put('/:name/-rev/:rev', login, publishable, mod.updateOrRemove);
+  app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
 
   // try to create a new user
   // https://registry.npmjs.org/-/user/org.couchdb.user:fengmk2
   app.put('/-/user/org.couchdb.user::name', user.add);
   app.get('/-/user/org.couchdb.user::name', user.show);
-  app.put('/-/user/org.couchdb.user::name/-rev/:rev', [login], user.update);
-
+  app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, user.update);
   // _session
   app.post('/_session', user.authSession);
 }

routes/web.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - routes/web.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -19,6 +19,7 @@ var pkg = require('../controllers/web/package');
 var user = require('../controllers/web/user');
 var sync = require('../controllers/sync');
 var total = require('../controllers/total');
+var dist = require('../controllers/web/dist');
 
 function routes(app) {
   app.get('/total', total.show);
@@ -34,6 +35,8 @@ function routes(app) {
   app.get('/sync', pkg.displaySync);
 
   app.get('/_list/search/search', pkg.rangeSearch);
+
+  app.get(/^\/dist(\/.+)?/, dist.redirect);
 }
 
 module.exports = routes;

servers/registry.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - servers/registry.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -15,69 +15,54 @@
  * Module dependencies.
  */
 
-require('response-patch');
+var koa = require('koa');
+var app = module.exports = koa();
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var responseCookie = require('response-cookie');
-var urlrouter = require('urlrouter');
-var forward = require('forward');
-var path = require('path');
+var microtime = require('microtime');
+var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
+var notFound = require('../middleware/registry_not_found');
 
-var rootdir = path.dirname(__dirname);
-var app = connect();
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
+app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
 
-app.use('/favicon.ico', forward(path.join(rootdir, 'public', 'favicon.png')));
-
-app.use('/dist', connect.static(config.uploadDir));
-
-app.use(responseCookie());
-app.use(connect.cookieParser());
-app.use(connect.query());
-app.use(connect.json());
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
+app.proxy = true;
 app.use(session);
+app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.gzip());
+app.use(middlewares.fresh());
+app.use(middlewares.etag());
 
 /**
  * Routes
  */
 
-app.use(urlrouter(routes));
-
-app.use(function (req, res, next) {
-  res.json(404, {error: 'not_found', reason: 'document not found'});
-});
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.json(500, {
-    error: err.name,
-    reason: err.message
-  });
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.registryPort);
+}
 
 module.exports = app;

servers/web.js

@@ -15,37 +15,43 @@
  * Module dependencies.
  */
 
-require('response-patch');
 var path = require('path');
 var http = require('http');
 var fs = require('fs');
-var connect = require('connect');
-var rt = require('connect-rt');
-var urlrouter = require('urlrouter');
-var connectMarkdown = require('connect-markdown');
+var microtime = require('microtime');
+var koa = require('koa');
+var middlewares = require('koa-middlewares');
+var markdown = require('koa-markdown');
+var session = require('../common/session');
+var opensearch = require('../middleware/opensearch');
+var notFound = require('../middleware/web_not_found');
+var auth = require('../middleware/auth');
 var routes = require('../routes/web');
 var logger = require('../common/logger');
 var config = require('../config');
-var session = require('../common/session');
-var render = require('connect-render');
-var opensearch = require('../middleware/opensearch');
-var app = connect();
+
+var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
-
-app.use('/public', connect.static(path.join(rootdir, 'public')));
-app.use('/opensearch.xml', opensearch);
-
-app.use(connect.cookieParser());
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
+  buffer: !config.debug,
+  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+  dir: path.join(rootdir, 'public')
+}));
+app.use(opensearch);
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
+app.proxy = true;
 app.use(session);
-app.use(connect.query());
-app.use(connect.json());
+app.use(middlewares.bodyParser());
+app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.gzip());
+app.use(middlewares.fresh());
+app.use(middlewares.etag());
 
 var viewDir = path.join(rootdir, 'view', 'web');
 var docDir = path.join(rootdir, 'docs', 'web');
@@ -57,7 +63,8 @@ var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
   .replace('{{logoURL}}', config.logoURL);
 fs.writeFileSync(layoutFile, layout);
 
-app.use('/', connectMarkdown({
+app.use(markdown({
+  baseUrl: '/',
   root: docDir,
   layout: layoutFile,
   titleHolder: '<%- locals.title %>',
@@ -65,41 +72,38 @@ app.use('/', connectMarkdown({
   indexName: 'readme'
 }));
 
-var helpers = {
+var locals = {
   config: config
 };
 
-app.use(render({
+middlewares.render(app, {
   root: viewDir,
-  viewExt: '.html',
+  viewExt: 'html',
   layout: '_layout',
   cache: config.viewCache,
-  helpers: helpers
-}));
+  debug: config.debug,
+  locals: locals
+});
 
 /**
  * Routes
  */
-
-app.use(urlrouter(routes));
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.statusCode = 500;
-  res.send('Server has some problems. :(');
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.webPort);
+}
 
 module.exports = app;

sync/index.js

@@ -22,8 +22,9 @@ var utility = require('utility');
 var debug = require('debug')('cnpmjs.org:sync:index');
 var Total = require('../proxy/total');
 var logger = require('../common/logger');
+var co = require('co');
 
-var sync;
+var sync = null;
 
 switch (config.syncModel) {
 case 'all':
@@ -34,28 +35,41 @@ case 'exist':
   break;
 }
 
+if (!sync && config.enableCluster) {
+  console.log('[%s] [sync_worker:%s] no need to sync, exit now', Date(), process.pid);
+  process.exit(0);
+}
+
+console.log('[%s] [sync_worker:%s] syncing with %s mode',
+  Date(), process.pid, config.syncModel);
+
 //set sync_status = 0 at first
 Total.updateSyncStatus(0, utility.noop);
 
 // the same time only sync once
 var syncing = false;
 
-function handleSync() {
+var handleSync = co(function *() {
   debug('mode: %s, syncing: %s', config.syncModel, syncing);
-  // check sync every one 30 minutes
   if (!syncing) {
     syncing = true;
     debug('start syncing');
-    sync(function (err, data) {
-      if (config.debug) {
-        console.log(err, data);
-      } else {
-        sendMailToAdmin(err, data, new Date());
-      }
-      syncing = false;
-    });
+    var data;
+    var error;
+    try {
+      var data = yield *sync();
+    } catch (err) {
+      error = err;
+    }
+    if (config.debug) {
+      error && console.error(error.stack);
+      data && console.log(data);
+    } else {
+      sendMailToAdmin(error, data, new Date());
+    }
+    syncing = false;
   }
-}
+});
 
 if (sync) {
   handleSync();

sync/status.js

@@ -19,18 +19,20 @@ var utility = require('utility');
 var Total = require('../proxy/total');
 
 function Status(options) {
-  this.worker = options.worker;
   this.need = options.need;
   this.lastSyncModule = '';
+  this.successes = 0;
+  this.fails = 0;
+  this.left = options.need;
 }
 
 Status.prototype.log = function (syncDone) {
   var params = {
     syncStatus: syncDone ? 0 : 1,
     need: this.need,
-    success: this.worker.successes.length,
-    fail: this.worker.fails.length,
-    left: this.worker.names.length,
+    success: this.successes,
+    fail: this.fails,
+    left: this.left,
     lastSyncModule: this.lastSyncModule,
   };
   Total.updateSyncNum(params, utility.noop);
@@ -43,19 +45,30 @@ Status.prototype.start = function () {
   this.started = true;
   //every 30s log it into mysql
   this.timer = setInterval(this.log.bind(this), 30000);
-  this.worker.on('success', function (moduleName) {
-    debug('sync [%s] success', moduleName);
-    this.lastSyncModule = moduleName;
-  }.bind(this));
-  this.worker.on('end', function () {
-    this.started = false;
-    this.log(true);
-    clearInterval(this.timer);
-  }.bind(this));
 };
 
-Status.init = function (options) {
-  return new Status(options);
+Status.init = function (options, worker) {
+  var status = new Status(options);
+  status.start();
+  worker.on('success', function (moduleName) {
+    debug('sync [%s] success', moduleName);
+    status.lastSyncModule = moduleName;
+    status.successes++;
+    status.left--;
+  });
+  worker.on('fail', function () {
+    status.fails++;
+    status.left--;
+  });
+  worker.on('add', function () {
+    status.left++;
+  });
+
+  worker.on('end', function () {
+    status.started = false;
+    status.log(true);
+    clearInterval(status.timer);
+  });
 };
 
 module.exports = Status;

sync/sync_all.js

@@ -24,6 +24,8 @@ var Npm = require('../proxy/npm');
 var Total = require('../proxy/total');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var Module = require('../proxy/module');
+var co = require('co');
+var thunkify = require('thunkify-wrap');
 
 function subtract(subtracter, minuend) {
   subtracter = subtracter || [];
@@ -55,109 +57,86 @@ function union(arrOne, arrTwo) {
  * when sync from official at the first time
  * get all packages by short and restart from last synced module
  * @param {String} lastSyncModule
- * @param {Function} callback
  */
-function getFirstSyncPackages(lastSyncModule, callback) {
-  Npm.getShort(function (err, pkgs) {
-    if (err || !lastSyncModule) {
-      return callback(err, pkgs);
-    }
-    // start from last success
-    var lastIndex = pkgs.indexOf(lastSyncModule);
-    if (lastIndex > 0) {
-      pkgs = pkgs.slice(lastIndex);
-    }
-    return callback(null, pkgs);
-  });
+function *getFirstSyncPackages(lastSyncModule) {
+  var pkgs = yield Npm.getShort();
+  if (!lastSyncModule) {
+    return pkgs;
+  }
+  // start from last success
+  var lastIndex = pkgs.indexOf(lastSyncModule);
+  if (lastIndex > 0) {
+    return pkgs.slice(lastIndex);
+  }
 }
 
 /**
  * get all the packages that update time > lastSyncTime
  * @param {Number} lastSyncTime
- * @param {Function} callback
  */
-function getCommonSyncPackages(lastSyncTime, callback) {
-  Npm.getAllSince(lastSyncTime, function (err, data) {
-    if (err || !data) {
-      return callback(err, []);
-    }
-    delete data._updated;
-    return callback(null, Object.keys(data));
-  });
+function *getCommonSyncPackages(lastSyncTime) {
+  var data = yield Npm.getAllSince(lastSyncTime);
+  if (!data) {
+    return [];
+  }
+  delete data._updated;
+  return Object.keys(data);
 }
 
 /**
  * get all the missing packages
  * @param {Function} callback
  */
-function getMissPackages(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
-  Npm.getShort(ep.doneLater('allPackages'));
-  Module.listAllModuleNames(ep.doneLater(function (rows) {
-    var existPackages = rows.map(function (row) {
-      return row.name;
-    });
-    ep.emit('existPackages', existPackages);
-  }));
-  ep.all('allPackages', 'existPackages', function (allPackages, existPackages) {
-    callback(null, subtract(allPackages, existPackages));
+function *getMissPackages(callback) {
+  var r = yield [Npm.getShort(), Module.listAllModuleNames];
+  var allPackages = r[0];
+  var existPackages = r[1].map(function (row) {
+    return row.name;
   });
+  return subtract(allPackages, existPackages);
 }
 
-//only sync not exist once
-var syncNotExist = true;
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
+  var info = yield Total.getTotalInfo();
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    debug('Last sync time %s', new Date(info.last_sync_time));
-    // TODO: 记录上次同步的最后一个模块名称
-    if (!info.last_sync_time) {
-      debug('First time sync all packages from official registry');
-      return getFirstSyncPackages(info.last_sync_module, ep.done('syncPackages'));
-    }
-    if (syncNotExist) {
-      getMissPackages(ep.done('missPackages'));
-      syncNotExist = false;
-    } else {
-      ep.emitLater('missPackages', []);
-    }
-    getCommonSyncPackages(info.last_sync_time - ms('10m'), ep.doneLater('newestPackages'));
-    ep.all('missPackages', 'newestPackages', function (missPackages, newestPackages) {
-      ep.emit('syncPackages', union(missPackages, newestPackages));
-    });
-  });
+  var packages;
+  debug('Last sync time %s', new Date(info.last_sync_time));
+  if (!info.last_sync_time) {
+    debug('First time sync all packages from official registry');
+    packages = yield getFirstSyncPackages(info.last_sync_module);
+  } else {
+    packages = yield getCommonSyncPackages(info.last_sync_time - ms('10m'));
+  }
 
-  ep.once('syncPackages', function (packages) {
-    packages = packages || [];
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      noDep: true,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      //only when all succss, set last sync time
-      !worker.fails.length && Total.setLastSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  packages = packages || [];
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    concurrency: config.syncConcurrency,
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  yield thunkify(worker)();
+
+  debug('All packages sync done, successes %d, fails %d',
+      worker.successes.length, worker.fails.length);
+  //only when all succss, set last sync time
+  if (!worker.fails.length) {
+    Total.setLastSyncTime(syncTime, utility.noop);
+  }
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

sync/sync_exist.js

@@ -24,6 +24,7 @@ var debug = require('debug')('cnpmjs.org:sync:sync_hot');
 var utility = require('utility');
 var Status = require('./status');
 var ms = require('ms');
+var thunkify = require('thunkify-wrap');
 
 function intersection(arrOne, arrTwo) {
   arrOne = arrOne || [];
@@ -39,71 +40,66 @@ function intersection(arrOne, arrTwo) {
   return results;
 }
 
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
 
-  Module.listShort(ep.doneLater(function (packages) {
-    packages = packages.map(function (p) {
-      return p.name;
-    });
-    ep.emit('existPackages', packages);
-  }));
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
-
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    if (!info.last_exist_sync_time) {
-      debug('First time sync all packages from official registry');
-      return Npm.getShort(ep.done(function (pkgs) {
-        if (!info.last_sync_module) {
-          return ep.emit('allPackages', pkgs);
-        }
-        // start from last success
-        var lastIndex = pkgs.indexOf(info.last_sync_module);
-        if (lastIndex > 0) {
-          pkgs = pkgs.slice(lastIndex);
-        }
-        ep.emit('allPackages', pkgs);
-      }));
-    }
-    Npm.getAllSince(info.last_exist_sync_time - ms('10m'), ep.done(function (data) {
-      if (!data) {
-        return ep.emit('allPackages', []);
-      }
-      if (data._updated) {
-        syncTime = data._updated;
-        delete data._updated;
-      }
-
-      return ep.emit('allPackages', Object.keys(data));
-    }));
+  var r = yield [Module.listShort(), Total.getTotalInfo()];
+  var existPackages = r[0].map(function (p) {
+    return p.name;
   });
+  var info = r[1];
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.all('existPackages', 'allPackages', function (existPackages, allPackages) {
-    var packages = intersection(existPackages, allPackages);
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      Total.setLastExistSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  var allPackages;
+  if (!info.last_exist_sync_time) {
+    debug('First time sync all packages from official registry');
+    var pkgs = yield Npm.getShort();
+
+    if (info.last_sync_module) {
+      // start from last success
+      var lastIndex = pkgs.indexOf(info.last_sync_module);
+      if (lastIndex > 0) {
+        pkgs = pkgs.slice(lastIndex);
+      }
+    }
+    allPackages = pkgs;
+  } else {
+    debug('sync new module from last exist sync time: %s', info.last_sync_time);
+    var data = yield Npm.getAllSince(info.last_exist_sync_time - ms('10m'));
+    if (!data) {
+      allPackages = [];
+    }
+    if (data._updated) {
+      syncTime = data._updated;
+      delete data._updated;
+    }
+    allPackages = Object.keys(data);
+  }
+
+  var packages = intersection(existPackages, allPackages);
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    concurrency: config.syncConcurrency
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  yield thunkify(worker)();
+
+  debug('All packages sync done, successes %d, fails %d',
+    worker.successes.length, worker.fails.length);
+
+  Total.setLastExistSyncTime(syncTime, utility.noop);
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

test/controllers/registry/module.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/registry/module.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,6 +17,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var thunkify = require('thunkify-wrap');
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -25,6 +26,7 @@ var app = require('../../../servers/registry');
 var Module = require('../../../proxy/module');
 var Npm = require('../../../proxy/npm');
 var controller = require('../../../controllers/registry/module');
+var ModuleDeps = require('../../../proxy/module_deps');
 
 var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
 
@@ -64,38 +66,76 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('GET /:name', function () {
-    it('should return module info', function (done) {
+  describe('GET /:name get module package info', function () {
+    var etag;
+
+    it('should return module info and etag', function (done) {
       request(app)
       .get('/cnpmjs.org')
+      .expect('content-type', 'application/json')
       .expect(200, function (err, res) {
         should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
-          'time', 'author', 'repository', '_attachments');
-        // res.body.author.should.eql({
-        //   "name": "fengmk2",
-        //   "email": "fengmk2@gmail.com",
-        //   "url": "http://fengmk2.github.com"
-        // });
+          'time', 'author', 'repository', '_attachments',
+          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
         res.body.name.should.equal('cnpmjs.org');
-        res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
+        res.body.versions[Object.keys(res.body.versions)[0]]
+          .dist.tarball.should.include('/cnpmjs.org/download');
+        res.body.time.should.have.property('modified');
+        res.body.time.modified.should.be.a.String;
+        res.body.time.should.have.property('created');
+        res.body.time.created.should.be.a.String;
+        // should not contains authSession cookie
+        should.not.exist(res.headers['set-cookie']);
         done();
       });
     });
+
+    it('should return module info and gzip when accept-encoding=gzip', function (done) {
+      request(app)
+      .get('/cnpmjs.org')
+      .set('accept-encoding', 'gzip')
+      .expect('content-encoding', 'gzip')
+      .expect(200, function (err, res) {
+        // console.log(res.headers)
+        should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
+        res.body.should.have.keys('_id', '_rev', 'name', 'description',
+          'versions', 'dist-tags', 'readme', 'maintainers',
+          'time', 'author', 'repository', '_attachments',
+          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
+        res.body.name.should.equal('cnpmjs.org');
+        res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
+        should.not.exist(res.headers['set-cookie']);
+        done();
+      });
+    });
+
+    it('should 304 when etag match', function (done) {
+      request(app)
+      .get('/cnpmjs.org')
+      .set('If-None-Match', etag)
+      .expect(304, done);
+    });
   });
 
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.2.1')
+      .get('/cnpmjs.org/0.3.9')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.2.1');
-        body._id.should.equal('cnpmjs.org@0.2.1');
-        body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
+        body.version.should.match(/\d+\.\d+\.\d+/);
+        body._id.should.match(/cnpmjs\.org@\d+\.\d+\.\d+/);
+        body.dist.tarball.should.match(/cnpmjs\.org-\d+\.\d+\.\d+\.tgz/);
         body.should.have.property('_cnpm_publish_time');
         body._cnpm_publish_time.should.be.a.Number;
         body.should.have.property('_publish_on_cnpm', true);
@@ -117,20 +157,97 @@ describe('controllers/registry/module.test.js', function () {
       });
     });
 
-    it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
+    it('should get cnpmjs.org@latest with _publish_on_cnpm=true', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.2.1')
+      .get('/cnpmjs.org/latest')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.2.1');
+        // body.version.should.equal('latest');
         body._publish_on_cnpm.should.equal(true);
         done();
       });
     });
   });
 
+  describe('PUT /:name/-rev/id update maintainers', function () {
+    before(function (done) {
+      request(app)
+      .put('/cnpmjs.org/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should add new maintainers', function (done) {
+      request(app)
+      .put('/cnpmjs.org/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }, {
+          name: 'fengmk2',
+          email: 'fengmk2@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should add again new maintainers', function (done) {
+      request(app)
+      .put('/cnpmjs.org/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }, {
+          name: 'fengmk2',
+          email: 'fengmk2@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should rm maintainers', function (done) {
+      request(app)
+      .put('/cnpmjs.org/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should rm again maintainers', function (done) {
+      request(app)
+      .put('/cnpmjs.org/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+  });
+
   describe('PUT /:name', function () {
     var pkg = {
       name: 'testputmodule',
@@ -142,7 +259,11 @@ describe('controllers/registry/module.test.js', function () {
       }],
       keywords: [
         'testputmodule', 'test', 'cnpmjstest'
-      ]
+      ],
+      dependencies: {
+        'foo-testputmodule': "*",
+        'bar-testputmodule': '*'
+      }
     };
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
@@ -204,7 +325,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags',
-          'readme', 'maintainers', 'time', '_attachments');
+          'readme', 'maintainers', 'time', '_attachments', 'users');
         res.body.versions.should.eql({});
         res.body.time.should.eql({});
         res.body['dist-tags'].should.eql({});
@@ -290,6 +411,7 @@ describe('controllers/registry/module.test.js', function () {
       var pkg = require(path.join(fixtures, 'testputmodule.json')).versions['0.1.8'];
       pkg.name = 'testputmodule';
       pkg.version = '0.1.9';
+      pkg.dependencies['foo-testputmodule'] = '*';
       request(app)
       .put('/' + pkg.name + '/' + pkg.version + '/-tag/latest')
       .set('authorization', baseauth)
@@ -297,7 +419,16 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('ok', 'rev');
-        done();
+        // should get deps foo-testputmodule contains 'testputmodule'
+        ModuleDeps.list('foo-testputmodule', function (err, rows) {
+          should.not.exist(err);
+          var exists = rows.filter(function (r) {
+            return r.deps === 'testputmodule';
+          });
+          exists.should.length(1);
+          exists[0].deps.should.equal('testputmodule');
+          done();
+        });
       });
     });
 
@@ -345,6 +476,60 @@ describe('controllers/registry/module.test.js', function () {
         done();
       });
     });
+
+    it('should publish with tgz base64, addPackageAndDist()', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      // delete first
+      request(app)
+      .del('/' + pkg.name + '/-rev/1')
+      .set('authorization', baseauth)
+      .expect({ok: true})
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+
+        request(app)
+        .put('/' + pkg.name)
+        .set('authorization', baseauth)
+        .send(pkg)
+        .expect(201, function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'rev');
+          res.body.ok.should.equal(true);
+
+          // upload again should 409
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', baseauth)
+          .send(pkg)
+          .expect(409, function (err, res) {
+            should.not.exist(err);
+            res.body.should.eql({
+              error: 'conflict',
+              reason: 'Document update conflict.'
+            });
+            done();
+          });
+
+        });
+      });
+    });
+
+    it('should version_error when versions missing', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      delete pkg.versions;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({
+          error: 'version_error',
+          reason: 'filename or version not found, filename: mk2testmodule-0.0.1.tgz, version: undefined'
+        });
+        done();
+      });
+    });
   });
 
   describe('GET /-/all', function () {
@@ -458,35 +643,7 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .get('/cutter/download/cutter-0.0.2.tgz')
       .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
-      .expect(302, done)
-    });
-
-    it('should download a file direct from nfs stream', function (done) {
-      var nfs = require('../../../common/nfs');
-      mm(nfs, 'downloadStream', function (key, writeStream, options, callback) {
-        options.timeout.should.equal(600000);
-        nfs._client.download(key, {writeStream: writeStream, timeout: options.timeout}, callback);
-      });
-      Module.__get__ = Module.get;
-      mm(Module, 'get', function (name, version, callback) {
-        Module.__get__(name, version, function (err, info) {
-          info.package.dist.key = 'cutter/-/cutter-0.0.2.tgz';
-          callback(err, info);
-        });
-      });
-      request(app)
-      .get('/cutter/download/cutter-0.0.2.tgz')
-      .expect('ETag', 'c61fde5e8c26d053574d0c722097029fd1bc963a')
-      .expect('Content-Type', 'application/octet-stream')
-      .expect('Content-Length', '3139')
-      .expect('Content-Disposition', 'attachment; filename="cutter-0.0.2.tgz"')
-      .expect(200)
-      .end(function (err, res) {
-        should.not.exist(err);
-        // TODO: why supertest change buffer to text?
-        // res.text.length.should.equal(3139);
-        done();
-      });
+      .expect(302, done);
     });
   });
 
@@ -554,7 +711,11 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .del('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
-      .expect(200, done);
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        should.not.exist(res.headers['set-cookie']);
+        done();
+      });
     });
   });
 });

test/controllers/registry/user.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/registry/user.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,30 +16,49 @@
 
 var should = require('should');
 var request = require('supertest');
+var mm = require('mm');
 var app = require('../../../servers/registry');
 var user = require('../../../proxy/user');
-var mm = require('mm');
+var mysql = require('../../../common/mysql');
 
 describe('controllers/registry/user.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
+
   afterEach(mm.restore);
+
   describe('GET /-/user/org.couchdb.user:name', function () {
     it('should return user info', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
       .expect(200, function (err, res) {
         should.not.exist(err);
-        res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type', 'roles', 'date');
+        res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type',
+          '_cnpm_meta', 'roles', 'date');
         res.body.name.should.equal('cnpmjstest1');
         done();
       });
     });
 
+    it('should return npm user info', function (done) {
+      request(app)
+      .get('/-/user/org.couchdb.user:fengmk2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.name.should.equal('fengmk2');
+        res.body.github.should.equal('fengmk2');
+        res.body._cnpm_meta.should.have.keys('id', 'npm_user', 'gmt_create',
+          'gmt_modified', 'admin');
+        res.body._cnpm_meta.admin.should.equal(true);
+        done();
+      });
+    });
+
     it('should return 404 when not eixst', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest_notexist')
@@ -47,7 +66,7 @@ describe('controllers/registry/user.test.js', function () {
     });
 
     it('should return 500 when mysql error', function (done) {
-      mm.error(user, 'get', 'mock error');
+      mm.error(mysql, 'query', 'mock mysql error');
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
       .expect(500, done);
@@ -91,7 +110,7 @@ describe('controllers/registry/user.test.js', function () {
         password_sha: 'password_sha',
         email: 'email'
       })
-      .expect(500, done);      
+      .expect(500, done);
     });
 
     it('should 201 when user.add ok', function (done) {
@@ -145,7 +164,12 @@ describe('controllers/registry/user.test.js', function () {
         ok: true,
         name: 'name',
         roles: []
-      }, done);
+      }, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers['set-cookie']);
+        res.headers['set-cookie'].join(';').should.include('AuthSession=');
+        done();
+      });
     });
   });
 
@@ -167,6 +191,11 @@ describe('controllers/registry/user.test.js', function () {
       mm.error(user, 'update', 'mock error');
       request(app)
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
+      .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
+        email: 'cnpmjstest10@cnpmjs.org'
+      })
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .expect(500, done);
     });
@@ -176,6 +205,8 @@ describe('controllers/registry/user.test.js', function () {
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org',
         rev: '1-123'
       })
@@ -188,6 +219,8 @@ describe('controllers/registry/user.test.js', function () {
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org',
         rev: '1-123'
       })

test/controllers/sync.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/sync.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,12 +17,12 @@
 
 var request = require('supertest');
 var should = require('should');
-var registryApp = require('../../servers/registry');
-var webApp = require('../../servers/web');
 var pedding = require('pedding');
 var mm = require('mm');
-var Npm = require('../../proxy/npm');
 var path = require('path');
+var Npm = require('../../proxy/npm');
+var registryApp = require('../../servers/registry');
+var webApp = require('../../servers/web');
 
 describe('controllers/sync.test.js', function () {
   before(function (done) {
@@ -45,8 +45,6 @@ describe('controllers/sync.test.js', function () {
       request(registryApp)
       .del('/utility/-rev/123')
       .set('authorization', baseauth)
-      // .expect(200)
-      // .expect({ok: true})
       .end(function (err, res) {
         should.not.exist(err);
 
@@ -59,20 +57,6 @@ describe('controllers/sync.test.js', function () {
           res.body.should.have.keys('ok', 'logId');
           logIdRegistry = res.body.logId;
           done();
-          // setTimeout(function () {
-          //   request(registryApp)
-          //   .get('/utility')
-          //   .expect(200)
-          //   .end(function (err, res) {
-          //     should.not.exist(err);
-          //     Object.keys(res.body.versions).length.should.above(0);
-          //     for (var v in res.body.versions) {
-          //       var pkg = res.body.versions[v];
-          //       pkg.should.have.property('_publish_on_cnpm', true);
-          //     }
-          //     done();
-          //   });
-          // }, 5000);
         });
       });
     });
@@ -88,9 +72,20 @@ describe('controllers/sync.test.js', function () {
       }, done);
     });
 
-    it('should sync success', function (done) {
+    it('should sync through web success', function (done) {
+      mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+      request(webApp)
+      .put('/sync/utility')
+      .end(function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('ok', 'logId');
+        logIdWeb = res.body.logId;
+        done();
+      });
+    });
+
+    it('should sync through registry success', function (done) {
       mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
-      done = pedding(2, done);
       request(registryApp)
       .put('/utility/sync')
       .set('authorization', baseauth)
@@ -100,14 +95,6 @@ describe('controllers/sync.test.js', function () {
         logIdRegistry = res.body.logId;
         done();
       });
-      request(webApp)
-      .put('/sync/utility')
-      .end(function (err, res) {
-        should.not.exist(err);
-        res.body.should.have.keys('ok', 'logId');
-        logIdWeb = res.body.logId;
-        done();
-      });
     });
 
     it('should get sync log', function (done) {

test/controllers/total.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/total.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,9 +17,9 @@
 
 var should = require('should');
 var request = require('supertest');
+var pedding = require('pedding');
 var registryApp = require('../../servers/registry');
 var webApp = require('../../servers/web');
-var pedding = require('pedding');
 
 describe('controllers/total.test.js', function () {
   before(function (done) {
@@ -44,6 +44,7 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
+
     it('should return total info by jsonp', function (done) {
       request(registryApp)
       .get('?callback=totalCallback')
@@ -51,7 +52,8 @@ describe('controllers/total.test.js', function () {
       .expect(/totalCallback\({.*}\)/, done);
     });
   });
-  describe('GET /total in web', function () {
+
+  describe.skip('GET /total in web', function () {
     it('should return total info', function (done) {
       request(webApp)
       .get('/total')
@@ -62,5 +64,5 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
-  });  
+  });
 });

test/controllers/web/dist.test.js

@@ -0,0 +1,37 @@
+/**!
+ * cnpmjs.org - test/controllers/web/dist.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../../servers/web');
+
+describe('controllers/web/dist.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('GET /dist', function (done) {
+    it('should 302 to config.disturl', function (done) {
+      request(app)
+      .get('/dist')
+      .expect('Location', 'http://dist.u.qiniudn.com/')
+      .expect(302, done);
+    });
+  });
+});

test/controllers/web/package.test.js

@@ -35,6 +35,7 @@ describe('controllers/web/package.test.js', function () {
     it('should search with "c"', function (done) {
       request(app)
       .get('/_list/search/search?startkey="c"&limit=2')
+      .expect('content-type', 'application/json')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('rows');
@@ -78,9 +79,16 @@ describe('controllers/web/package.test.js', function () {
       request(app)
       .get('/package/cutter')
       .expect(200)
+      .expect('content-encoding', 'gzip')
+      .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
-      .expect(/<th>Version<\/th>/, done);
+      .expect(/<th>Version<\/th>/, function (err, res) {
+        should.not.exist(err);
+        res.should.have.header('etag');
+        res.text.should.include('<meta charset="utf-8">');
+        done();
+      });
     });
 
     it('should get 404', function (done) {
@@ -129,6 +137,13 @@ describe('controllers/web/package.test.js', function () {
       .expect(/Packages match/, done);
     });
 
+    it('should list by keyword with json ok', function (done) {
+      request(app)
+      .get('/browse/keyword/cnpm?type=json')
+      .expect(200)
+      .expect('content-type', 'application/json; charset=utf-8', done);
+    });
+
     it('should list no match ok', function (done) {
       request(app)
       .get('/browse/keyword/notexistpackage')
@@ -141,7 +156,7 @@ describe('controllers/web/package.test.js', function () {
       request(app)
       .get('/browse/keyword/notexistpackage')
       .expect(500)
-      .expect(/MockError: mm mock error/, done);
+      .expect(/Internal Server Error/, done);
     });
   });
 
@@ -155,7 +170,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://opensource.org/licenses/MIT'
       });
 
-      var p = {license: ['http://foo/MIT']};
+      p = {license: ['http://foo/MIT']};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({
@@ -163,7 +178,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://foo/MIT'
       });
 
-      var p = {license: {name: 'mit', url: 'http://foo/mit'}};
+      p = {license: {name: 'mit', url: 'http://foo/mit'}};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({

test/controllers/web/user.test.js

@@ -31,6 +31,7 @@ describe('controllers/web/user.test.js', function () {
       request(app)
       .get('/~dead_horse')
       .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="profile">/)
       .expect(/Packages by /, done);
     });

test/fixtures/fengmk2.json

@@ -0,0 +1 @@
+{"_id":"org.couchdb.user:fengmk2","_rev":"25-ba29aa975022944b4ba547ab11ce3af5","name":"fengmk2","email":"fengmk2@gmail.com","type":"user","roles":[],"date":"2014-03-15T08:06:18.870Z","fullname":"fengmk2","avatar":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=50&d=retro","freenode":"","github":"fengmk2","homepage":"http://fengmk2.github.com","twitter":"fengmk2","avatarMedium":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=100&d=retro","avatarLarge":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=496&d=retro","fields":[{"name":"fullname","value":"fengmk2","title":"Full Name","show":"fengmk2"},{"name":"email","value":"fengmk2@gmail.com","title":"Email","show":"<a href=\"mailto:fengmk2@gmail.com\">fengmk2@gmail.com</a>"},{"name":"github","value":"fengmk2","title":"Github","show":"<a rel=\"me\" href=\"https://github.com/fengmk2\">fengmk2</a>"},{"name":"twitter","value":"fengmk2","title":"Twitter","show":"<a rel=\"me\" href=\"https://twitter.com/fengmk2\">@fengmk2</a>"},{"name":"appdotnet","value":"","title":"App.net","show":""},{"name":"homepage","value":"http://fengmk2.github.com","title":"Homepage","show":"<a rel=\"me\" href=\"http://fengmk2.github.com/\">http://fengmk2.github.com</a>"},{"name":"freenode","value":"","title":"IRC Handle","show":""}],"appdotnet":"fengmk2"}

test/fixtures/package_and_tgz.json

@@ -0,0 +1 @@
+{"_id":"mk2testmodule","name":"mk2testmodule","description":"","dist-tags":{"latest":"0.0.1"},"versions":{"0.0.1":{"name":"mk2testmodule","version":"0.0.1","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"author":"","license":"ISC","readme":"ERROR: No README data found!","_id":"mk2testmodule@0.0.1","dist":{"shasum":"fa475605f88bab9b1127833633ca3ae0a477224c","tarball":"http://127.0.0.1:7001/mk2testmodule/-/mk2testmodule-0.0.1.tgz"},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}]}},"readme":"ERROR: No README data found!","maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_attachments":{"mk2testmodule-0.0.1.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2SsWrDMBCGPfspDg2ZinOyEgeylg6Zu2YR8rVRHEtGkkOg5N0jWaFdujVQAv6W4/7/dHcSGqTq5Ccthxyro7emeDCI2KxWkOKmaaaIdc4TouZQ8FqgwI3AdVMgF8ijho9e5DdGH6SLq/y1T74LfMcn4asEYEb2xLbA+q4O5ENv2/FE7CVZZ3JeW5NcrLDiWW3JK6eHcHey2Es9Zdq0dIkfKau50EcjjYpCmpDKSB0s7Nmbc9ZtwVhIBviBlP7Q1O4ZLBZAFx2As3jyOnWTYzhY9zPzpBUZPy2/e39l5bX87wedmZmZeRJuheTX2wAIAAA=","length":251}}}

test/middleware/opensearch.test.js

@@ -30,8 +30,8 @@ describe('middleware/opensearch.test.js', function () {
     it('should get 200', function (done) {
       request(app)
       .get('/opensearch.xml')
-      .set('host', 'localhost:7002')
-      .expect(/http:\/\/localhost:7002/, done);
+      .set('host', 'localhost')
+      .expect(/http:\/\/localhost/, done);
     });
   });
 });

test/middleware/web_not_found.test.js

@@ -0,0 +1,63 @@
+/**!
+ * cnpmjs.org - test/middleware/web_not_found.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../servers/web');
+
+describe('middleware/web_not_found.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('web_not_found()', function () {
+    it('should redirect /byte to /package/byte', function (done) {
+      request(app)
+      .get('/byte')
+      .expect('Location', '/package/byte')
+      .expect(302, done);
+    });
+
+    it('should redirect /byte/ to /package/byte', function (done) {
+      request(app)
+      .get('/byte/')
+      .expect('Location', '/package/byte')
+      .expect(302, done);
+    });
+
+    it('should 404 /~byte', function (done) {
+      request(app)
+      .get('/~byte')
+      .expect(404, done);
+    });
+
+    it('should 200 /package/byte', function (done) {
+      request(app)
+      .get('/package/byte')
+      .expect(200, done);
+    });
+
+    it('should 404 /package/byte404', function (done) {
+      request(app)
+      .get('/package/byte404')
+      .expect(404, done);
+    });
+  });
+});

test/proxy/module.test.js

@@ -158,7 +158,7 @@ describe('proxy/module.test.js', function () {
           should.not.exist(err);
           data.package.readme.should.equal('test');
           done();
-        })
+        });
       });
     });
   });

test/proxy/module_deps.test.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - test/proxy/module_deps.test.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var pedding = require('pedding');
+var ModuleDeps = require('../../proxy/module_deps');
+
+describe('proxy/module_deps.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    ModuleDeps.remove('testmodule', 'foo', done);
+    ModuleDeps.remove('testmodule', 'bar', done);
+  });
+
+  describe('add()', function () {
+    it('should add foo, bar success', function (done) {
+      done = pedding(2, done);
+      ModuleDeps.add('testmodule', 'foo', function (err) {
+        should.not.exist(err);
+        // add again should work
+        ModuleDeps.add('testmodule', 'foo', function (err) {
+          should.not.exist(err);
+          done();
+        });
+      });
+
+      ModuleDeps.add('testmodule', 'bar', done);
+    });
+  });
+
+  describe('list()', function () {
+    it('should list testmodule deps', function (done) {
+      ModuleDeps.list('testmodule', function (err, rows) {
+        should.not.exist(err);
+        should.exist(rows);
+        rows.should.be.an.Array;
+        rows.should.length(2);
+        rows.forEach(function (row) {
+          row.should.have.property('deps');
+        });
+        done();
+      });
+    });
+  });
+});

test/proxy/module_star.test.js

@@ -0,0 +1,64 @@
+/**!
+ * cnpmjs.org - test/proxy/module_star.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var co = require('co');
+var Star = require('../../proxy/module_star');
+
+describe('proxy/module_star.test.js', function () {
+  before(function (done) {
+    co(function *() {
+      yield Star.remove('testmodule', 'fengmk2');
+      yield Star.remove('testmodule', 'mk1');
+      yield Star.remove('testmodule', 'mk2');
+      done();
+    })();
+  });
+
+  it('should add a star', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      // again should be ok
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule', 'fengmk2');
+      done();
+    })();
+  });
+
+  it('should get all star users', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule', 'mk1');
+      yield Star.add('testmodule', 'mk2');
+
+      var rows = yield Star.listUsers('testmodule');
+      rows.should.containDeep(['fengmk2', 'mk1', 'mk2']);
+      done();
+    })();
+  });
+
+  it('should get user all star modules', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule1', 'fengmk2');
+      yield Star.add('testmodule2', 'fengmk2');
+
+      var rows = yield Star.listUserModules('fengmk2');
+      rows.should.containDeep(['testmodule', 'testmodule1', 'testmodule2']);
+      done();
+    })();
+  });
+});

test/proxy/npm.test.js

@@ -18,6 +18,8 @@ var should = require('should');
 var mm = require('mm');
 var fs = require('fs');
 var path = require('path');
+var co = require('co');
+var ChunkStream = require('chunkstream');
 var npm = require('../../proxy/npm');
 
 var fixtures = path.join(path.dirname(__dirname), 'fixtures');
@@ -25,42 +27,36 @@ var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 describe('proxy/npm.test.js', function () {
   afterEach(mm.restore);
 
-  it('should return a module info from source npm', function (done) {
-    npm.get('pedding', function (err, data) {
-      should.not.exist(err);
-      should.exist(data);
-      data.name.should.equal('pedding');
-      done();
-    });
-  });
+  it('should return a module info from source npm', co(function *() {
+    var data = yield npm.get('pedding');
+    data.name.should.equal('pedding');
+  }));
 
-  it('should return null when module not exist', function (done) {
-    npm.get('pedding-not-exists', function (err, data) {
-      should.not.exist(err);
-      should.not.exist(data);
-      done();
-    });
-  });
+  it('should return null when module not exist', co(function *() {
+    var data = yield npm.get('pedding-not-exists');
+    should.not.exist(data);
+  }));
 
-  it('should return error when http error', function (done) {
-    mm.http.request(/\//, '{');
-    npm.get('pedding-not-exists', function (err, data) {
-      should.exist(err);
+  it('should return error when http error', co(function *() {
+    mm.http.request(/\//, new ChunkStream(['{']));
+    try {
+      var data = yield npm.get('pedding-not-exists');
+      throw new Error('should not run this');
+    } catch (err) {
       err.name.should.equal('JSONResponseFormatError');
-      should.not.exist(data);
-      done();
-    });
-  });
+    }
+  }));
 
-  it('should return ServerError when http 500 response', function (done) {
-    var content = fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8');
+  it('should return ServerError when http 500 response', co(function *() {
+    var content = fs.createReadStream(path.join(fixtures, '500.txt'));
     mm.http.request(/\//, content, { statusCode: 500 });
     // http://registry.npmjs.org/octopie
-    npm.get('octopie', function (err, data) {
-      should.exist(err);
+    try {
+      var data = yield npm.get('octopie');
+      throw new Error('should not run this');
+    } catch (err) {
       err.name.should.equal('NPMServerError');
-      err.message.should.equal('Status 500, ' + content);
-      done();
-    });
-  });
+      err.message.should.equal('Status 500, ' + fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8'));
+    }
+  }));
 });

test/proxy/user.test.js

@@ -6,6 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
  */
 
 'use strict';
@@ -14,10 +15,15 @@
  * Module dependencies.
  */
 
-var mysql = require('../../common/mysql');
 var should = require('should');
-var user = require('../../proxy/user');
+var co = require('co');
 var mm = require('mm');
+var path = require('path');
+var fs = require('fs');
+var mysql = require('../../common/mysql');
+var user = require('../../proxy/user');
+
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 
 var mockUser = {
   name: 'mockuser',
@@ -49,7 +55,9 @@ describe('proxy/user.test.js', function () {
     it('should get user ok', function (done) {
       user.get('mockuser', function (err, data) {
         should.not.exist(err);
-        data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
+        data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
+          'json', 'npm_user',
+          'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
         done();
       });
     });
@@ -68,7 +76,9 @@ describe('proxy/user.test.js', function () {
     it('should auth user ok', function (done) {
       user.auth(mockUser.name, mockUser.password, function (err, data) {
         should.not.exist(err);
-        data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
+        data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
+          'json', 'npm_user',
+          'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
         done();
       });
     });
@@ -153,4 +163,32 @@ describe('proxy/user.test.js', function () {
       });
     });
   });
+
+  describe('saveNpmUser()', function () {
+    var existUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
+    var notExistUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
+    notExistUser.name = 'fengmk2-not-exists';
+
+    before(co(function *() {
+      yield mysql.query('delete from user where name=?', [notExistUser.name]);
+    }));
+
+    it('should save npm user to exists user', co(function *() {
+      yield user.saveNpmUser(existUser);
+      var r = yield mysql.queryOne('select rev, json, npm_user from user where name=?', existUser.name);
+      should.exist(r);
+      r.npm_user.should.equal(0);
+      r.rev.should.equal(existUser._rev);
+      JSON.parse(r.json).should.eql(existUser);
+    }));
+
+    it('should save npm user to not exists user and create it', co(function *() {
+      yield user.saveNpmUser(notExistUser);
+      var r = yield mysql.queryOne('select name, json, npm_user from user where name=?', notExistUser.name);
+      r.name.should.equal(notExistUser.name);
+      should.exist(r);
+      r.npm_user.should.equal(1);
+      JSON.parse(r.json).should.eql(notExistUser);
+    }));
+  });
 });

test/sync.js

@@ -18,20 +18,23 @@ var SyncModuleWorker = require('../proxy/sync_module_worker');
 var mysql = require('../common/mysql');
 var Log = require('../proxy/module_log');
 
-var name = process.argv[2] || 'address,pedding';
-var names = name.split(',');
+var names = process.argv[2] || 'byte';
+names = names.split(',');
 
 Log.create({
   name: names[0],
   username: 'fengmk2',
 }, function (err, result) {
+  if (err) {
+    throw err;
+  }
   var worker = new SyncModuleWorker({
     logId: result.id,
     name: names,
     username: 'fengmk2',
     concurrency: names.length,
-    noDep: true,
-    publish: true,
+    // noDep: true,
+    // publish: true,
   });
 
   worker.start();

test/sync/sync_all.test.js

@@ -19,6 +19,7 @@ var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
 var should = require('should');
 var Module = require('../../proxy/module');
+var co = require('co');
 
 describe('sync/sync_all.js', function () {
   describe('sync()', function () {
@@ -27,16 +28,14 @@ describe('sync/sync_all.js', function () {
     it('should sync first time ok', function (done) {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync;
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         mm.restore();
-        Total.getTotalInfo(function (err, result) {
-          should.not.exist(err);
-          result.last_sync_module.should.equal('cutter');
-          done();
-        });
-      });
+        var result = yield Total.getTotalInfo();
+        result.last_sync_module.should.equal('cutter');
+        done();
+      })();
     });
 
     it('should sync common ok', function (done) {
@@ -48,11 +47,12 @@ describe('sync/sync_all.js', function () {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
       mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpm', 'cnpmjs.org', 'cutter']);
+      co(function *() {
+        var data = yield sync;
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        mm.restore();
         done();
-      });
+      })();
     });
   });
 });

test/sync/sync_exist.test.js

@@ -18,6 +18,7 @@ var mm = require('mm');
 var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
 var should = require('should');
+var co = require('co');
 
 describe('sync/sync_exist.js', function () {
   describe('sync()', function () {
@@ -26,11 +27,11 @@ describe('sync/sync_exist.js', function () {
     it('should sync first time ok', function (done) {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync();
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });
+      })();
     });
 
     it('should sync common ok', function (done) {
@@ -40,11 +41,11 @@ describe('sync/sync_exist.js', function () {
         cutter: {}
       });
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync();
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });
+      })();
     });
   });
 });

view/web/layout.html

@@ -1,6 +1,7 @@
 <!DOCTYPE html>
 <html>
   <head>
+    <meta charset="utf-8">
     <title><%- locals.title %></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <link rel="shortcut icon" href="/public/favicon.png">

view/web/package.html

@@ -22,10 +22,10 @@
 
     <ul class="nav nav-tabs">
       <% if (package.readme) { %>
-        <li class="active"><a href="#readme">README</a></li>
-        <li><a href="#meta">Meta</a></li>
+        <li class="active"><a href="#readme">Readme.md</a></li>
+        <li><a href="#meta">package.json</a></li>
       <% } else { %>
-        <li class="active"><a href="#meta">Meta</a></li>
+        <li class="active"><a href="#meta">package.json</a></li>
       <% } %>
     </ul>
 
@@ -141,7 +141,7 @@
             var deps = Object.keys(package.dependencies || {})
             var l = deps.length
           %>
-          <th>Dependencies<%= l > 5 ? ' (' + l + ')' : '' %></th>
+          <th>Dependencies<%= l > 0 ? ' (' + l + ')' : '' %></th>
           <td>
             <%
             if (l === 0) {
@@ -162,27 +162,53 @@
           </td>
         </tr>
 
-        <% if (package.users) {
-          var starredBy = package.starredBy
-          var l = starredBy.length
+        <tr>
+          <%
+            var deps = package.dependents || [];
+            var l = deps.length
+          %>
+          <th>Dependents<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <% if (package.users.length > 0) {
+          var users = package.users
+          var l = users.length
         %>
         <tr>
-          <th>Starred by<%= l > 5 ? ' (' + l + ')' : '' %></th>
+          <th>Starred by (<%= l %>)</th>
           <td>
             <%
             var max = 20
             if (l > max) {
-              starredBy = starredBy.sort(function (a, b) {
+              users = users.sort(function (a, b) {
                 return Math.random() * 2 - 1
               }).slice(0, max)
             }
-            starredBy.forEach(function (user, i) {
+            users.forEach(function (user, i) {
               if (i > 0) { %>, <% }
               %><a href="/~<%= user %>"><%= user %></a><%
             })
             if (l > max) {
               %><br><a href="/browse/star/<%= package.name %>">and
-              <%= (l-max) %> more</a><%
+              <%= (l - max) %> more</a><%
             }
             %>
           </td>

worker.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,7 +16,7 @@
  */
 
 var graceful = require('graceful');
-
+var logger = require('./common/logger');
 var config = require('./config');
 var registry = require('./servers/registry');
 var web = require('./servers/web');
@@ -34,6 +34,7 @@ graceful({
     if (err.message) {
       err.message += ' (uncaughtException throw ' + throwErrorCount + ' times on pid:' + process.pid + ')';
     }
-    console.error(err);
+    console.error(err.stack);
+    logger.error(err);
   }
 });