Release 0.3.7

update to koa@0.5.1

.gitignore

@@ -19,3 +19,7 @@ config/config.js
 backup/*.json
 backup/*.gz
 docs/web/history.md
+view/web/_layout.html
+bin/mysql.js
+bin/test.sql
+coverage/

.jshintignore

@@ -0,0 +1,4 @@
+node_modules/
+coverage/
+.tmp/
+.git/

.jshintrc

@@ -0,0 +1,95 @@
+{
+    // JSHint Default Configuration File (as on JSHint website)
+    // See http://jshint.com/docs/ for more details
+
+    "maxerr"        : 50,       // {int} Maximum error before stopping
+
+    // Enforcing
+    "bitwise"       : true,     // true: Prohibit bitwise operators (&, |, ^, etc.)
+    "camelcase"     : false,    // true: Identifiers must be in camelCase
+    "curly"         : true,     // true: Require {} for every new block or scope
+    "eqeqeq"        : true,     // true: Require triple equals (===) for comparison
+    "forin"         : false,    // true: Require filtering for..in loops with obj.hasOwnProperty()
+    "immed"         : false,    // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
+    "indent"        : false,    // {int} Number of spaces to use for indentation
+    "latedef"       : false,    // true: Require variables/functions to be defined before being used
+    "newcap"        : false,    // true: Require capitalization of all constructor functions e.g. `new F()`
+    "noarg"         : true,     // true: Prohibit use of `arguments.caller` and `arguments.callee`
+    "noempty"       : true,     // true: Prohibit use of empty blocks
+    "nonew"         : false,    // true: Prohibit use of constructors for side-effects (without assignment)
+    "plusplus"      : false,    // true: Prohibit use of `++` & `--`
+    "quotmark"      : false,    // Quotation mark consistency:
+                                //   false    : do nothing (default)
+                                //   true     : ensure whatever is used is consistent
+                                //   "single" : require single quotes
+                                //   "double" : require double quotes
+    "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
+    "unused"        : false,    // true: Require all defined variables be used
+    "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
+    "trailing"      : false,    // true: Prohibit trailing whitespaces
+    "maxparams"     : false,    // {int} Max number of formal params allowed per function
+    "maxdepth"      : false,    // {int} Max depth of nested blocks (within functions)
+    "maxstatements" : false,    // {int} Max number statements per function
+    "maxcomplexity" : false,    // {int} Max cyclomatic complexity per function
+    "maxlen"        : false,    // {int} Max number of characters per line
+
+    // Relaxing
+    "asi"           : false,     // true: Tolerate Automatic Semicolon Insertion (no semicolons)
+    "boss"          : true,      // true: Tolerate assignments where comparisons would be expected
+    "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
+    "eqnull"        : false,     // true: Tolerate use of `== null`
+    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
+    "esnext"        : true,      // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
+                                 // (ex: `for each`, multiple try/catch, function expression…)
+    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`
+    "expr"          : true,      // true: Tolerate `ExpressionStatement` as Programs
+    "funcscope"     : false,     // true: Tolerate defining variables inside control statements"
+    "globalstrict"  : false,     // true: Allow global "use strict" (also enables 'strict')
+    "iterator"      : false,     // true: Tolerate using the `__iterator__` property
+    "lastsemic"     : false,     // true: Tolerate omitting a semicolon for the last statement of a 1-line block
+    "laxbreak"      : true,      // true: Tolerate possibly unsafe line breakings
+    "laxcomma"      : false,     // true: Tolerate comma-first style coding
+    "loopfunc"      : false,     // true: Tolerate functions being defined in loops
+    "multistr"      : true,      // true: Tolerate multi-line strings
+    "proto"         : false,     // true: Tolerate using the `__proto__` property
+    "scripturl"     : false,     // true: Tolerate script-targeted URLs
+    "smarttabs"     : false,     // true: Tolerate mixed tabs/spaces when used for alignment
+    "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
+    "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
+    "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
+    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+
+    // Environments
+    "browser"       : true,     // Web Browser (window, document, etc)
+    "couch"         : false,    // CouchDB
+    "devel"         : true,     // Development/debugging (alert, confirm, etc)
+    "dojo"          : false,    // Dojo Toolkit
+    "jquery"        : false,    // jQuery
+    "mootools"      : false,    // MooTools
+    "node"          : true,     // Node.js
+    "nonstandard"   : false,    // Widely adopted globals (escape, unescape, etc)
+    "prototypejs"   : false,    // Prototype and Scriptaculous
+    "rhino"         : false,    // Rhino
+    "worker"        : false,    // Web Workers
+    "wsh"           : false,    // Windows Scripting Host
+    "yui"           : false,    // Yahoo User Interface
+    "noyield"       : true,     // allow generators without a yield
+
+    // Legacy
+    "nomen"         : false,    // true: Prohibit dangling `_` in variables
+    "onevar"        : false,    // true: Allow only one `var` statement per function
+    "passfail"      : false,    // true: Stop on first error
+    "white"         : false,    // true: Check against strict whitespace and indentation rules
+
+    // Custom Globals
+    "globals"       : {         // additional predefined global variables
+      // mocha
+      "describe": true,
+      "it": true,
+      "before": true,
+      "afterEach": true,
+      "beforeEach": true,
+      "after": true
+    }
+}

.npmignore

@@ -8,3 +8,8 @@ logo.png
 public/dist/
 backup/*.json
 backup/*.gz
+view/web/_layout.html
+bin/mysql.js
+coverage/
+.jshintrc
+.jshintignore

.travis.yml

@@ -1,5 +1,3 @@
 language: node_js
 node_js:
-  - '0.10'
-install: npm install --registry=http://registry.cnpmjs.org
-script: make test-coveralls
+  - '0.11'

AUTHORS

@@ -0,0 +1,8 @@
+# Ordered by date of first contribution.
+# Auto-generated by 'contributors' on Mon, 03 Mar 2014 13:01:28 GMT.
+# https://github.com/xingrz/node-contributors
+
+fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)
+dead-horse <dead_horse@qq.com> (https://github.com/dead-horse)
+alsotang <alsotang@gmail.com> (https://github.com/alsotang)
+4simple <wondger@qq.com> (https://github.com/4simple)

History.md

@@ -1,4 +1,276 @@
 
+0.3.7 / 2014-03-11 
+==================
+
+  * show worker die date time
+  * update to koa@0.5.1
+  * hotfix for star user
+  * fix yield gather, sync missing deps even no missing versions
+  * fix return versions
+  * fix makefile, remove eventproxy
+  * refactor sync_module_worker
+  * add make test-dev, fixed #259
+  * change npm.js to generator
+  * update urllib, proxy/npm.js use generator
+  * sync_all and sync_exist to generator
+  * change function to generator
+  * need node >= v0.11.9
+
+0.3.6 / 2014-03-06 
+==================
+
+  * install missing package should sync it from source npm. fixed #252
+  * npm publish dont contains .jshint*
+  * npm test run jshint
+  * Add jshint check: $ make jshint
+  * use `yield* next` instead of `yield next`
+  * replace dist.u.qiniudn.com with cnpmjs.org/dist
+
+0.3.5 / 2014-03-05 
+==================
+
+  * redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
+  * redirect /name to /package/name when /name is 404. fixed #245
+  * Add missing properies and sync missing star users. fixed #235
+
+0.3.4 / 2014-03-04 
+==================
+
+  * add cov
+  * use istanbul run test coverage
+  * gzip support. fix #241
+  * readme spelling patch (@stanzheng)
+  * default readme to null, fixed #233
+  * remove readme in versions
+
+0.3.3 / 2014-02-28 
+==================
+
+  * Merge pull request #232 from cnpm/host-hotfix
+  * get request host from request.headers
+  * Merge pull request #231 from cnpm/bug-fix
+  * fix deps display bug#230 and nsf.url TypeError#229
+
+0.3.2 / 2014-02-28 
+==================
+
+  * update koa-sess and koa-redis
+  * fix sync all test
+  * remove nfs.downloadStream first, fix tmppath error
+  * fix fengmk2/giturl#1 bug
+
+0.3.1 / 2014-02-27 
+==================
+
+  * add etag fixed #224
+  * travis ci install on source npm
+
+0.3.0 / 2014-02-27 
+==================
+
+  * fix typo and dont sync not exists pkgs
+  * use koa-middlewares
+  * fix signed cookie not work on npm@1.3.25; node --harmony-generators
+  * fix opensearch test case
+  * update koa bodyparser
+  * logger.error(err) should send err stack email notice
+  * json body parse limit and bug fix.
+  * fix sync 404 reason not clear
+  * all controllers to koa
+  * controller/web/user.js to koa
+  * change web connect to koa
+  * use outputError
+  * use yield exports.addPackageAndDist.call(this, next);
+  * add end() when ws write end
+  * fix yield coWrite
+  * fix all the test of registry module.test.js
+  * convert registry/module.js to koa type
+  * fix auth middleware
+  * finish registry user controller koa and update mm to support thunkify. fixed #196
+  * change controllers/user.js to koa
+  * thunkify all proxy
+  * convert all middlewares to koa type
+  * change regsitry sync to koa
+  * addd koa-jsonp, koa-bodyparser, fix / controller
+  * first koa run registry home page /
+  * Merge pull request #212 from cnpm/fix-sync-404
+  * return friendly 404 reason
+  * Merge pull request #211 from cnpm/bug-fix
+  * override json limit to default 10mb. fixed #209
+  * fix #210 addPackageAndDist package version detect bug
+
+0.2.27 / 2014-02-19 
+==================
+
+  * support json result in search, fixed #189
+
+0.2.26 / 2014-02-19 
+==================
+
+  * npm publish also need to add deps
+
+0.2.25 / 2014-02-19 
+==================
+
+  * max handle number of package.json `dependencies` property
+  * Dependents support. fixed #190
+
+0.2.24 / 2014-02-13 
+==================
+
+  * fix if delete all the versions
+  * refactor remove module, fixed #186
+
+0.2.23 / 2014-01-26 
+==================
+
+  * system admin can add, publish, remove the packages. fixed #176
+
+0.2.22 / 2014-01-26 
+==================
+
+  * add keyword and search support keyword. #181
+
+0.2.21 / 2014-01-24 
+==================
+
+  * refactor code styles on package.html
+  * nav-tabs e.preventDefault
+  * Show registry server error response. fixed #178
+  * nav-tabs for package.html (@4simple)
+
+0.2.20 / 2014-01-23 
+==================
+
+  * hotfix sync missing dependencies and readmes
+  * fix sync readme error, fixed #174
+  * add updateReadme in module
+
+0.2.19 / 2014-01-22 
+==================
+
+  * npm install no need to check authorization header. fixed #171
+
+0.2.18 / 2014-01-20 
+==================
+
+  * Support gitlab git url to display and click. fixed #160
+  * fix redis crash
+
+0.2.17 / 2014-01-17 
+==================
+
+  * custom logo url
+  * hotfix layout bug
+
+0.2.16 / 2014-01-16 
+==================
+
+  * fix publish-time bug
+
+0.2.15 / 2014-01-16 
+==================
+
+  * add publish_time to debug
+
+0.2.14 / 2014-01-16 
+==================
+
+  * add make autod
+  * update publish_time, fixed #163
+
+0.2.13 / 2014-01-15 
+==================
+
+  * markdown tmpl not support footer, need to wrap on app start
+
+0.2.12 / 2014-01-15 
+==================
+
+  * add footer and npm client name customable
+
+0.2.11 / 2014-01-15 
+==================
+
+  * package page contributor link to search, default is true
+
+0.2.10 / 2014-01-14 
+==================
+
+  * fix #155 Content-Disposition wrong.
+
+0.2.9 / 2014-01-14 
+==================
+
+  * support startkey=c and startkey="c"
+  * support couch db search api. fixed #153
+  * fix fork me image link
+  * support sync by query.name
+
+0.2.8 / 2014-01-14 
+==================
+
+  * dont show err stack on test env
+  * add download link for package page
+
+0.2.7 / 2014-01-13 
+==================
+
+  * add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
+
+0.2.6 / 2014-01-13 
+==================
+
+  * support custom session store, fixed #146
+
+0.2.5 / 2014-01-13 
+==================
+
+  * add download timeout and unit test
+  * use downloadStream() first
+  * nfs download to a writeable stream.
+
+0.2.4 / 2014-01-10 
+==================
+
+  * set main script to  index.js, fixed #142
+
+0.2.3 / 2014-01-10 
+==================
+
+  * Dont show sync button on private package
+  * Sync package as publish with no deps. fixed #138
+
+0.2.2 / 2014-01-10 
+==================
+
+  * keep compatibility
+  * qnfs upload only callback a url
+  * compat remove package
+  * set tarball url
+  * new npm publish in one request, add _publish_in_cnpm
+  * support unsure name ufs
+  * contributors maybe a object
+  * Object #<Object> has no method 'forEach' fixed #134
+  * support custom config as a module, fixed issue #132
+  * support npm new publish flow. fixed #129
+  * add toString and constructor to test admin
+  * fix #119 hasOwnProperty check admin bug.
+
+0.2.0 / 2013-12-27 
+==================
+
+  * remove to lower case
+  * fix #127 execSync and execsync.
+  * add contributors list on package page
+  * mv blanket to config
+  * sync typeerror fix #statusCode
+  * add disturl
+  * fix #122 admin security bug
+  * fixed #121, let pkg 404 as success
+  * fix sql insert error
+  * fix typos
+
 0.1.3 / 2013-12-20
 ==================
 

Makefile

@@ -4,27 +4,47 @@ TIMEOUT = 30000
 MOCHA_OPTS =
 
 install:
-	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm
+	@npm install --registry=http://registry.cnpmjs.org \
+		--disturl=http://cnpmjs.org/dist
 
-test: install
+jshint:
+	@-./node_modules/.bin/jshint ./
+
+test:
 	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
+		--harmony-generators \
+		--reporter $(REPORTER) \
+		--timeout $(TIMEOUT) \
+		--require should \
+		$(MOCHA_OPTS) \
+		$(TESTS)
+
+
+test-cov:
+	@NODE_ENV=test node --harmony \
+		node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
+		-- -u exports \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		$(MOCHA_OPTS) \
 		$(TESTS)
+	@-$(MAKE) check-coverage
 
-test-cov:
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
+check-coverage:
+	@./node_modules/.bin/istanbul check-coverage \
+		--statements 100 \
+		--functions 100 \
+		--branches 100 \
+		--lines 100
 
-test-cov-html:
-	@rm -f coverage.html
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
-	@ls -lh coverage.html
+cov:
+	@./node_modules/.bin/cov coverage
 
-test-coveralls: test
-	@echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
-	@-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
+contributors:
+	@./node_modules/.bin/contributors -f plain -o AUTHORS
 
-test-all: test test-cov
+autod:
+	@./node_modules/.bin/autod -w -e public,view,docs,backup,coverage
+	@$(MAKE) install
 
 .PHONY: test

README.md

@@ -1,46 +1,56 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)[![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
-![logo](https://raw.github.com/fengmk2/cnpmjs.org/master/logo.png)
+![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
+* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
+
 ## Install
 
 ```bash
-$ npm install
+$ npm install --registry=http://r.cnpmjs.org --disturl=http://cnpmjs.org/dist
 ```
 
 ## Usage
 
 ```js
-$ node dispatch.js
+$ node --harmony-generators dispatch.js
 ```
 
+**Notice**: need node version >=0.11.9
+
+## Guide
+
+* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
+* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
+
 ## Authors
 
 ```bash
 $ git summary
 
  project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
+ repo age : 3 months
+ active   : 145 days
+ commits  : 366
+ files    : 94
  authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
+   217  fengmk2                 59.3%
+   146  dead_horse              39.9%
+     2  4simple                 0.5%
+     1  Alsotang                0.3%
 ```
 
 ## License

bin/nodejsctl

@@ -7,7 +7,7 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS=node
+NODEJS='node --harmony-generators'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
 STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\

bin/restore_module_deps.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - bin/restore_module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var Module = require('../proxy/module');
+var ModuleDeps = require('../proxy/module_deps');
+
+var addCount = 0;
+
+function restore(id, callback) {
+  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
+  mysql.query(sql, [id], function (err, rows) {
+    if (err) {
+      return callback(err);
+    }
+    if (rows.length === 0) {
+      return callback(null, []);
+    }
+
+    console.log('[%s] got %d rows', id, rows.length);
+
+    rows.forEach(function (r) {
+      Module.parseRow(r);
+      if (!r.package) {
+        return;
+      }
+      var deps = Object.keys(r.package.dependencies || {});
+      if (!Array.isArray(deps) || !deps.length) {
+        return;
+      }
+      deps.forEach(function (dep) {
+        ModuleDeps.add(dep, r.name, function (err) {
+          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
+        });
+      });
+      addCount += deps.length;
+    });
+    setTimeout(function () {
+      console.log('[%s] add %d relations', id, addCount);
+      callback(null, rows);
+    }, 1000);
+  });
+}
+
+var id = 0;
+function run() {
+  restore(id, function (err, rows) {
+    if (err) {
+      throw err;
+    }
+    if (rows.length === 0) {
+      console.log('finished, last id: %s, exit.', id);
+      process.exit(0);
+    }
+
+    id = rows[rows.length - 1].id;
+    run();
+  });
+}
+
+run();

common/logger.js

@@ -1,7 +1,12 @@
-/*!
+/**!
  * cnpmjs.org - common/logger.js
- * Copyright(c) 2013 
- * Author: dead_horse <undefined>
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
  */
 
 'use strict';
@@ -10,11 +15,12 @@
  * Module dependencies.
  */
 
-var config = require('../config');
 var util = require('util');
 var moment = require('moment');
 var logstream = require('logfilestream');
 var ms = require('ms');
+var config = require('../config');
+var mail = require('./mail');
 
 var isTEST = process.env.NODE_ENV === 'test';
 var ONE_DAY = ms('1d');
@@ -29,7 +35,9 @@ levels.forEach(function (catetory) {
   var stream = logstream(options);
   function write(msg) {
     var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
+    var subject = null;
     if (msg instanceof Error) {
+      subject = msg.name;
       var err = {
         name: msg.name,
         code: msg.code,
@@ -57,12 +65,21 @@ levels.forEach(function (catetory) {
     } else {
       msg = time + ' ' + util.format.apply(util, arguments) + '\n';
     }
+
     if (!isTEST) {
+      stream.write(msg);
       if (config.debug) {
         var level = catetory;
         console.log('[' + level + '] ' + msg);
       } else {
-        stream.write(msg);
+        if (catetory === 'error' && subject) {
+          // send error email
+          var to = [];
+          for (var name in config.admins) {
+            to.push(config.admins[name]);
+          }
+          mail.error(to, subject, msg);
+        }
       }
     }
   }

common/mail.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - common/mail.js
  *
  * Copyright(c) cnpmjs.org and other contributors.

common/mysql.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var ready = require('ready');
 var mysql = require('mysql');
 var config = require('../config');
@@ -35,7 +36,13 @@ var pool = mysql.createPool({
 exports.pool = pool;
 
 exports.query = function (sql, values, cb) {
-  pool.query(sql, values, cb);
+  if (typeof values === 'function') {
+    cb = values;
+    values = null;
+  }
+  pool.query(sql, values, function (err, rows) {
+    cb(err, rows);
+  });
 };
 
 exports.queryOne = function (sql, values, cb) {
@@ -57,6 +64,8 @@ exports.escape = function (val) {
 
 ready(exports);
 
+thunkify(exports);
+
 function init() {
   exports.query('show tables', function (err, rows) {
     if (err) {

common/qnfs.js

@@ -14,11 +14,13 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var qn = require('qn');
 var config = require('../config');
-
 var client = qn.create(config.qn);
 
+exports._client = client;
+
 /**
  * Upload file
  *
@@ -32,7 +34,23 @@ var client = qn.create(config.qn);
  */
 exports.upload = function (filepath, options, callback) {
   client.delete(options.key, function (err, data) {
-    client.uploadFile(filepath, {key: options.key, size: options.size}, callback);
+    client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
+  });
+};
+
+exports.uploadBuffer = function (buf, options, callback) {
+  client.delete(options.key, function (err, data) {
+    client.upload(buf, {key: options.key}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
   });
 };
 
@@ -43,3 +61,5 @@ exports.url = function (key) {
 exports.remove = function (key, callback) {
   client.delete(key, callback);
 };
+
+thunkify(exports);

common/session.js

@@ -15,27 +15,18 @@
  * Module dependencies.
  */
 
-var connect = require('connect');
-var RedisStore = require('connect-redis')(connect);
+var middlewares = require('koa-middlewares');
 var config = require('../config');
 
-var session;
 var key = 'AuthSession';
-var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 30 };
+var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 365, signed: false };
+var options = {
+  key: key,
+  cookie: cookie,
+};
 
-if (config.debug) {
-  session = connect.cookieSession({
-    secret: config.sessionSecret,
-    key: key,
-    cookie: cookie
-  });
-} else {
-  session = connect.session({
-    key: key,
-    secret: config.sessionSecret,
-    store: new RedisStore(config.redis),
-    cookie: cookie,
-  });
+if (!config.debug) {
+  options.store = config.sessionStore || middlewares.RedisStore(config.redis);
 }
 
-module.exports = session;
+module.exports = middlewares.session(options);

config/index.js

@@ -17,6 +17,7 @@
 
 var path = require('path');
 var fs = require('fs');
+var os = require('os');
 var mkdirp = require('mkdirp');
 
 fs.existsSync = fs.existsSync || path.existsSync;
@@ -29,6 +30,7 @@ var config = {
   registryPort: 7001,
   webPort: 7002,
   enableCluster: false,
+  numCPUs: os.cpus().length,
   debug: true, // if debug
   logdir: path.join(root, '.tmp', 'logs'),
   viewCache: false,
@@ -51,6 +53,7 @@ var config = {
     port: 19533,
     pass: 'cnpmjs_dev'
   },
+  jsonLimit: '10mb', // max request json body size
   uploadDir: path.join(root, 'public', 'dist'),
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
   qn: {
@@ -68,14 +71,20 @@ var config = {
     user: 'address@gmail.com',
     pass: 'your password',
     ssl: true,
-    debug: false 
+    debug: false
   },
 
+  disturl: 'http://dist.u.qiniudn.com',
+  logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
+  registryHost: 'r.cnpmjs.org',
+  customFooter: '', // you can add copyright and site total script html here
+  npmClientName: 'cnpm', // use `${name} install package`
+  packagePageContributorSearch: true, // package page contributor link to search, default is true
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
-    admin: 'admin@cnpmjs.org',
     fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
     dead_horse: 'dead_horse@qq.com',
     cnpmjstest10: 'cnpmjstest10@cnpmjs.org',
   },
@@ -83,6 +92,7 @@ var config = {
   backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
   syncModel: 'none', // 'none', 'all', 'exist'
   syncConcurrency: 1,
+  maxDependencies: 200, // max handle number of package.json `dependencies` property
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js
@@ -98,3 +108,12 @@ mkdirp.sync(config.logdir);
 mkdirp.sync(config.uploadDir);
 
 module.exports = config;
+
+config.loadConfig = function (customConfig) {
+  if (!customConfig) {
+    return;
+  }
+  for (var key in customConfig) {
+    config[key] = customConfig[key];
+  }
+};

controllers/download.js

@@ -14,10 +14,15 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var moment = require('moment');
 var DownloadTotal = require('../proxy/download');
 
 exports.total = function (name, callback) {
+  if (typeof name === 'function') {
+    callback = name;
+    name = null;
+  }
   var end = moment();
   var start = end.clone().subtract('months', 1).startOf('month');
   var lastday = end.clone().subtract('days', 1).format('YYYY-MM-DD');
@@ -74,3 +79,5 @@ exports.total = function (name, callback) {
 
   DownloadTotal[method].apply(DownloadTotal, args);
 };
+
+thunkify(exports);

controllers/registry/user.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - controllers/registry/user.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,32 +16,25 @@
  */
 
 var debug = require('debug')('cnpmjs.org:controllers:registry');
-var logger = require('../../common/logger');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  User.get(name, function (err, row) {
-    if (err) {
-      return next(err);
-    }
-    if (!row) {
-      return next();
-    }
-
-    res.setHeader('etag', '"' + row.rev + '"');
-    var data = {
-      _id: 'org.couchdb.user:' + row.name,
-      _rev: row.rev,
-      name: row.name,
-      email: row.email,
-      type: 'user',
-      roles: [],
-      date: row.gmt_modified,
-    };
-    res.json(data);
-  });
+exports.show = function *(next) {
+  var name = this.params.name;
+  var user = yield User.get(name);
+  if (!user) {
+    return yield* next;
+  }
+  this.etag = '"' + user.rev + '"';
+  var data = {
+    _id: 'org.couchdb.user:' + user.name,
+    _rev: user.rev,
+    name: user.name,
+    email: user.email,
+    type: 'user',
+    roles: [],
+    date: user.gmt_modified,
+  };
+  this.body = data;
 };
 
 // json:
@@ -53,110 +46,109 @@ exports.show = function (req, res, next) {
 //    type: 'user',
 //    roles: [],
 //    date: '2013-12-04T12:56:13.714Z' } }
-exports.add = function (req, res, next) {
-  var name = req.params.name;
-  var body = req.body || {};
+exports.add = function *() {
+  var name = this.params.name;
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     // roles: body.roles || [],
   };
+
   if (!user.name || !user.salt || !user.password_sha || !user.email) {
-    return res.json(422, {
+    this.status = 422;
+    this.body = {
       error: 'paramError',
       reason: 'params missing'
-    });
+    };
+    return;
   }
   debug('add user: %j', user);
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  User.get(name, ep.doneLater(function (row) {
-    if (row) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    User.add(user, ep.done('add'));
-  }));
+  var existUser = yield User.get(name);
+  if (existUser) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
 
-  ep.once('add', function (result) {
-    res.setHeader('etag', '"' + result.rev + '"');
-    // location: 'http://registry.npmjs.org/_users/org.couchdb.user:cnpmjstest1',
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.add(user);
+  this.etag = '"' + result.rev + '"';
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
 };
 
-exports.authSession = function (req, res, next) {
+exports.authSession = function *() {
   // body: {"name":"foo","password":"****"}
-  var body = req.body || {};
+  var body = this.request.body || {};
   var name = body.name;
   var password = body.password;
-  User.auth(name, password, function (err, user) {
-    debug('authSession %s: %j', name, user);
-    if (err) {
-      return next(err);
-    }
-    if (!user) {
-      return res.json(401, {ok: false, name: null, roles: []});
-    }
+  var user = yield User.auth(name, password);
+  debug('authSession %s: %j', name, user);
 
-    req.session.name = user.name;
-    res.json(200, {ok: true, name: user.name, roles: []});
-  });
+  if (!user) {
+    this.status = 401;
+    this.body = {ok: false, name: null, roles: []};
+    return;
+  }
+
+  this.session.name = user.name;
+  this.body = {ok: true, name: user.name, roles: []};
 };
 
-exports.update = function (req, res, next) {
-  var name = req.params.name;
-  var rev = req.params.rev;
+exports.update = function *(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
   if (!name || !rev) {
-    return next();
+    return yield* next;
   }
 
-  debug('update: %s, rev: %s, session.name: %s', name, rev, req.session.name);
+  debug('update: %s, rev: %s, session.name: %s', name, rev, this.session.name);
 
-  if (name !== req.session.name) {
+  if (name !== this.session.name) {
     // must authSession first
-    res.statusCode = 401;
-    return res.json({
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Name is incorrect.'
-    });
+    };
+    return;
   }
 
-  var body = req.body || {};
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     rev: body.rev || body._rev,
     // roles: body.roles || [],
   };
-  User.update(user, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    //check rev error
-    if (!result) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + user.name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.update(user);
+  if (!result) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
 };

controllers/sync.js

@@ -13,38 +13,63 @@
 /**
  * Module dependencies.
  */
+
 var Log = require('../proxy/module_log');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 
-exports.sync = function (req, res, next) {
-  var username = req.session.name || 'anonymous';
-  var name = req.params.name;
-  SyncModuleWorker.sync(name, username, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    if (!result.ok) {
-      return res.json(result.statusCode, result.pkg);
-    }
-    res.json(201, {
-      ok: true,
-      logId: result.logId
-    });
-  });
+exports.sync = function *() {
+  var username = this.session.name || 'anonymous';
+  var name = this.params.name;
+  var publish = this.query.publish === 'true';
+  var noDep = this.query.nodeps === 'true';
+  if (publish && !this.session.isAdmin) {
+    this.status = 403;
+    this.body = {
+      error: 'no_perms',
+      reason: 'Only admin can publish'
+    };
+    return;
+  }
+
+  var options = {
+    publish: publish,
+    noDep: noDep,
+  };
+
+  var result = yield SyncModuleWorker.sync(name, username, options);
+
+  // friendly 404 reason info
+  if (result.staticCache === 404) {
+    this.status = 404;
+    this.body = {
+      ok: false,
+      reason: 'can not found ' + name + ' in the source registry'
+    };
+    return;
+  }
+  if (!result.ok) {
+    this.status = result.statusCode;
+    this.body = result.pkg;
+    return;
+  }
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logId: result.logId
+  };
 };
 
-exports.getSyncLog = function (req, res, next) {
-  var logId = req.params.id;
-  var name = req.params.name;
-  var offset = Number(req.query.offset) || 0;
-  Log.get(logId, function (err, row) {
-    if (err || !row) {
-      return next(err);
-    }
-    var log = row.log.trim();
-    if (offset > 0) {
-      log = log.split('\n').slice(offset).join('\n');
-    }
-    res.json(200, {ok: true, log: log});
-  });
+exports.getSyncLog = function *(next) {
+  var logId = this.params.id;
+  var offset = Number(this.query.offset) || 0;
+  var row = yield Log.get(logId);
+  if (!row) {
+    return yield* next;
+  }
+
+  var log = row.log.trim();
+  if (offset > 0) {
+    log = log.split('\n').slice(offset).join('\n');
+  }
+  this.body = {ok: true, log: log};
 };

controllers/total.js

@@ -16,31 +16,25 @@
  */
 
 var microtime = require('microtime');
-var eventproxy = require('eventproxy');
 var Total = require('../proxy/total');
-var down = require('./download');
+var Download = require('./download');
 var version = require('../package.json').version;
 var config = require('../config');
 
 var startTime = '' + microtime.now();
 
-exports.show = function (req, res, next) {
-  var ep = eventproxy.create();
-  ep.fail(next);
+exports.show = function *() {
+  var r = yield [Total.get(), Download.total()];
+  var total = r[0];
+  var download = r[1];
 
-  Total.get(ep.done('total'));
-  down.total(null, ep.done('download'));
-  ep.all('total', 'download', function (total, download) {
-    total.download = download;
-    total.db_name = 'registry';
-    total.instance_start_time = startTime;
-    total.node_version = process.version;
-    total.app_version = version;
-    total.donate = 'https://me.alipay.com/imk2';
-    total.sync_model = config.syncModel;
-    if (req.query.callback) {
-      return res.jsonp(total, req.query.callback);
-    }
-    res.json(total);
-  });
+  total.download = download;
+  total.db_name = 'registry';
+  total.instance_start_time = startTime;
+  total.node_version = process.version;
+  total.app_version = version;
+  total.donate = 'https://me.alipay.com/imk2';
+  total.sync_model = config.syncModel;
+
+  this.body = total;
 };

controllers/web/dist.js

@@ -0,0 +1,23 @@
+/**!
+ * cnpmjs.org - controllers/web/dist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../../config');
+
+exports.redirect = function *(next) {
+  var params = this.params;
+  var url = config.disturl + (params[0] || '/');
+  this.redirect(url);
+};

controllers/web/package.js

@@ -14,87 +14,164 @@
  * Module dependencies.
  */
 
+var giturl = require('giturl');
 var moment = require('moment');
 var eventproxy = require('eventproxy');
 var semver = require('semver');
 var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
+var config = require('../../config');
 var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var ModuleDeps = require('../../proxy/module_deps');
+var setDownloadURL = require('../../lib/common').setDownloadURL;
+var ModuleStar = require('../../proxy/module_star');
 
-exports.display = function (req, res, next) {
-  var params = req.params;
+exports.display = function *(next) {
+  var params = this.params;
   var name = params.name;
   var tag = params.version;
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  if (tag) {
-    var version = semver.valid(tag);
-    if (version) {
-      Module.get(name, version, ep.done('pkg'));
-    } else {
-      Module.getByTag(name, tag, ep.done('pkg'));
-    }
+  var getPackageMethod;
+  var getPackageArgs;
+  var version = semver.valid(tag || '');
+  if (version) {
+    getPackageMethod = 'get';
+    getPackageArgs = [name, version];
   } else {
-    Module.getByTag(name, 'latest', ep.done('pkg'));
+    getPackageMethod = 'getByTag';
+    getPackageArgs = [name, tag || 'latest'];
+  }
+  var r = yield [
+    Module[getPackageMethod].apply(Module, getPackageArgs),
+    down.total(name),
+    ModuleDeps.list(name),
+    ModuleStar.listUsers(name),
+  ];
+  var pkg = r[0];
+  var download = r[1];
+  var dependents = (r[2] || []).map(function (item) {
+    return item.deps;
+  });
+  var users = r[3];
+
+  if (!pkg || !pkg.package) {
+    return yield* next;
   }
 
-  down.total(name, ep.done('download'));
+  pkg.package.fromNow = moment(pkg.publish_time).fromNow();
+  pkg = pkg.package;
+  pkg.users = users;
+  pkg.readme = marked(pkg.readme || '');
+  if (!pkg.readme) {
+    pkg.readme = pkg.description || '';
+  }
 
-  ep.all('pkg', 'download', function (pkg, download) {
-    if (!pkg || !pkg.package) {
-      return next();
-    }
-
-    pkg.package.fromNow = moment(pkg.publish_time).fromNow();
-    pkg = pkg.package;
-    pkg.readme = marked(pkg.readme || '');
-    if (pkg.maintainers) {
-      for (var i = 0; i < pkg.maintainers.length; i++) {
-        var maintainer = pkg.maintainers[i];
-        if (maintainer.email) {
-          maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
-        }
+  if (pkg.maintainers) {
+    for (var i = 0; i < pkg.maintainers.length; i++) {
+      var maintainer = pkg.maintainers[i];
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
       }
     }
+  }
 
-    setLicense(pkg);
-
-    for (var k in download) {
-      download[k] = humanize(download[k]);
+  if (pkg.contributors) {
+    // registry.cnpmjs.org/compressible
+    if (!Array.isArray(pkg.contributors)) {
+      pkg.contributors = [pkg.contributors];
     }
+    for (var i = 0; i < pkg.contributors.length; i++) {
+      var contributor = pkg.contributors[i];
+      if (contributor.email) {
+        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
+      }
+      if (config.packagePageContributorSearch || !contributor.url) {
+        contributor.url = '/~' + encodeURIComponent(contributor.name);
+      }
+    }
+  }
 
-    res.render('package', {
-      title: 'Package - ' + pkg.name,
-      package: pkg,
-      download: download
-    });
+  if (pkg.repository && pkg.repository.url) {
+    pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
+  }
+
+  setLicense(pkg);
+
+  for (var k in download) {
+    download[k] = humanize(download[k]);
+  }
+  setDownloadURL(pkg, this, config.registryHost);
+
+  pkg.dependents = dependents;
+
+  yield this.render('package', {
+    title: 'Package - ' + pkg.name,
+    package: pkg,
+    download: download
   });
 };
 
-exports.search = function (req, res, next) {
-  var params = req.params;
-  var word = req.params.word;
-  Module.search(word, function (err, packages) {
-    if (err) {
-      return next(err);
-    }
-
-    res.render('search', {
-      title: 'Keyword - ' + word,
+exports.search = function *(next) {
+  var params = this.params;
+  var word = params.word;
+  var result = yield Module.search(word);
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.body = {
       keyword: word,
-      packages: packages || []
-    });
+      packages: result.searchMatchs,
+      keywords: result.keywordMatchs
+    };
+    this.type = 'application/json; charset=utf-8';
+    return;
+  }
+
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
   });
 };
 
-exports.displaySync = function (req, res, next) {
-  var name = req.params.name;
-  res.render('sync', {
+exports.rangeSearch = function *(next) {
+  var startKey = this.query.startkey || '';
+  if (startKey[0] === '"') {
+    startKey = startKey.substring(1);
+  }
+  if (startKey[startKey.length - 1] === '"') {
+    startKey = startKey.substring(0, startKey.length - 1);
+  }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield Module.search(startKey, {limit: limit});
+
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
+
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
+};
+
+exports.displaySync = function *(next) {
+  var name = this.params.name || this.query.name;
+  yield this.render('sync', {
     name: name,
     title: 'Sync - ' + name
   });

controllers/web/user.js

@@ -15,31 +15,24 @@
  */
 var Module = require('../../proxy/module');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.display = function (req, res, next) {
-  var name = req.params.name;
+exports.display = function *(next) {
+  var name = this.params.name;
 
-  var ep = eventproxy.create();
-  ep.fail(next);
-  Module.listByAuthor(name, ep.done('packages'));
-  User.get(name, ep.done('user'));
+  var r = yield [Module.listByAuthor(name), User.get(name)];
+  var packages = r[0];
+  var user = r[1];
+  if (!user && !packages.length) {
+    return yield* next;
+  }
+  user = {
+    name: name,
+    email: user && user.email
+  };
 
-  ep.all('packages', 'user', function (packages, user) {
-    //because of sync, maybe no this user in database,
-    //but his packages in this registry
-    if (!user && !packages.length) {
-      return next();
-    }
-    user = {
-      name: name,
-      email: user && user.email
-    };
-
-    return res.render('profile', {
-      title: 'User - ' + name,
-      packages: packages || [],
-      user: user
-    });
+  yield this.render('profile', {
+    title: 'User - ' + name,
+    packages: packages || [],
+    user: user
   });
 };

dispatch.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - dispatch.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,7 +17,6 @@
 
 var path = require('path');
 var util = require('util');
-var fs = require('fs');
 var cluster = require('cluster');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
@@ -30,25 +29,25 @@ if (config.enableCluster) {
   });
 
   cluster.on('fork', function (worker) {
-    console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
+    console.log('[%s] [worker:%d] new worker start', Date(), worker.process.pid);
   });
 
   cluster.on('disconnect', function (worker) {
     var w = cluster.fork();
-    console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
-      new Date(), process.pid, worker.process.pid, w.process.pid);
+    console.error('[%s] [master:%s] wroker:%s disconnect, suicide: %s, state: %s. New worker:%s fork',
+      Date(), process.pid, worker.process.pid, worker.suicide, worker.state, w.process.pid);
   });
 
   cluster.on('exit', function (worker, code, signal) {
     var exitCode = worker.process.exitCode;
-    var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
+    var err = new Error(util.format('worker %s died (code: %s, signal: %s, suicide: %s, state: %s)',
+      worker.process.pid, exitCode, signal, worker.suicide, worker.state));
     err.name = 'WorkerDiedError';
-    console.error(err);
+    console.error('[%s] [master:%s] wroker exit: %s', Date(), process.pid, err.stack);
   });
 
-  var numCPUs = require('os').cpus().length;
   // Fork workers.
-  for (var i = 0; i < numCPUs; i++) {
+  for (var i = 0; i < config.numCPUs; i++) {
     cluster.fork();
   }
 

docs/db.sql

@@ -14,12 +14,33 @@ CREATE TABLE `user` (
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
 
+CREATE TABLE `module_keyword` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `description` longtext,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `keyword_module_name` (`keyword`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
+
+CREATE TABLE `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
+
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `author` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `description` longtext,
  `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
@@ -38,23 +59,25 @@ CREATE TABLE `module` (
 -- ALTER TABLE `module` CHANGE `package` `package` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- ALTER TABLE `module` CHANGE `description` `description` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- show create table module\G
+-- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `module_log` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `username` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `log` longtext,
  PRIMARY KEY (`id`),
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+-- ALTER TABLE  `module_log` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `tag` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `tag` varchar(30) NOT NULL COMMENT 'tag name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
@@ -62,6 +85,7 @@ CREATE TABLE `tag` (
  UNIQUE KEY `name` (`name`, `tag`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
 -- ALTER TABLE  `tag` ADD  `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
+-- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `total` (
  `name` varchar(100) NOT NULL COMMENT 'total name',
@@ -77,7 +101,7 @@ CREATE TABLE `total` (
  `last_sync_module` varchar(100) COMMENT 'last sync success module name',
  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
-INSERT INTO total(name, gmt_modified, module_delete) VALUES('total', now(), 0, 0);
+INSERT INTO total(name, gmt_modified) VALUES('total', now());
 -- ALTER TABLE `total` ADD `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry'
 -- ALTER TABLE `total` ADD `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry'
 -- ALTER TABLE `total` ADD `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status'
@@ -92,8 +116,19 @@ CREATE TABLE `download_total` (
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
  PRIMARY KEY (`id`),
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

docs/web/install.md

@@ -91,23 +91,25 @@ $ open http://localhost:7001
 $ open http://localhost:7002
 ```
 
-## use cnpm cli with your own registry  
-You do not need to write another command line tool with your own registry, 
+## use cnpm cli with your own registry
+You do not need to write another command line tool with your own registry,
 just alias [cnpm](http://github.com/fengmk2/cnpm), then you can get a npm client for you own registry.
 
 ```
 # install cnpm first
-npm install -g cnpm 
+npm install -g cnpm
 
 # then alias lnpm to cnpm, but change config to your own registry
 alias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'
 
  #or put this in .zshrc or .bashrc
  echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
 ```

docs/web/readme.md

@@ -1,14 +1,15 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org) [![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
 ## What is this?
 
-> Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+> Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
+* @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ## Install your private npm registry
 
@@ -16,7 +17,7 @@
 
 ## Registry
 
-* Our public registry: [registry.cnpmjs.org](http://registry.cnpmjs.org)
+* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
 * Current [cnpmjs.org](/) version: <span id="app-version"></span>
 
 <table class="downloads">
@@ -119,13 +120,15 @@ $(function () {
 alias it:
 
 ```bash
-alias cnpm="npm --registry=http://registry.cnpmjs.org \
+alias cnpm="npm --registry=http://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
+--disturl=http://cnpmjs.org/dist \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
+  --disturl=http://cnpmjs.org/dist \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -137,7 +140,7 @@ $ npm install cnpm -g
 
 ### install
 
-Install package from [registry.cnpmjs.org](http://registry.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
 ```
 $ cnpm install [name]
@@ -166,12 +169,17 @@ $ cnpm publish [name]
 $ cnpm unpublish [name]
 ```
 
-### Orther command
-Support all the other npm command.
+### Other commands
+
+Support all the other npm commands. e.g.:
+
+```bash
+$ cnpm info cnpm
+```
 
 ## TODO list
 
-@see Github [Issues](https://github.com/fengmk2/cnpmjs.org/issues)
+@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
 
 ## Authors
 
@@ -181,14 +189,15 @@ Release [History](/history).
 $ git summary
 
  project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
+ repo age : 7 weeks
+ active   : 132 days
+ commits  : 315
+ files    : 88
  authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
+   190  fengmk2                 60.3%
+   122  dead_horse              38.7%
+     2  4simple                 0.6%
+     1  Alsotang                0.3%
 ```
 
 ## npm and cnpm relation

index.js

@@ -0,0 +1,30 @@
+/**!
+ * cnpmjs.org - index.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('./config');
+
+exports.loadConfig = config.loadConfig;
+exports.config = config;
+
+exports.startWorker = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./worker');
+};
+
+exports.startSync = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./sync');
+};

lib/common.js

@@ -17,6 +17,7 @@
 var crypto = require('crypto');
 var path = require('path');
 var config = require('../config');
+var util = require('util');
 
 exports.getTarballFilepath = function (filename) {
   // ensure download file path unique
@@ -28,8 +29,29 @@ exports.getCDNKey = function (name, filename) {
   return '/' + name + '/-/' + filename;
 };
 
-exports.downloadURL = function (pkg, req) {
-  if (pkg.dist && pkg.dist.tarball) {
-    pkg.dist.tarball = 'http://' + req.headers.host + '/' + pkg.name + '/download/' + path.basename(pkg.dist.tarball);
+exports.setDownloadURL = function (pkg, ctx, host) {
+  if (pkg.dist) {
+    host = host || ctx.host;
+    pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
+      ctx.protocol,
+      host, pkg.name, pkg.name, pkg.version);
   }
 };
+
+exports.isAdmin = function (username) {
+  return typeof config.admins[username] === 'string';
+};
+
+exports.isMaintainer = function (ctx, maintainers) {
+  if (ctx.session.isAdmin) {
+    return true;
+  }
+
+  var username = ctx.session.name;
+  maintainers = maintainers || [];
+  var match = maintainers.filter(function (item) {
+    return item.name === username;
+  });
+
+  return match.length > 0;
+};

middleware/auth.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - middleware/auth.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,47 +17,49 @@
 var debug = require('debug')('cnpmjs.org:middleware:auth');
 var User = require('../proxy/user');
 var config = require('../config');
+var common = require('../lib/common');
 
 module.exports = function (options) {
-  return function auth(req, res, next) {
-    req.session.onlySync = config.enablePrivate ? true : false;
-    if (req.session.name) {
-      if (config.admins[req.session.name]) {
-        req.session.isAdmin = true;
-      }
-      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
-        req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
-      return next();
+  return function *auth(next) {
+    debug('%s, %s, %j', this.url, this.sessionId, this.session);
+    if (!this.session) {
+      // redis crash
+      this.session = {};
+      return yield *next;
     }
-    var authorization = (req.headers.authorization || '').split(' ')[1] || '';
+    this.session.onlySync = config.enablePrivate ? true : false;
+    if (this.session.name) {
+      this.session.isAdmin = common.isAdmin(this.session.name);
+      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
+        this.session.name, this.session.onlySync, this.session.isAdmin, this.header);
+      return yield *next;
+    }
+    var authorization = (this.get('authorization') || '').split(' ')[1] || '';
     authorization = authorization.trim();
     if (!authorization) {
-      return next();
+      return yield *next;
     }
+
     authorization = new Buffer(authorization, 'base64').toString().split(':');
+    if (authorization.length !== 2) {
+      return yield *next;
+    }
+
     var username = authorization[0];
     var password = authorization[1];
 
-    User.auth(username, password, function (err, row) {
-      if (err) {
-        return next(err);
-      }
+    var row = yield User.auth(username, password);
+    if (!row) {
+      debug('auth fail user: %j, headers: %j', row, this.header);
+      this.session.name = null;
+      this.session.isAdmin = false;
+      return yield *next;
+    }
 
-      if (!row) {
-        debug('auth fail user: %j, headers: %j', row, req.headers);
-        return res.json(401, {
-          error: 'unauthorized',
-          reason: 'Name or password is incorrect.'
-        });
-      }
-
-      req.session.name = row.name;
-      if (config.admins.hasOwnProperty(req.session.name)) {
-        req.session.isAdmin = true;
-      }
-      debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
-        row, req.session.onlySync, req.session.isAdmin, req.headers);
-      next();
-    });
+    this.session.name = row.name;
+    this.session.isAdmin = common.isAdmin(this.session.name);
+    debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
+      row, this.session.onlySync, this.session.isAdmin, this.header);
+    yield *next;
   };
 };

middleware/login.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
-module.exports = function login(req, res, next) {
-  if (!req.session.name) {
-    return res.json(401, {
+module.exports = function *login(next) {
+  if (!this.session.name) {
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Login first.'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/opensearch.js

@@ -24,8 +24,11 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
 
 var lastModifyDate = new Date();
 
-module.exports = function publishable(req, res, next) {
-  res.charset = res.charset || 'utf-8';
-  res.setHeader('Content-Type', 'text/xml');
-  res.send(template.replace('${host}', req.headers.host));
+module.exports = function *publishable(next) {
+  if (this.path === '/opensearch.xml') {
+    this.type = 'text/xml';
+    this.charset = 'utf-8';
+    this.body = template.replace('${host}', this.host);
+  }
+  yield *next;
 };

middleware/publishable.js

@@ -14,13 +14,15 @@
  * Module dependencies.
  */
 
-module.exports = function publishable(req, res, next) {
-  if (req.session.onlySync && !req.session.isAdmin) {
+module.exports = function *publishable(next) {
+  if (this.session.onlySync && !this.session.isAdmin) {
     // private mode, only admin user can publish
-    return res.json(403, {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Private mode enable, only admin can publish this module'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/registry_not_found.js

@@ -0,0 +1,28 @@
+/**!
+ * cnpmjs.org - middleware/registry_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+  this.status = 404;
+  this.body = {
+    error: 'not_found',
+    reason: 'document not found'
+  };
+};

middleware/sync_by_install.js

@@ -14,27 +14,29 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
 var config = require('../config');
 
 /**
  * req.session.allowSync  -  allow sync triggle by cnpm install
  */
 
-module.exports = function (req, res, next) {
+module.exports = function *(next) {
   if (!config.syncByInstall || !config.enablePrivate) {
     // only config.enablePrivate should enable sync on install
-    return next();
+    return yield *next;
   }
   // request not by node, consider it request from web
-  if (req.headers['user-agent'] && req.headers['user-agent'].indexOf('node') !== 0) {
-    return next();
+  if (this.get('user-agent') && this.get('user-agent').indexOf('node') !== 0) {
+    return yield *next;
   }
 
-  req.session.allowSync = true;
-  if (req.session.isAdmin) {
+  this.session.allowSync = true;
+  if (this.session.isAdmin) {
     // if current user is admin, should not enable auto sync on install, because it would be unpublish
-    req.session.allowSync = false;
+    this.session.allowSync = false;
   }
 
-  next();
+  debug('%s allowSync: %s', this.session.name, this.session.allowSync);
+  yield *next;
 };

middleware/web_not_found.js

@@ -0,0 +1,33 @@
+/**!
+ * cnpmjs.org - middleware/web_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+
+  var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  if (m) {
+    return this.redirect('/package/' + m[1]);
+  }
+
+  this.status = 404;
+  this.type = 'text/html';
+  this.charset = 'utf-8';
+  this.body = 'Cannot GET ' + this.path;
+};

package.json

@@ -1,79 +1,76 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.1.3",
-  "description": "Private npm registry and web",
-  "main": "dispatch.js",
+  "version": "0.3.7",
+  "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
+  "main": "index.js",
   "scripts": {
-    "test": "make test-all",
+    "test": "make install && make jshint && make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
-    "stop": "./bin/nodejsctl stop",
-    "blanket": {
-      "pattern": "//^((?!(node_modules|test|common)).)*$/",
-      "data-cover-flags": {
-        "debug": false
-      }
-    },
-    "travis-cov": {
-      "threshold": 90
+    "stop": "./bin/nodejsctl stop"
+  },
+  "config": {
+    "cov": {
+      "threshold": 83
     }
   },
   "dependencies": {
-    "forward": ">=0.0.4",
-    "humanize-number": ">=0.0.2",
+    "co": "3.0.4",
+    "co-gather": "0.0.1",
+    "co-read": "0.0.1",
+    "co-write": "0.3.0",
+    "debug": "0.7.4",
+    "eventproxy": "0.3.0",
+    "giturl": "0.0.2",
+    "graceful": "0.0.6",
     "gravatar": "1.0.6",
-    "urllib": ">=0.5.5",
-    "connect-markdown": "0.0.3",
-    "qn": ">=0.2.0",
+    "humanize-number": "0.0.2",
+    "koa": "0.5.1",
+    "koa-markdown": "0.0.2",
+    "koa-middlewares": "0.0.8",
+    "logfilestream": "0.1.0",
+    "marked": "0.3.1",
     "microtime": "0.5.1",
-    "debug": ">=0.7.4",
-    "utility": ">=0.1.9",
-    "ready": "0.1.1",
-    "connect": "2.12.0",
-    "connect-rt": "0.0.2",
-    "connect-redis": "1.4.6",
-    "connect-render": "0.3.2",
-    "urlrouter": ">=0.5.4",
-    "graceful": ">=0.0.5",
-    "moment": "2.4.0",
-    "logfilestream": ">=0.1.0",
-    "ms": "0.6.2",
+    "mime": "1.2.11",
     "mkdirp": "0.3.5",
-    "mysql": "2.0.0-rc2",
-    "response-patch": "0.1.1",
-    "response-cookie": "0.0.2",
-    "bagpipe": "0.3.5",
+    "moment": "2.5.1",
+    "ms": "0.6.2",
+    "mysql": "2.1.0",
+    "nodemailer": "0.6.1",
+    "qn": "0.2.0",
+    "ready": "0.1.1",
     "semver": "2.2.1",
-    "marked": "0.2.10",
-    "nodemailer": "0.5.15",
-    "eventproxy": ">=0.2.6"
+    "thunkify-wrap": "0.0.5",
+    "urllib": "0.5.8",
+    "utility": "0.1.10"
   },
   "devDependencies": {
-    "supertest": "*",
-    "should": "*",
-    "blanket": "*",
-    "travis-cov": "*",
-    "coveralls": "*",
-    "mocha-lcov-reporter": "*",
+    "autod": ">=0.0.13",
+    "contributors": "*",
+    "cov": "*",
+    "istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",
+    "jshint": "*",
+    "mm": "0.2.0",
     "mocha": "*",
-    "mm": "*",
-    "pedding": "*"
+    "pedding": "0.0.3",
+    "should": "3.1.3",
+    "supertest": "0.9.0"
   },
-  "homepage": "https://github.com/fengmk2/cnpmjs.org",
+  "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
     "type": "git",
-    "url": "git://github.com/fengmk2/cnpmjs.org.git",
-    "web": "https://github.com/fengmk2/cnpmjs.org"
+    "url": "git://github.com/cnpm/cnpmjs.org.git",
+    "web": "https://github.com/cnpm/cnpmjs.org"
   },
   "bugs": {
-    "url": "https://github.com/fengmk2/cnpmjs.org/issues",
+    "url": "https://github.com/cnpm/cnpmjs.org/issues",
     "email": "fengmk2@gmail.com"
   },
   "keywords": [
     "cnpmjs.org", "npm", "npmjs", "npmjs.org", "registry"
   ],
   "engines": {
-    "node": ">= 0.8.0"
+    "node": ">= 0.11.9"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

proxy/download.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var mysql = require('../common/mysql');
 
@@ -41,3 +42,5 @@ var SELECT_ALL_TOTAL_SQL = 'SELECT date, sum(count) AS count \
 exports.getTotal = function (start, end, callback) {
   mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
 };
+
+thunkify(exports);

proxy/module.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var eventproxy = require('eventproxy');
 var config = require('../config');
@@ -29,6 +30,7 @@ var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
     dist_tarball=VALUES(dist_tarball), dist_shasum=VALUES(dist_shasum), dist_size=VALUES(dist_size);';
 
 exports.add = function (mod, callback) {
+  var keywords = mod.package.keywords;
   var pkg;
   try {
     pkg = stringifyPackage(mod.package);
@@ -51,6 +53,74 @@ exports.add = function (mod, callback) {
       return callback(err);
     }
     callback(null, {id: result.insertId, gmt_modified: new Date()});
+
+    if (typeof keywords === 'string') {
+      keywords = [keywords];
+    }
+
+    if (!Array.isArray(keywords)) {
+      return;
+    }
+
+    var words = [];
+    for (var i = 0; i < keywords.length; i++) {
+      var w = keywords[i];
+      if (typeof w === 'string') {
+        w = w.trim();
+        if (w) {
+          words.push(w);
+        }
+      }
+    }
+
+    if (words.length === 0) {
+      return;
+    }
+
+    // add keywords
+    exports.addKeywords(mod, description, words, utility.noop);
+  });
+};
+
+var GET_KEYWORD_SQL = 'SELECT keyword FROM module_keyword WHERE name=? ORDER BY keyword;';
+
+exports.getKeywords = function (name, callback) {
+  mysql.query(GET_KEYWORD_SQL, [name], function (err, rows) {
+    var keywords = [];
+    if (rows && rows.length) {
+      keywords = rows.map(function (r) {
+        return r.keyword;
+      });
+    }
+    callback(err, keywords);
+  });
+};
+
+var ADD_KEYWORD_SQL = 'INSERT INTO module_keyword(gmt_create, keyword, name, description) \
+  VALUES(now(), ?, ?, ?) \
+  ON DUPLICATE KEY UPDATE description=VALUES(description);';
+
+exports.addKeywords = function (name, description, keywords, callback) {
+  var sql = '';
+  var values = [];
+  for (var i = 0; i < keywords.length; i++) {
+    sql += ADD_KEYWORD_SQL;
+    values.push(keywords[i]);
+    values.push(name);
+    values.push(description);
+  }
+  mysql.query(sql, values, function (err, results) {
+    if (err) {
+      return callback(err);
+    }
+    var ids = [];
+    for (var i = 0; i < results.length; i++) {
+      var r = results[i];
+      if (r.insertId) {
+        ids.push(r.insertId);
+      }
+    }
+    callback(null, ids);
   });
 };
 
@@ -59,7 +129,21 @@ exports.updateDescription = function (id, description, callback) {
   mysql.query(UPDATE_DESC_SQL, [description, id], callback);
 };
 
-var UPDATE_DIST_SQL = 'UPDATE module SET version=?, package=?, dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
+var UPDATE_PACKAGE_SQL = 'UPDATE module SET package=? WHERE id=?;';
+exports.updateReadme = function (id, readme, callback) {
+  exports.getById(id, function (err, data) {
+    if (err) {
+      return callback(err);
+    }
+    data.package = data.package || {};
+    data.package.readme = readme;
+    var pkg = stringifyPackage(data.package);
+    mysql.query(UPDATE_PACKAGE_SQL, [pkg, id], callback);
+  });
+};
+
+var UPDATE_DIST_SQL = 'UPDATE module SET publish_time=?, version=?, package=?, \
+  dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
 
 exports.update = function (mod, callback) {
   var pkg;
@@ -69,7 +153,8 @@ exports.update = function (mod, callback) {
     return callback(e);
   }
   var dist = mod.package.dist;
-  mysql.query(UPDATE_DIST_SQL, [mod.version, pkg, dist.tarball, dist.shasum, dist.size, mod.id],
+  mysql.query(UPDATE_DIST_SQL,
+    [mod.publish_time, mod.version, pkg, dist.tarball, dist.shasum, dist.size, mod.id],
   function (err, result) {
     if (err) {
       return callback(err);
@@ -91,6 +176,7 @@ function parseRow(row) {
     }
   }
 }
+exports.parseRow = parseRow;
 
 function stringifyPackage(pkg) {
   return encodeURIComponent(JSON.stringify(pkg));
@@ -171,7 +257,12 @@ exports.removeTags = function (name, callback) {
   mysql.query(DELETE_TAGS_SQL, [name], callback);
 };
 
-var SELECT_ALL_TAGS_SQL = 'SELECT tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
+var DELETE_TAGS_BY_IDS_SQL = 'DELETE FROM tag WHERE id in (?)';
+exports.removeTagsByIds = function (ids, callback) {
+  mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
+};
+
+var SELECT_ALL_TAGS_SQL = 'SELECT id, tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
 
 exports.listTags = function (name, callback) {
   mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
@@ -295,26 +386,69 @@ exports.listByAuthor = function (author, callback) {
   });
 };
 
-var SEARCH_SQLS = [
-  'SELECT module_id FROM tag WHERE name LIKE ? AND  tag="latest" ORDER BY name LIMIT 100;',
-  'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;'
-];
-exports.search = function (word, callback) {
-  word = word.replace(/^%/, '') + '%'; //ignore prefix %
+var SEARCH_MODULES_SQL = 'SELECT module_id FROM tag WHERE name LIKE ? AND tag="latest" ORDER BY name LIMIT ?;';
+var SEARCH_MODULES_BY_KEYWORD_SQL = 'SELECT name, description FROM module_keyword WHERE keyword = ? ORDER BY id DESC LIMIT ?;';
+var QUERY_MODULES_BY_ID_SQL = 'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;';
+
+exports.search = function (word, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
+  var limit = options.limit || 100;
+  word = word.replace(/^%/, ''); //ignore prefix %
   var ep = eventproxy.create();
   ep.fail(callback);
-  mysql.query(SEARCH_SQLS[0], [word], ep.done(function (rows) {
-    if (!rows || rows.length === 0) {
-      return callback(null, []);
+
+  // search flows:
+  // 1. prefix search by name
+  // 2. like search by name
+  // 3. keyword equal search
+  var ids = {};
+
+  mysql.query(SEARCH_MODULES_SQL, [word + '%', limit ], ep.done(function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
     }
-    ep.emit('ids', rows.map(function (r) {
-      return r.module_id;
-    }));
+    if (rows.length >= 20) {
+      return ep.emit('ids', Object.keys(ids));
+    }
+
+    mysql.query(SEARCH_MODULES_SQL, [ '%' + word + '%', limit ], ep.done('likeSearch'));
   }));
 
-  ep.on('ids', function (ids) {
-    mysql.query(SEARCH_SQLS[1], [ids], ep.done(function (modules) {
-      callback(null, modules);
+  mysql.query(SEARCH_MODULES_BY_KEYWORD_SQL, [ word, limit ], ep.done('keywordRows'));
+
+  ep.on('likeSearch', function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
+    }
+
+    ep.emit('ids', Object.keys(ids));
+  });
+
+  ep.all('ids', 'keywordRows', function (ids, keywordRows) {
+    keywordRows = keywordRows || [];
+    var data = {
+      keywordMatchs: keywordRows,
+      searchMatchs: []
+    };
+    if (ids.length === 0) {
+      return callback(null, data);
+    }
+
+    mysql.query(QUERY_MODULES_BY_ID_SQL, [ids], ep.done(function (modules) {
+      data.searchMatchs = modules;
+      callback(null, data);
     }));
   });
 };
+
+thunkify(exports);

proxy/module_deps.js

@@ -0,0 +1,44 @@
+/**!
+ * cnpmjs.org - proxy/module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var thunkify = require('thunkify-wrap');
+var mysql = require('../common/mysql');
+
+var LIST_DEPS_SQL = 'SELECT deps FROM module_deps WHERE name=?;';
+
+exports.list = function (name, callback) {
+  mysql.query(LIST_DEPS_SQL, [name], callback);
+};
+
+var INSERT_DEPS_SQL = 'INSERT INTO module_deps(gmt_create, name, deps) \
+  VALUES(now(), ?, ?);';
+
+exports.add = function (name, deps, callback) {
+  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
+    if (err && err.code === 'ER_DUP_ENTRY') {
+      err = null;
+    }
+    callback(err);
+  });
+};
+
+var DELETE_DEPS_SQL = 'DELETE FROM module_deps WHERE name=? AND deps=?;';
+
+exports.remove = function (name, deps, callback) {
+  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
+};
+
+thunkify(exports);

proxy/module_log.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var mysql = require('../common/mysql');
 
 var INSERT_LOG_SQL = 'INSERT INTO module_log(gmt_create, gmt_modified, name, username, log) \
@@ -40,3 +41,5 @@ var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?;';
 exports.get = function (id, callback) {
   mysql.queryOne(SELECT_SQL, [id], callback);
 };
+
+thunkify(exports);

proxy/module_star.js

@@ -0,0 +1,48 @@
+/**!
+ * cnpmjs.org - proxy/module_star.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+
+exports.add = function *add(name, user) {
+  var sql = 'INSERT INTO module_star(name, user) VALUES(?, ?);';
+  try {
+    yield mysql.query(sql, [name, user]);
+  } catch (err) {
+    if (err.code !== 'ER_DUP_ENTRY') {
+      throw err;
+    }
+  }
+};
+
+exports.remove = function *(name, user) {
+  var sql = 'DELETE FROM module_star WHERE name = ? AND user = ?;';
+  return yield mysql.query(sql, [name, user]);
+};
+
+exports.listUsers = function *(name) {
+  var sql = 'SELECT user FROM module_star WHERE name = ?;';
+  var rows = yield mysql.query(sql, [name]);
+  return rows.map(function (r) {
+    return r.user;
+  });
+};
+
+exports.listUserModules = function *(user) {
+  var sql = 'SELECT name FROM module_star WHERE user = ?;';
+  return (yield mysql.query(sql, [user])).map(function (r) {
+    return r.name;
+  });
+};

proxy/npm.js

@@ -14,46 +14,53 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var urllib = require('urllib');
 var config = require('../config');
+thunkify(urllib, ['request']);
 
-function request(url, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = {
-      dataType: 'json',
-      timeout: 10000
-    };
-  }
+function *request (url, options) {
+  options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
   url = config.sourceNpmRegistry + url;
-  urllib.request(url, options, callback);
+  var r;
+  try {
+    r = yield urllib.request(url, options);
+  } catch (err) {
+    var statusCode = err.res ? err.res.statusCode : 200;
+    var data = err.data || '[empty]';
+    if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
+      err.name = 'NPMServerError';
+      err.message = 'Status ' + statusCode + ', ' + data.toString();
+    }
+    throw err;
+  }
+  return r;
 }
 
-exports.get = function (name, callback) {
-  request('/' + name, function (err, data, res) {
-    if (err) {
-      return callback(err);
-    }
-    res = res || {};
-    if (res.statusCode === 404) {
-      data = null;
-    }
-    callback(null, data, res);
+exports.get = function *(name) {
+  var r = yield request('/' + name);
+  var data = r[0];
+  var res = r[1];
+  if (res && res.statusCode === 404) {
+    data = null;
+  }
+  return data;
+};
+
+exports.getAllSince = function *(startkey) {
+  var r = yield request('/-/all/since?stale=update_after&startkey=' + startkey, {
+    dataType: 'json',
+    timeout: 300000
   });
+  return r[0];
 };
 
-exports.getAllSince = function (startkey, callback) {
-  request('/-/all/since?stale=update_after&startkey=' + startkey, {
+exports.getShort = function *() {
+  var r = yield request('/-/short', {
     dataType: 'json',
     timeout: 300000
-  }, callback);
-};
-
-exports.getShort = function (callback) {
-  request('/-/short', {
-    dataType: 'json',
-    timeout: 300000
-  }, callback);
+  });
+  return r[0];
 };

proxy/sync_module_worker.js

@@ -15,21 +15,28 @@
  * Module dependencies.
  */
 
+var co = require('co');
+var gather = require('co-gather');
+var thunkify = require('thunkify-wrap');
 var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
 var util = require('util');
 var fs = require('fs');
 var path = require('path');
 var crypto = require('crypto');
-var eventproxy = require('eventproxy');
 var urllib = require('urllib');
 var utility = require('utility');
+var ms = require('ms');
 var nfs = require('../common/nfs');
 var npm = require('./npm');
 var common = require('../lib/common');
 var Module = require('./module');
+var ModuleDeps = require('./module_deps');
 var Log = require('./module_log');
-var ms = require('ms');
+var config = require('../config');
+var ModuleStar = require('./module_star');
+
+var request = thunkify(urllib.request);
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -40,13 +47,15 @@ function SyncModuleWorker(options) {
   }
 
   this.names = options.name;
-  for (var i = 0; i < this.names.length; i++) {
-    // ensure package name is lower case
-    this.names[i] = this.names[i].toLowerCase();
-  }
+  // for (var i = 0; i < this.names.length; i++) {
+  //   // ensure package name is lower case
+  //   this.names[i] = this.names[i].toLowerCase();
+  // }
 
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
+  this._publish = options.publish; // _publish_on_cnpm
+
   this.syncingNames = {};
   this.nameMap = {};
   this.names.forEach(function (name) {
@@ -82,10 +91,26 @@ SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
 };
 
 SyncModuleWorker.prototype.start = function () {
-  this.log('user: %s, sync %s worker start, %d concurrency.', this.username, this.names[0], this.concurrency);
-  for (var i = 0; i < this.concurrency; i++) {
-    this.next(i);
-  }
+  this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
+    this.username, this.names[0], this.concurrency, this.noDep, this._publish);
+  var self = this;
+  co(function *() {
+    var arr = [];
+    for (var i = 0; i < self.concurrency; i++) {
+      arr.push(self.next(i));
+    }
+    yield arr;
+  })();
+};
+
+SyncModuleWorker.prototype.pushSuccess = function (name) {
+  this.successes.push(name);
+  this.emit('success', name);
+};
+
+SyncModuleWorker.prototype.pushFail = function (name) {
+  this.fails.push(name);
+  this.emit('fail', name);
 };
 
 SyncModuleWorker.prototype.add = function (name) {
@@ -94,10 +119,11 @@ SyncModuleWorker.prototype.add = function (name) {
   }
   this.nameMap[name] = true;
   this.names.push(name);
+  this.emit('add', name);
   this.log('    add dependencies: %s', name);
 };
 
-SyncModuleWorker.prototype.next = function (concurrencyId) {
+SyncModuleWorker.prototype.next = function *(concurrencyId) {
   var name = this.names.shift();
   if (!name) {
     return process.nextTick(this.finish.bind(this));
@@ -105,324 +131,447 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
 
   var that = this;
   that.syncingNames[name] = true;
-  npm.get(name, function (err, pkg, response) {
-    if (!err && !pkg) {
-      err = new Error('Module ' + name + ' not exists, http status ' + response.statusCode);
-      err.name = 'NpmModuleNotExsitsError';
-    }
-    if (err) {
-      that.fails.push(name);
-      that.log('[error] [%s] get package error: %s', name, err.stack);
-      delete that.syncingNames[name];
-      return that.next(concurrencyId);
+  var pkg;
+  // get from npm
+  try {
+    pkg = yield npm.get(name);
+  } catch (err) {
+    // if 404
+    if (err.res && err.res.statusCode === 404) {
+      that.pushSuccess(name);
+    } else {
+      that.pushFail(name);
     }
+    that.log('[error] [%s] get package error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  if (!pkg) {
+    that.log('[error] [%s] get package error: package not exist', name);
+    delete that.syncingNames[name];
+    yield that.next(concurrencyId);
+    return;
+  }
 
-    that.log('[c#%d] [%s] start...', concurrencyId, pkg.name);
-    that._sync(pkg, function (err, versions) {
-      delete that.syncingNames[name];
-      if (err) {
-        that.fails.push(pkg.name);
-        that.log('[error] [%s] sync error: %s', name, err.stack);
-        return that.next(concurrencyId);
-      }
-      that.log('[%s] synced success, %d versions: %s',
-        name, versions.length, versions.join(', '));
-      that.successes.push(name);
-      that.emit('success', name);
-      that.next(concurrencyId);
-    });
-  });
+  that.log('[c#%d] [%s] start...', concurrencyId, name);
+  var versions;
+  try {
+    versions = yield that._sync(name, pkg);
+  } catch (err) {
+    that.pushFail(name);
+    that.log('[error] [%s] sync error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  that.log('[%s] synced success, %d versions: %s',
+    name, versions.length, versions.join(', '));
+  that.pushSuccess(name);
+  delete that.syncingNames[name];
+  yield that.next(concurrencyId);
 };
 
-SyncModuleWorker.prototype._sync = function (pkg, callback) {
-  var username = this.name;
-  var name = pkg.name;
+function *_listStarUsers(modName) {
+  var users = yield ModuleStar.listUsers(modName);
+  var userMap = {};
+  users.forEach(function (user) {
+    userMap[user] = true;
+  });
+  return userMap;
+}
+
+function *_addStar(modName, username) {
+  yield ModuleStar.add(modName, username);
+}
+
+SyncModuleWorker.prototype._sync = function *(name, pkg) {
+  var username = this.username;
   var that = this;
-  var ep = eventproxy.create();
-  ep.fail(callback);
 
   var hasModules = false;
-  Module.listByName(name, ep.done(function (rows) {
-    hasModules = rows.length > 0;
-    var map = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.package || !r.package.dist) {
-        // package json parse error
-        continue;
-      }
+  var result = yield [
+    Module.listByName(name),
+    Module.listTags(name),
+    _listStarUsers(name)
+  ];
+  var moduleRows = result[0];
+  var tagRows = result[1];
+  var existsStarUsers = result[2];
 
-      if (r.package && r.package._publish_on_cnpm) {
-        // publish on cnpm, dont sync this version package
-        that.log('  [%s] publish on local cnpm, don\'t sync.', pkg.name);
-        ep.unbind();
-        callback(null, []);
-        return;
-      }
-
-      if (r.version === 'next') {
-        continue;
-      }
-      if (!map.latest) {
-        map.latest = r;
-      }
-      map[r.version] = r;
+  hasModules = moduleRows.length > 0;
+  var map = {};
+  for (var i = 0; i < moduleRows.length; i++) {
+    var r = moduleRows[i];
+    if (!r.package || !r.package.dist) {
+      // package json parse error
+      continue;
     }
-    ep.emit('existsMap', map);
-  }));
 
-  Module.listTags(name, ep.done(function (rows) {
-    var tags = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.module_id) {
-        // no module_id, need to sync tags
-        continue;
-      }
-      tags[r.tag] = r.version;
+    if (r.package && r.package._publish_on_cnpm) {
+      // publish on cnpm, dont sync this version package
+      that.log('  [%s] publish on local cnpm, don\'t sync', name);
+      return [];
     }
-    ep.emit('existsTags', tags);
-  }));
+
+    if (r.version === 'next') {
+      continue;
+    }
+    if (!map.latest) {
+      map.latest = r;
+    }
+    map[r.version] = r;
+  }
+
+  var tags = {};
+  for (var i = 0; i < tagRows.length; i++) {
+    var r = tagRows[i];
+    if (!r.module_id) {
+      // no module_id, need to sync tags
+      continue;
+    }
+    tags[r.tag] = r.version;
+  }
 
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
-  ep.all('existsMap', 'existsTags', function (map, tags) {
-    var times = pkg.time || {};
-    pkg.versions = pkg.versions || {};
-    var versionNames = Object.keys(times);
-    if (versionNames.length === 0) {
-      versionNames = Object.keys(pkg.versions);
+  var missingReadmes = [];
+  var missingStarUsers = [];
+
+  // get the missing star users
+  var starUsers = pkg.users || {};
+  for (var k in starUsers) {
+    if (!existsStarUsers[k]) {
+      missingStarUsers.push(k);
     }
-    if (versionNames.length === 0) {
-      that.log('  [%s] no times and no versions, hasModules: %s', pkg.name, hasModules);
-      if (!hasModules) {
-        // save a next module
-        var maintainer = pkg.maintainers && pkg.maintainers[0];
-        if (maintainer && maintainer.name) {
-          maintainer = maintainer.name;
-        }
-        if (!maintainer) {
-          maintainer = '-';
-        }
-        var nextMod = {
-          name: pkg.name,
+  }
+  that.log('  [%s] found %d missing star users', name, missingStarUsers.length);
+
+
+  var times = pkg.time || {};
+  pkg.versions = pkg.versions || {};
+  var versionNames = Object.keys(times);
+  if (versionNames.length === 0) {
+    versionNames = Object.keys(pkg.versions);
+  }
+  if (versionNames.length === 0) {
+    that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
+    if (!hasModules) {
+      // save a next module
+      var maintainer = pkg.maintainers && pkg.maintainers[0];
+      if (maintainer && maintainer.name) {
+        maintainer = maintainer.name;
+      }
+      if (!maintainer) {
+        maintainer = '-';
+      }
+      var nextMod = {
+        name: name,
+        version: 'next',
+        author: maintainer,
+        package: {
+          name: name,
           version: 'next',
-          author: maintainer,
-          package: {
-            name: pkg.name,
-            version: 'next',
-            description: pkg.description || '',
-            readme: pkg.readme || '',
-            maintainers: pkg.maintainers || {
-              name: maintainer
-            },
+          description: pkg.description || '',
+          readme: pkg.readme || '',
+          maintainers: pkg.maintainers || {
+            name: maintainer
           },
-        };
-        Module.add(nextMod, function (err, result) {
-          that.log('  [%s] save next module, %j, error: %s', pkg.name, result, err);
-        });
+        },
+      };
+      try {
+        var result = yield Module.add(nextMod);
+        that.log('  [%s] save next module, %j', name, result);
+      } catch (err) {
+        that.log('  [%s] save next module error %s', err.message);
       }
     }
-    var versions = [];
-    for (var i = 0; i < versionNames.length; i++) {
-      var v = versionNames[i];
-      var exists = map[v] || {};
-      var version = pkg.versions[v];
-      if (!version || !version.dist) {
+  }
+
+  var versions = [];
+  for (var i = 0; i < versionNames.length; i++) {
+    var v = versionNames[i];
+    var exists = map[v] || {};
+    var version = pkg.versions[v];
+    if (!version || !version.dist || !version.dist.tarball) {
+      continue;
+    }
+    //patch for readme
+    if (!version.readme) {
+      version.readme = pkg.readme;
+    }
+    var publish_time = times[v];
+    version.publish_time = publish_time ? Date.parse(publish_time) : null;
+    if (!version.maintainers || !version.maintainers[0]) {
+      version.maintainers = pkg.maintainers;
+    }
+    var sourceAuthor = version.maintainers && version.maintainers[0] &&
+      version.maintainers[0].name || exists.author;
+
+    if (exists.package && exists.package.dist.shasum === version.dist.shasum &&
+      exists.author === sourceAuthor) {
+      // * author make sure equal
+      // * shasum make sure equal
+      if ((version.publish_time === exists.publish_time) ||
+          (!version.publish_time && exists.publish_time)) {
+        // debug('  [%s] %s publish_time equal: %s, %s',
+        //   name, version.version, version.publish_time, exists.publish_time);
+        // * publish_time make sure equal
+        if (exists.description === null && version.description) {
+          // * make sure description exists
+          missingDescriptions.push({
+            id: exists.id,
+            description: version.description
+          });
+        }
+
+        if (!exists.package.readme && version.readme) {
+          // * make sure readme exists
+          missingReadmes.push({
+            id: exists.id,
+            readme: version.readme
+          });
+        }
         continue;
       }
-
-      var publish_time = times[v];
-      version.publish_time = publish_time ? Date.parse(publish_time) : null;
-      if (!version.maintainers || !version.maintainers[0]) {
-        version.maintainers = pkg.maintainers;
-      }
-      var sourceAuthor = version.maintainers && version.maintainers[0] && version.maintainers[0].name || exists.author;
-
-      if (exists.package && exists.package.dist.shasum === version.dist.shasum && exists.author === sourceAuthor) {
-        // * author make sure equal
-        // * shasum make sure equal
-        if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
-          debug('  [%s] %s publish_time equal: %s, %s',
-            pkg.name, version.version, version.publish_time, exists.publish_time);
-          // * publish_time make sure equal
-          if (exists.description === null && version.description) {
-            // * make sure description exists
-            missingDescriptions.push({
-              id: exists.id,
-              description: version.description
-            });
-          }
-          continue;
-        }
-      }
-      versions.push(version);
     }
+    versions.push(version);
+  }
 
-    var sourceTags = pkg['dist-tags'] || {};
-    for (var t in sourceTags) {
-      var sourceTagVersion = sourceTags[t];
-      if (sourceTagVersion && tags[t] !== sourceTagVersion) {
-        missingTags.push([t, sourceTagVersion]);
-      }
-    }
-
-    if (versions.length === 0) {
-      that.log('  [%s] all versions are exists', pkg.name);
-      return ep.emit('syncDone');
+  var sourceTags = pkg['dist-tags'] || {};
+  for (var t in sourceTags) {
+    var sourceTagVersion = sourceTags[t];
+    if (sourceTagVersion && tags[t] !== sourceTagVersion) {
+      missingTags.push([t, sourceTagVersion]);
     }
+  }
 
+  if (versions.length === 0) {
+    that.log('  [%s] all versions are exists', name);
+  } else {
     versions.sort(function (a, b) {
       return a.publish_time - b.publish_time;
     });
-    missingVersions = versions;
-    that.log('  [%s] %d versions', pkg.name, versions.length);
-    ep.emit('syncModule', missingVersions.shift());
-  });
+    that.log('  [%s] %d versions need to sync', name, versions.length);
+  }
 
+  missingVersions = versions;
   var versionNames = [];
   var syncIndex = 0;
-  ep.on('syncModule', function (syncModule) {
+
+  // sync missing versions
+  while (missingVersions.length) {
     var index = syncIndex++;
-    that._syncOneVersion(index, syncModule, function (err, result) {
-      if (err) {
-        that.log('    [%s:%d] error, version: %s, %s: %s',
-          syncModule.name, index, syncModule.version, err.name, err.message);
-      } else {
-        versionNames.push(syncModule.version);
-      }
+    var syncModule = missingVersions.shift();
+    if (!syncModule.dist.tarball) {
+      continue;
+    }
+    try {
+      var result = yield that._syncOneVersion(index, syncModule);
+      versionNames.push(syncModule.version);
+    } catch (err) {
+      that.log('    [%s:%d] error, version: %s, %s: %s',
+        syncModule.name, index, syncModule.version, err.name, err.message);
+    }
+  }
 
-      var nextVersion = missingVersions.shift();
-      if (!nextVersion) {
-        return ep.emit('syncDone', result);
-      }
-      ep.emit('syncModule', nextVersion);
-    });
-  });
-
-  ep.on('syncDone', function () {
+  // sync missing descriptions
+  function *syncDes() {
     if (missingDescriptions.length === 0) {
-      return ep.emit('descriptionDone');
+      return;
     }
+    that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
+    var res = yield gather(missingDescriptions.map(function (item) {
+      return Module.updateDescription(item.id, item.description);
+    }));
 
-    that.log('  [%s] saving %d descriptions', pkg.name, missingDescriptions.length);
-    missingDescriptions.forEach(function (item) {
-      Module.updateDescription(item.id, item.description, function (err, result) {
-        if (err) {
-          return that.log('    save error, %s: description %j, error: %s', item.id, item.description, err);
-        }
-        that.log('    saved, id: %s, description length: %d', item.id, item.description.length);
-        ep.emit('saveDescription');
-      });
-    });
+    for (var i = 0; i < res.length; i++) {
+      var item = missingDescriptions[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, description: %s, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    saved, id: %s, description length: %d',
+          item.id, item.description.length);
+      }
+    }
+  }
 
-    ep.after('saveDescription', missingDescriptions.length, function () {
-      ep.emit('descriptionDone');
-    });
-  });
-
-  ep.on('syncDone', function () {
+  // sync missing tags
+  function *syncTag() {
     if (missingTags.length === 0) {
-      return ep.emit('tagDone');
+      return;
+    }
+    that.log('  [%s] adding %d tags', name, missingTags.length);
+    // sync tags
+
+    var res = yield gather(missingTags.map(function (item) {
+      return Module.addTag(name, item[0], item[1]);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingTags[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    add tag %s:%s error, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    added tag %s:%s, module_id: %s',
+          item[0], item[1], r.value && r.value.module_id);
+      }
+    }
+  }
+
+  // sycn missing readme
+  function *syncReadme() {
+    if (missingReadmes.length === 0) {
+      return;
+    }
+    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
+
+    var res = yield gather(missingReadmes.map(function (item) {
+      return Module.updateReadme(item.id, item.readme);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingReadmes[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, error: %s', item.id, r.error.message);
+      } else {
+        that.log('    saved, id: %s', item.id);
+      }
+    }
+  }
+
+  // sync missing star users
+  function *syncUser() {
+    if (missingStarUsers.length === 0) {
+      return;
     }
 
-    that.log('  [%s] adding %d tags', pkg.name, missingTags.length);
-    // sync tags
-    missingTags.forEach(function (item) {
-      Module.addTag(pkg.name, item[0], item[1], ep.done(function (result) {
-        that.log('    added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
-        ep.emit('addTag');
-      }));
-    });
+    that.log('  [%s] saving %d star users', name, missingStarUsers.length);
+    var res = yield gather(missingStarUsers.map(function (username) {
+      return _addStar(name, username);
+    }));
 
-    ep.after('addTag', missingTags.length, function () {
-      ep.emit('tagDone');
-    });
-  });
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    add star user error, %s', r.error.message);
+      }
+    }
+  }
 
-  ep.all('tagDone', 'descriptionDone', function () {
-    // TODO: set latest version
-    callback(null, versionNames);
-  });
+  yield [syncDes(), syncTag(), syncReadme(), syncUser()];
+  return versionNames;
 };
 
-SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePackage, callback) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
   var that = this;
   var username = this.username;
   var downurl = sourcePackage.dist.tarball;
   var filename = path.basename(downurl);
   var filepath = common.getTarballFilepath(filename);
   var ws = fs.createWriteStream(filepath);
+
   var options = {
     writeStream: ws,
     followRedirect: true,
     timeout: 600000, // 10 minutes download
   };
-  var ep = eventproxy.create();
-  ep.fail(function (err) {
-    // remove tmp file whatever
-    fs.unlink(filepath, utility.noop);
-    callback(err);
-  });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j',
-    sourcePackage.name, versionIndex, sourcePackage.version, sourcePackage.dist);
+  var dependencies = Object.keys(sourcePackage.dependencies || {});
+  var devDependencies = Object.keys(sourcePackage.devDependencies || {});
+
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, ' +
+   'publish on cnpm: %s, dependencies: %d, devDependencies: %d',
+    sourcePackage.name, versionIndex, sourcePackage.version,
+    sourcePackage.dist, that.noDep, that._publish,
+    dependencies.length, devDependencies.length);
+
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
+  }
+
+  if (devDependencies.length > config.maxDependencies) {
+    devDependencies = devDependencies.slice(0, config.maxDependencies);
+  }
+
   if (!that.noDep) {
-    for (var k in sourcePackage.dependencies) {
-      that.add(k);
+    for (var i = 0; i < dependencies.length; i++) {
+      that.add(dependencies[i]);
     }
 
-    for (var k in sourcePackage.devDependencies) {
-      that.add(k);
+    for (var i = 0; i < devDependencies.length; i++) {
+      that.add(devDependencies[i]);
     }
   }
 
+  // add module dependence
+  try {
+    yield dependencies.map(function (depName) {
+      return ModuleDeps.add(depName, sourcePackage.name);
+    });
+  } catch (err) {
+    // ignore
+  }
+
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
-  urllib.request(downurl, options, ep.done(function (_, response) {
+
+  try {
+    // get tarball
+    var r = yield request(downurl, options);
+    var response = r[1];
     var statusCode = response && response.statusCode || -1;
     if (statusCode === 404) {
-      // just copy source dist
       shasum = sourcePackage.dist.shasum;
-      return ep.emit('uploadResult', {
+      return yield afterUpload({
         url: downurl
       });
     }
-
     if (statusCode !== 200) {
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
       err.data = sourcePackage;
-      return ep.emit('error', err);
+      throw err;
     }
 
+    // read and check
     var rs = fs.createReadStream(filepath);
-    rs.once('error', function (err) {
-      ep.emit('error', err);
-    });
     rs.on('data', function (data) {
       shasum.update(data);
       dataSize += data.length;
     });
-    rs.on('end', function () {
-      shasum = shasum.digest('hex');
-      if (shasum !== sourcePackage.dist.shasum) {
-        var err = new Error('Download ' + downurl + ' shasum:' + shasum + ' not match ' + sourcePackage.dist.shasum);
-        err.name = 'DownloadTarballShasumError';
-        err.data = sourcePackage;
-        return ep.emit('error', err);
-      }
+    yield thunkify(rs)(); // after end event emit
 
-      var key = common.getCDNKey(sourcePackage.name, filename);
-      nfs.upload(filepath, {key: key, size: dataSize}, ep.done('uploadResult'));
-    });
-  }));
+    // check shasum
+    shasum = shasum.digest('hex');
+    if (shasum !== sourcePackage.dist.shasum) {
+      var err = new Error('Download ' + downurl + ' shasum:' + shasum +
+        ' not match ' + sourcePackage.dist.shasum);
+      err.name = 'DownloadTarballShasumError';
+      err.data = sourcePackage;
+      throw err;
+    }
 
-  ep.on('uploadResult', function (result) {
+    options = {
+      key: common.getCDNKey(sourcePackage.name, filename),
+      size: dataSize,
+      shasum: shasum
+    };
+    // upload to NFS
+    var result = yield nfs.upload(filepath, options);
+    return yield afterUpload(result);
+  } finally {
     // remove tmp file whatever
     fs.unlink(filepath, utility.noop);
+  }
 
+  function *afterUpload(result) {
     //make sure sync module have the correct author info
     //only if can not get maintainers, use the username
     var author = username;
@@ -437,50 +586,61 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
       author: author,
       publish_time: sourcePackage.publish_time,
     };
+
+    if (that._publish) {
+      // sync as publish
+      mod.package._publish_on_cnpm = true;
+    }
+
     var dist = {
-      tarball: result.url,
       shasum: shasum,
       size: dataSize,
       noattachment: dataSize === 0,
     };
+
+    if (result.url) {
+      dist.tarball = result.url;
+    } else if (result.key) {
+      dist.key = result.key;
+      dist.tarball = result.key;
+    }
+
     mod.package.dist = dist;
-    Module.add(mod, ep.done(function (result) {
-      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j',
-        sourcePackage.name, versionIndex,
-        result.id,
-        author, mod.version, dataSize, new Date(mod.publish_time));
-      callback(null, result);
-    }));
-  });
+    var r = yield Module.add(mod);
+
+    that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, ' +
+    'size: %d, publish_time: %j, publish on cnpm: %s',
+    sourcePackage.name, versionIndex,
+    r.id,
+    author, mod.version, dataSize,
+    new Date(mod.publish_time),
+    that._publish);
+
+    return r;
+  }
 };
 
-SyncModuleWorker.sync = function (name, username, callback) {
-  npm.get(name, function (err, pkg, response) {
-    if (err) {
-      return callback(err);
-    }
-    if (!pkg || !pkg._rev) {
-      return callback(null, {
-        ok: false,
-        statusCode: response.statusCode,
-        pkg: pkg
-      });
-    }
-    Log.create({name: name, username: username}, function (err, result) {
-      if (err) {
-        return callback(err);
-      }
-      var worker = new SyncModuleWorker({
-        logId: result.id,
-        name: name,
-        username: username,
-      });
-      worker.start();
-      callback(null, {
-        ok: true,
-        logId: result.id,
-        pkg: pkg
-      });
-    });
+SyncModuleWorker.sync = function *(name, username, options) {
+  options = options || {};
+  var pkg = yield npm.get(name);
+  if (!pkg || !pkg._rev) {
+    return {
+      ok: false,
+      pkg: pkg
+    };
+  }
+  var result = yield Log.create({name: name, username: username});
+  var worker = new SyncModuleWorker({
+    logId: result.id,
+    name: name,
+    username: username,
+    noDep: options.noDep,
+    publish: options.publish,
   });
+  worker.start();
+  return {
+    ok: true,
+    logId: result.id,
+    pkg: pkg
+  };
 };

proxy/total.js

@@ -14,9 +14,10 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
+var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var eventproxy = require('eventproxy');
 
 var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
   FROM information_schema.tables \
@@ -118,3 +119,5 @@ exports.updateSyncNum = function (params, callback) {
   ];
   mysql.query(UPDATE_SYNC_NUM_SQL, query, callback);
 };
+
+thunkify(exports);

proxy/user.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var config = require('../config');
 var mysql = require('../common/mysql');
@@ -29,7 +30,6 @@ function sha1(s) {
 function passwordSha(password, salt) {
   return sha1(password + salt);
 }
-exports.passwordSha = passwordSha;
 
 exports.get = function (name, callback) {
   mysql.queryOne(SELECT_USER_SQL, [name], function (err, row) {
@@ -105,3 +105,5 @@ exports.update = function (user, callback) {
   });
 };
 
+thunkify(exports);
+exports.passwordSha = passwordSha;

routes/registry.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var middlewares = require('koa-middlewares');
 var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
@@ -24,7 +25,7 @@ var user = require('../controllers/registry/user');
 var sync = require('../controllers/sync');
 
 function routes(app) {
-  app.get('/', total.show);
+  app.get('/', middlewares.jsonp(), total.show);
 
   //before /:name/:version
   //get all modules, for npm search
@@ -34,10 +35,10 @@ function routes(app) {
   app.get('/-/short', mod.listAllModuleNames);
 
   // module
-  app.get('/:name', [syncByInstall], mod.show);
-  app.get('/:name/:version', [syncByInstall], mod.get);
+  app.get('/:name', syncByInstall, mod.show);
+  app.get('/:name/:version', syncByInstall, mod.get);
   // try to add module
-  app.put('/:name', [login, publishable], mod.add);
+  app.put('/:name', login, publishable, mod.add);
 
   // sync from source npm
   app.put('/:name/sync', sync.sync);
@@ -47,23 +48,22 @@ function routes(app) {
 
   // put tarball
   // https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
-  app.put('/:name/-/:filename/-rev/:rev', [login, publishable], mod.upload);
+  app.put('/:name/-/:filename/-rev/:rev', login, publishable, mod.upload);
   // delete tarball
-  app.delete('/:name/download/:filename/-rev/:rev', [login, publishable], mod.removeTar);
+  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
 
   // put package.json to module
-  app.put('/:name/:version/-tag/latest', [login, publishable], mod.updateLatest);
+  app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
 
   // update module, unpublish will PUT this
-  app.put('/:name/-rev/:rev', [login, publishable], mod.removeWithVersions);
-  app.delete('/:name/-rev/:rev', [login, publishable], mod.removeAll);
+  app.put('/:name/-rev/:rev', login, publishable, mod.removeWithVersions);
+  app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
 
   // try to create a new user
   // https://registry.npmjs.org/-/user/org.couchdb.user:fengmk2
   app.put('/-/user/org.couchdb.user::name', user.add);
   app.get('/-/user/org.couchdb.user::name', user.show);
   app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, user.update);
-
   // _session
   app.post('/_session', user.authSession);
 }

routes/web.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - routes/web.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -19,6 +19,7 @@ var pkg = require('../controllers/web/package');
 var user = require('../controllers/web/user');
 var sync = require('../controllers/sync');
 var total = require('../controllers/total');
+var dist = require('../controllers/web/dist');
 
 function routes(app) {
   app.get('/total', total.show);
@@ -31,6 +32,11 @@ function routes(app) {
   app.get('/sync/:name', pkg.displaySync);
   app.put('/sync/:name', sync.sync);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
+  app.get('/sync', pkg.displaySync);
+
+  app.get('/_list/search/search', pkg.rangeSearch);
+
+  app.get(/^\/dist(\/.+)?/, dist.redirect);
 }
 
 module.exports = routes;

servers/registry.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - servers/registry.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -15,69 +15,53 @@
  * Module dependencies.
  */
 
-require('response-patch');
+var koa = require('koa');
+var app = module.exports = koa();
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var responseCookie = require('response-cookie');
-var urlrouter = require('urlrouter');
-var forward = require('forward');
-var path = require('path');
+var microtime = require('microtime');
+var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
+var notFound = require('../middleware/registry_not_found');
 
-var rootdir = path.dirname(__dirname);
-var app = connect();
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
+app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
 
-app.use('/favicon.ico', forward(path.join(rootdir, 'public', 'favicon.png')));
-
-app.use('/dist', connect.static(config.uploadDir));
-
-app.use(responseCookie());
-app.use(connect.cookieParser());
-app.use(connect.query());
-app.use(connect.json());
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
 app.use(session);
+app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.gzip());
+app.use(middlewares.fresh());
+app.use(middlewares.etag());
 
 /**
  * Routes
  */
 
-app.use(urlrouter(routes));
-
-app.use(function (req, res, next) {
-  res.json(404, {error: 'not_found', reason: 'document not found'});
-});
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-    logger.error(err);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.json(500, {
-    error: err.name,
-    reason: err.message
-  });
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
+  logger.error(err);
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.registryPort);
+}
 
 module.exports = app;

servers/web.js

@@ -15,81 +15,92 @@
  * Module dependencies.
  */
 
-require('response-patch');
 var path = require('path');
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var urlrouter = require('urlrouter');
-var connectMarkdown = require('connect-markdown');
+var fs = require('fs');
+var microtime = require('microtime');
+var koa = require('koa');
+var middlewares = require('koa-middlewares');
+var markdown = require('koa-markdown');
+var session = require('../common/session');
+var opensearch = require('../middleware/opensearch');
+var notFound = require('../middleware/web_not_found');
 var routes = require('../routes/web');
 var logger = require('../common/logger');
 var config = require('../config');
-var session = require('../common/session');
-var render = require('connect-render');
-var opensearch = require('../middleware/opensearch');
-var app = connect();
+
+var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
-
-app.use('/public', connect.static(path.join(rootdir, 'public')));
-app.use('/opensearch.xml', opensearch);
-
-app.use(connect.cookieParser());
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
+  buffer: !config.debug,
+  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+  dir: path.join(rootdir, 'public')
+}));
+app.use(opensearch);
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
 app.use(session);
-app.use(connect.query());
-app.use(connect.json());
+app.use(middlewares.bodyParser());
+app.use(notFound);
+
+app.use(middlewares.gzip());
+app.use(middlewares.fresh());
+app.use(middlewares.etag());
 
 var viewDir = path.join(rootdir, 'view', 'web');
 var docDir = path.join(rootdir, 'docs', 'web');
 
-app.use('/', connectMarkdown({
+var layoutFile = path.join(viewDir, '_layout.html');
+var footer = config.customFooter || fs.readFileSync(path.join(viewDir, 'footer.html'), 'utf8');
+var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
+  .replace('{{footer}}', footer)
+  .replace('{{logoURL}}', config.logoURL);
+fs.writeFileSync(layoutFile, layout);
+
+app.use(markdown({
+  baseUrl: '/',
   root: docDir,
-  layout: path.join(viewDir, 'layout.html'),
+  layout: layoutFile,
   titleHolder: '<%- locals.title %>',
   bodyHolder: '<%- locals.body %>',
   indexName: 'readme'
 }));
 
-var helpers = {
+var locals = {
   config: config
 };
 
-app.use(render({
+middlewares.render(app, {
   root: viewDir,
-  viewExt: '.html',
-  layout: 'layout',
+  viewExt: 'html',
+  layout: '_layout',
   cache: config.viewCache,
-  helpers: helpers
-}));
+  debug: config.debug,
+  locals: locals
+});
 
 /**
  * Routes
  */
-
-app.use(urlrouter(routes));
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
-  console.error(err.stack);
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (config.debug) {
-    return next(err);
-  }
-  res.statusCode = 500;
-  res.send('Server has some problems. :(');
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.webPort);
+}
 
 module.exports = app;

sync/index.js

@@ -22,8 +22,9 @@ var utility = require('utility');
 var debug = require('debug')('cnpmjs.org:sync:index');
 var Total = require('../proxy/total');
 var logger = require('../common/logger');
+var co = require('co');
 
-var sync;
+var sync = null;
 
 switch (config.syncModel) {
 case 'all':
@@ -34,28 +35,41 @@ case 'exist':
   break;
 }
 
+if (!sync && config.enableCluster) {
+  console.log('[%s] [sync_worker:%s] no need to sync, exit now', Date(), process.pid);
+  process.exit(0);
+}
+
+console.log('[%s] [sync_worker:%s] syncing with %s mode',
+  Date(), process.pid, config.syncModel);
+
 //set sync_status = 0 at first
 Total.updateSyncStatus(0, utility.noop);
 
 // the same time only sync once
 var syncing = false;
 
-function handleSync() {
+var handleSync = co(function *() {
   debug('mode: %s, syncing: %s', config.syncModel, syncing);
-  // check sync every one 30 minutes
   if (!syncing) {
     syncing = true;
     debug('start syncing');
-    sync(function (err, data) {
-      if (config.debug) {
-        console.log(err, data);
-      } else {
-        sendMailToAdmin(err, data, new Date());
-      }
-      syncing = false;
-    });
+    var data;
+    var error;
+    try {
+      var data = yield *sync();
+    } catch (err) {
+      error = err;
+    }
+    if (config.debug) {
+      error && console.error(error.stack);
+      data && console.log(data);
+    } else {
+      sendMailToAdmin(error, data, new Date());
+    }
+    syncing = false;
   }
-}
+});
 
 if (sync) {
   handleSync();

sync/status.js

@@ -19,18 +19,20 @@ var utility = require('utility');
 var Total = require('../proxy/total');
 
 function Status(options) {
-  this.worker = options.worker;
   this.need = options.need;
   this.lastSyncModule = '';
+  this.successes = 0;
+  this.fails = 0;
+  this.left = options.need;
 }
 
 Status.prototype.log = function (syncDone) {
   var params = {
     syncStatus: syncDone ? 0 : 1,
     need: this.need,
-    success: this.worker.successes.length,
-    fail: this.worker.fails.length,
-    left: this.worker.names.length,
+    success: this.successes,
+    fail: this.fails,
+    left: this.left,
     lastSyncModule: this.lastSyncModule,
   };
   Total.updateSyncNum(params, utility.noop);
@@ -43,19 +45,30 @@ Status.prototype.start = function () {
   this.started = true;
   //every 30s log it into mysql
   this.timer = setInterval(this.log.bind(this), 30000);
-  this.worker.on('success', function (moduleName) {
-    debug('sync [%s] success', moduleName);
-    this.lastSyncModule = moduleName;
-  }.bind(this));
-  this.worker.on('end', function () {
-    this.started = false;
-    this.log(true);
-    clearInterval(this.timer);
-  }.bind(this));
 };
 
-Status.init = function (options) {
-  return new Status(options);
+Status.init = function (options, worker) {
+  var status = new Status(options);
+  status.start();
+  worker.on('success', function (moduleName) {
+    debug('sync [%s] success', moduleName);
+    status.lastSyncModule = moduleName;
+    status.successes++;
+    status.left--;
+  });
+  worker.on('fail', function () {
+    status.fails++;
+    status.left--;
+  });
+  worker.on('add', function () {
+    status.left++;
+  });
+
+  worker.on('end', function () {
+    status.started = false;
+    status.log(true);
+    clearInterval(status.timer);
+  });
 };
 
 module.exports = Status;

sync/sync_all.js

@@ -24,6 +24,8 @@ var Npm = require('../proxy/npm');
 var Total = require('../proxy/total');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var Module = require('../proxy/module');
+var co = require('co');
+var thunkify = require('thunkify-wrap');
 
 function subtract(subtracter, minuend) {
   subtracter = subtracter || [];
@@ -31,13 +33,11 @@ function subtract(subtracter, minuend) {
   var map = {};
   var results = [];
   minuend.forEach(function (name) {
-    map[name.toLowerCase()] = true;
+    map[name] = true;
   });
   subtracter.forEach(function (name) {
-    var lowerName = name.toLowerCase();
-    if (!map[lowerName] && !/[A-Z]/.test(name)) {
-      // ensure package name is lower case
-      results.push(lowerName);
+    if (!map[name]) {
+      results.push(name);
     }
   });
   return results;
@@ -57,109 +57,86 @@ function union(arrOne, arrTwo) {
  * when sync from official at the first time
  * get all packages by short and restart from last synced module
  * @param {String} lastSyncModule
- * @param {Function} callback
  */
-function getFirstSyncPackages(lastSyncModule, callback) {
-  Npm.getShort(function (err, pkgs) {
-    if (err || !lastSyncModule) {
-      return callback(err, pkgs);
-    }
-    // start from last success
-    var lastIndex = pkgs.indexOf(lastSyncModule);
-    if (lastIndex > 0) {
-      pkgs = pkgs.slice(lastIndex);
-    }
-    return callback(null, pkgs);
-  });
+function *getFirstSyncPackages(lastSyncModule) {
+  var pkgs = yield Npm.getShort();
+  if (!lastSyncModule) {
+    return pkgs;
+  }
+  // start from last success
+  var lastIndex = pkgs.indexOf(lastSyncModule);
+  if (lastIndex > 0) {
+    return pkgs.slice(lastIndex);
+  }
 }
 
 /**
  * get all the packages that update time > lastSyncTime
  * @param {Number} lastSyncTime
- * @param {Function} callback
  */
-function getCommonSyncPackages(lastSyncTime, callback) {
-  Npm.getAllSince(lastSyncTime, function (err, data) {
-    if (err || !data) {
-      return callback(err, []);
-    }
-    delete data._updated;
-    return callback(null, Object.keys(data));
-  });
+function *getCommonSyncPackages(lastSyncTime) {
+  var data = yield Npm.getAllSince(lastSyncTime);
+  if (!data) {
+    return [];
+  }
+  delete data._updated;
+  return Object.keys(data);
 }
 
 /**
  * get all the missing packages
  * @param {Function} callback
  */
-function getMissPackages(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
-  Npm.getShort(ep.doneLater('allPackages'));
-  Module.listAllModuleNames(ep.doneLater(function (rows) {
-    var existPackages = rows.map(function (row) {
-      return row.name;
-    });
-    ep.emit('existPackages', existPackages);
-  }));
-  ep.all('allPackages', 'existPackages', function (allPackages, existPackages) {
-    callback(null, subtract(allPackages, existPackages));
+function *getMissPackages(callback) {
+  var r = yield [Npm.getShort(), Module.listAllModuleNames];
+  var allPackages = r[0];
+  var existPackages = r[1].map(function (row) {
+    return row.name;
   });
+  return subtract(allPackages, existPackages);
 }
 
-//only sync not exist once
-var syncNotExist = true;
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
+  var info = yield Total.getTotalInfo();
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    debug('Last sync time %s', new Date(info.last_sync_time));
-    // TODO: 记录上次同步的最后一个模块名称
-    if (!info.last_sync_time) {
-      debug('First time sync all packages from official registry');
-      return getFirstSyncPackages(info.last_sync_module, ep.done('syncPackages'));
-    }
-    if (syncNotExist) {
-      getMissPackages(ep.done('missPackages'));
-      syncNotExist = false;
-    } else {
-      ep.emitLater('missPackages', []);
-    }
-    getCommonSyncPackages(info.last_sync_time - ms('10m'), ep.doneLater('newestPackages'));
-    ep.all('missPackages', 'newestPackages', function (missPackages, newestPackages) {
-      ep.emit('syncPackages', union(missPackages, newestPackages));
-    });
-  });
+  var packages;
+  debug('Last sync time %s', new Date(info.last_sync_time));
+  if (!info.last_sync_time) {
+    debug('First time sync all packages from official registry');
+    packages = yield getFirstSyncPackages(info.last_sync_module);
+  } else {
+    packages = yield getCommonSyncPackages(info.last_sync_time - ms('10m'));
+  }
 
-  ep.once('syncPackages', function (packages) {
-    packages = packages || [];
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      noDep: true,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      //only when all succss, set last sync time
-      !worker.fails.length && Total.setLastSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  packages = packages || [];
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    concurrency: config.syncConcurrency,
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  yield thunkify(worker)();
+
+  debug('All packages sync done, successes %d, fails %d',
+      worker.successes.length, worker.fails.length);
+  //only when all succss, set last sync time
+  if (!worker.fails.length) {
+    Total.setLastSyncTime(syncTime, utility.noop);
+  }
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

sync/sync_exist.js

@@ -24,6 +24,7 @@ var debug = require('debug')('cnpmjs.org:sync:sync_hot');
 var utility = require('utility');
 var Status = require('./status');
 var ms = require('ms');
+var thunkify = require('thunkify-wrap');
 
 function intersection(arrOne, arrTwo) {
   arrOne = arrOne || [];
@@ -39,71 +40,66 @@ function intersection(arrOne, arrTwo) {
   return results;
 }
 
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
 
-  Module.listShort(ep.doneLater(function (packages) {
-    packages = packages.map(function (p) {
-      return p.name;
-    });
-    ep.emit('existPackages', packages);
-  }));
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
-
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    if (!info.last_exist_sync_time) {
-      debug('First time sync all packages from official registry');
-      return Npm.getShort(ep.done(function (pkgs) {
-        if (!info.last_sync_module) {
-          return ep.emit('allPackages', pkgs);
-        }
-        // start from last success
-        var lastIndex = pkgs.indexOf(info.last_sync_module);
-        if (lastIndex > 0) {
-          pkgs = pkgs.slice(lastIndex);
-        }
-        ep.emit('allPackages', pkgs);
-      }));
-    }
-    Npm.getAllSince(info.last_exist_sync_time - ms('10m'), ep.done(function (data) {
-      if (!data) {
-        return ep.emit('allPackages', []);
-      }
-      if (data._updated) {
-        syncTime = data._updated;
-        delete data._updated;
-      }
-
-      return ep.emit('allPackages', Object.keys(data));
-    }));
+  var r = yield [Module.listShort(), Total.getTotalInfo()];
+  var existPackages = r[0].map(function (p) {
+    return p.name;
   });
+  var info = r[1];
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.all('existPackages', 'allPackages', function (existPackages, allPackages) {
-    var packages = intersection(existPackages, allPackages);
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      Total.setLastExistSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  var allPackages;
+  if (!info.last_exist_sync_time) {
+    debug('First time sync all packages from official registry');
+    var pkgs = yield Npm.getShort();
+
+    if (info.last_sync_module) {
+      // start from last success
+      var lastIndex = pkgs.indexOf(info.last_sync_module);
+      if (lastIndex > 0) {
+        pkgs = pkgs.slice(lastIndex);
+      }
+    }
+    allPackages = pkgs;
+  } else {
+    debug('sync new module from last exist sync time: %s', info.last_sync_time);
+    var data = yield Npm.getAllSince(info.last_exist_sync_time - ms('10m'));
+    if (!data) {
+      allPackages = [];
+    }
+    if (data._updated) {
+      syncTime = data._updated;
+      delete data._updated;
+    }
+    allPackages = Object.keys(data);
+  }
+
+  var packages = intersection(existPackages, allPackages);
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    concurrency: config.syncConcurrency
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  yield thunkify(worker)();
+
+  debug('All packages sync done, successes %d, fails %d',
+    worker.successes.length, worker.fails.length);
+
+  Total.setLastExistSyncTime(syncTime, utility.noop);
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

test/controllers/registry/module.test.js

@@ -17,6 +17,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var thunkify = require('thunkify-wrap');
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -25,6 +26,7 @@ var app = require('../../../servers/registry');
 var Module = require('../../../proxy/module');
 var Npm = require('../../../proxy/npm');
 var controller = require('../../../controllers/registry/module');
+var ModuleDeps = require('../../../proxy/module_deps');
 
 var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
 
@@ -64,38 +66,75 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('GET /:name', function () {
-    it('should return module info', function (done) {
+  describe('GET /:name get module package info', function () {
+    var etag;
+
+    it('should return module info and etag', function (done) {
       request(app)
       .get('/cnpmjs.org')
       .expect(200, function (err, res) {
         should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
-          'time', 'author', 'repository', '_attachments');
-        // res.body.author.should.eql({
-        //   "name": "fengmk2",
-        //   "email": "fengmk2@gmail.com",
-        //   "url": "http://fengmk2.github.com"
-        // });
+          'time', 'author', 'repository', '_attachments',
+          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
+        res.body.name.should.equal('cnpmjs.org');
+        res.body.versions[Object.keys(res.body.versions)[0]]
+          .dist.tarball.should.include('/cnpmjs.org/download');
+        res.body.time.should.have.property('modified');
+        res.body.time.modified.should.be.a.String;
+        res.body.time.should.have.property('created');
+        res.body.time.created.should.be.a.String;
+        done();
+      });
+    });
+
+    it('should return module info and gzip when accept-encoding=gzip', function (done) {
+      request(app)
+      .get('/cnpmjs.org')
+      .set('accept-encoding', 'gzip')
+      .expect('content-encoding', 'gzip')
+      .expect(200, function (err, res) {
+        // console.log(res.headers)
+        should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
+        res.body.should.have.keys('_id', '_rev', 'name', 'description',
+          'versions', 'dist-tags', 'readme', 'maintainers',
+          'time', 'author', 'repository', '_attachments',
+          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
         res.body.name.should.equal('cnpmjs.org');
         res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
         done();
       });
     });
+
+    it('should 304 when etag match', function (done) {
+      request(app)
+      .get('/cnpmjs.org')
+      .set('If-None-Match', etag)
+      .expect(304, done);
+    });
   });
 
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.0.0')
+      .get('/cnpmjs.org/0.2.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.0.0');
-        body._id.should.equal('cnpmjs.org@0.0.0');
-        body.dist.tarball.should.include('cnpmjs.org-0.0.0.tgz');
+        body.version.should.equal('0.2.1');
+        body._id.should.equal('cnpmjs.org@0.2.1');
+        body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
+        body.should.have.property('_cnpm_publish_time');
+        body._cnpm_publish_time.should.be.a.Number;
+        body.should.have.property('_publish_on_cnpm', true);
         done();
       });
     });
@@ -114,14 +153,14 @@ describe('controllers/registry/module.test.js', function () {
       });
     });
 
-    it('should get cnpmjs.org@0.1.0 with _publish_on_cnpm=true', function (done) {
+    it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.1.0')
+      .get('/cnpmjs.org/0.2.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.1.0');
+        body.version.should.equal('0.2.1');
         body._publish_on_cnpm.should.equal(true);
         done();
       });
@@ -137,6 +176,13 @@ describe('controllers/registry/module.test.js', function () {
         name: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org'
       }],
+      keywords: [
+        'testputmodule', 'test', 'cnpmjstest'
+      ],
+      dependencies: {
+        'foo-testputmodule': "*",
+        'bar-testputmodule': '*'
+      }
     };
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
@@ -172,14 +218,7 @@ describe('controllers/registry/module.test.js', function () {
       .put('/' + pkg.name)
       .set('authorization', baseauth)
       .send(pkg)
-      .expect(409, function (err, res) {
-        should.not.exist(err);
-        res.body.should.eql({
-          error: 'conflict',
-          reason: 'Document update conflict.'
-        });
-        done();
-      });
+      .expect(201, done);
     });
 
     it('should try to add return 403 when not module user and only next module exists',
@@ -205,7 +244,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags',
-          'readme', 'maintainers', 'time', '_attachments');
+          'readme', 'maintainers', 'time', '_attachments', 'users');
         res.body.versions.should.eql({});
         res.body.time.should.eql({});
         res.body['dist-tags'].should.eql({});
@@ -291,6 +330,7 @@ describe('controllers/registry/module.test.js', function () {
       var pkg = require(path.join(fixtures, 'testputmodule.json')).versions['0.1.8'];
       pkg.name = 'testputmodule';
       pkg.version = '0.1.9';
+      pkg.dependencies['foo-testputmodule'] = '*';
       request(app)
       .put('/' + pkg.name + '/' + pkg.version + '/-tag/latest')
       .set('authorization', baseauth)
@@ -298,7 +338,16 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('ok', 'rev');
-        done();
+        // should get deps foo-testputmodule contains 'testputmodule'
+        ModuleDeps.list('foo-testputmodule', function (err, rows) {
+          should.not.exist(err);
+          var exists = rows.filter(function (r) {
+            return r.deps === 'testputmodule';
+          });
+          exists.should.length(1);
+          exists[0].deps.should.equal('testputmodule');
+          done();
+        });
       });
     });
 
@@ -346,6 +395,60 @@ describe('controllers/registry/module.test.js', function () {
         done();
       });
     });
+
+    it('should publish with tgz base64, addPackageAndDist()', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      // delete first
+      request(app)
+      .del('/' + pkg.name + '/-rev/1')
+      .set('authorization', baseauth)
+      .expect({ok: true})
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+
+        request(app)
+        .put('/' + pkg.name)
+        .set('authorization', baseauth)
+        .send(pkg)
+        .expect(201, function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'rev');
+          res.body.ok.should.equal(true);
+
+          // upload again should 409
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', baseauth)
+          .send(pkg)
+          .expect(409, function (err, res) {
+            should.not.exist(err);
+            res.body.should.eql({
+              error: 'conflict',
+              reason: 'Document update conflict.'
+            });
+            done();
+          });
+
+        });
+      });
+    });
+
+    it('should version_error when versions missing', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      delete pkg.versions;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({
+          error: 'version_error',
+          reason: 'filename or version not found, filename: mk2testmodule-0.0.1.tgz, version: undefined'
+        });
+        done();
+      });
+    });
   });
 
   describe('GET /-/all', function () {
@@ -442,6 +545,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should remove version ok', function (done) {
       //do not really remove it here
       mm.empty(Module, 'removeByNameAndVersions');
+      mm.empty(Module, 'removeTagsByIds');
       request(app)
       .put('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
@@ -453,6 +557,15 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
+  describe('GET /:name/download/:filename', function () {
+    it('should download a file with 302 redirect', function (done) {
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
+      .expect(302, done);
+    });
+  });
+
   describe('DELETE /:name/download/:filename/-rev/:rev', function () {
     var lastRev;
     before(function (done) {

test/controllers/registry/user.test.js

@@ -16,18 +16,22 @@
 
 var should = require('should');
 var request = require('supertest');
+var mm = require('mm');
 var app = require('../../../servers/registry');
 var user = require('../../../proxy/user');
-var mm = require('mm');
+var mysql = require('../../../common/mysql');
 
 describe('controllers/registry/user.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
+
   afterEach(mm.restore);
+
   describe('GET /-/user/org.couchdb.user:name', function () {
     it('should return user info', function (done) {
       request(app)
@@ -47,7 +51,7 @@ describe('controllers/registry/user.test.js', function () {
     });
 
     it('should return 500 when mysql error', function (done) {
-      mm.error(user, 'get', 'mock error');
+      mm.error(mysql, 'query', 'mock mysql error');
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
       .expect(500, done);
@@ -91,7 +95,7 @@ describe('controllers/registry/user.test.js', function () {
         password_sha: 'password_sha',
         email: 'email'
       })
-      .expect(500, done);      
+      .expect(500, done);
     });
 
     it('should 201 when user.add ok', function (done) {

test/controllers/sync.test.js

@@ -40,6 +40,54 @@ describe('controllers/sync.test.js', function () {
   describe('sync source npm package', function () {
     var logIdRegistry;
     var logIdWeb;
+
+    it('should sync as publish success', function (done) {
+      request(registryApp)
+      .del('/utility/-rev/123')
+      .set('authorization', baseauth)
+      // .expect(200)
+      // .expect({ok: true})
+      .end(function (err, res) {
+        should.not.exist(err);
+
+        mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+        request(registryApp)
+        .put('/utility/sync?publish=true&nodeps=true')
+        .set('authorization', baseauth)
+        .end(function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'logId');
+          logIdRegistry = res.body.logId;
+          done();
+          // setTimeout(function () {
+          //   request(registryApp)
+          //   .get('/utility')
+          //   .expect(200)
+          //   .end(function (err, res) {
+          //     should.not.exist(err);
+          //     Object.keys(res.body.versions).length.should.above(0);
+          //     for (var v in res.body.versions) {
+          //       var pkg = res.body.versions[v];
+          //       pkg.should.have.property('_publish_on_cnpm', true);
+          //     }
+          //     done();
+          //   });
+          // }, 5000);
+        });
+      });
+    });
+
+    it('should sync as publish 403 when user not admin', function (done) {
+      mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+      request(registryApp)
+      .put('/utility_unit_test/sync?publish=true&nodeps=true')
+      .expect(403)
+      .expect({
+        error: 'no_perms',
+        reason: 'Only admin can publish'
+      }, done);
+    });
+
     it('should sync success', function (done) {
       mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
       done = pedding(2, done);

test/controllers/total.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/total.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,9 +17,9 @@
 
 var should = require('should');
 var request = require('supertest');
+var pedding = require('pedding');
 var registryApp = require('../../servers/registry');
 var webApp = require('../../servers/web');
-var pedding = require('pedding');
 
 describe('controllers/total.test.js', function () {
   before(function (done) {
@@ -44,6 +44,7 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
+
     it('should return total info by jsonp', function (done) {
       request(registryApp)
       .get('?callback=totalCallback')
@@ -51,7 +52,8 @@ describe('controllers/total.test.js', function () {
       .expect(/totalCallback\({.*}\)/, done);
     });
   });
-  describe('GET /total in web', function () {
+
+  describe.skip('GET /total in web', function () {
     it('should return total info', function (done) {
       request(webApp)
       .get('/total')
@@ -62,5 +64,5 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
-  });  
+  });
 });

test/controllers/web/dist.test.js

@@ -0,0 +1,37 @@
+/**!
+ * cnpmjs.org - test/controllers/web/dist.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../../servers/web');
+
+describe('controllers/web/dist.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('GET /dist', function (done) {
+    it('should 302 to config.disturl', function (done) {
+      request(app)
+      .get('/dist')
+      .expect('Location', 'http://dist.u.qiniudn.com/')
+      .expect(302, done);
+    });
+  });
+});

test/controllers/web/package.test.js

@@ -31,14 +31,61 @@ describe('controllers/web/package.test.js', function () {
 
   afterEach(mm.restore);
 
+  describe('GET /_list/search/search', function () {
+    it('should search with "c"', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="c"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search with c', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey=c&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="cddddsdasdaasds"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({rows: []});
+        done();
+      });
+    });
+  });
+
   describe('GET /package/:name', function (done) {
     it('should get 200', function (done) {
       request(app)
       .get('/package/cutter')
       .expect(200)
+      .expect('content-encoding', 'gzip')
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
-      .expect(/<th>Version<\/th>/, done);
+      .expect(/<th>Version<\/th>/, function (err, res) {
+        should.not.exist(err);
+        res.should.have.header('etag');
+        done();
+      });
     });
 
     it('should get 404', function (done) {
@@ -87,6 +134,13 @@ describe('controllers/web/package.test.js', function () {
       .expect(/Packages match/, done);
     });
 
+    it('should list by keyword with json ok', function (done) {
+      request(app)
+      .get('/browse/keyword/cnpm?type=json')
+      .expect(200)
+      .expect('content-type', 'application/json; charset=utf-8', done);
+    });
+
     it('should list no match ok', function (done) {
       request(app)
       .get('/browse/keyword/notexistpackage')
@@ -99,7 +153,7 @@ describe('controllers/web/package.test.js', function () {
       request(app)
       .get('/browse/keyword/notexistpackage')
       .expect(500)
-      .expect(/MockError: mm mock error/, done);
+      .expect(/Internal Server Error/, done);
     });
   });
 
@@ -113,7 +167,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://opensource.org/licenses/MIT'
       });
 
-      var p = {license: ['http://foo/MIT']};
+      p = {license: ['http://foo/MIT']};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({
@@ -121,7 +175,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://foo/MIT'
       });
 
-      var p = {license: {name: 'mit', url: 'http://foo/mit'}};
+      p = {license: {name: 'mit', url: 'http://foo/mit'}};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({

test/fixtures/500.txt

@@ -0,0 +1,7 @@
+Internal routing error
+
+Sorry, we cannot connect to the intended server.
+
+We have just been notified of this problem. We will correct it as soon as possible.
+
+Feel free to contact us if you have any questions: support@iriscouch.com

test/fixtures/package_and_tgz.json

@@ -0,0 +1 @@
+{"_id":"mk2testmodule","name":"mk2testmodule","description":"","dist-tags":{"latest":"0.0.1"},"versions":{"0.0.1":{"name":"mk2testmodule","version":"0.0.1","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"author":"","license":"ISC","readme":"ERROR: No README data found!","_id":"mk2testmodule@0.0.1","dist":{"shasum":"fa475605f88bab9b1127833633ca3ae0a477224c","tarball":"http://127.0.0.1:7001/mk2testmodule/-/mk2testmodule-0.0.1.tgz"},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}]}},"readme":"ERROR: No README data found!","maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_attachments":{"mk2testmodule-0.0.1.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2SsWrDMBCGPfspDg2ZinOyEgeylg6Zu2YR8rVRHEtGkkOg5N0jWaFdujVQAv6W4/7/dHcSGqTq5Ccthxyro7emeDCI2KxWkOKmaaaIdc4TouZQ8FqgwI3AdVMgF8ijho9e5DdGH6SLq/y1T74LfMcn4asEYEb2xLbA+q4O5ENv2/FE7CVZZ3JeW5NcrLDiWW3JK6eHcHey2Es9Zdq0dIkfKau50EcjjYpCmpDKSB0s7Nmbc9ZtwVhIBviBlP7Q1O4ZLBZAFx2As3jyOnWTYzhY9zPzpBUZPy2/e39l5bX87wedmZmZeRJuheTX2wAIAAA=","length":251}}}

test/lib/common.test.js

@@ -0,0 +1,29 @@
+/*!
+ * cnpmjs.org - test/lib/common.test.js.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com>
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var common = require('../../lib/common');
+
+describe('lib/common.test.js', function () {
+  describe('isAdmin()', function () {
+    it('should admin is admin', function () {
+      common.isAdmin('admin').should.equal(true);
+      common.isAdmin('fengmk2').should.equal(true);
+      common.isAdmin('constructor').should.equal(false);
+      common.isAdmin('toString').should.equal(false);
+    });
+  });
+});

test/middleware/auth.test.js

@@ -29,8 +29,9 @@ describe('middleware/auth.test.js', function () {
     app.close(done);
   });
 
+  afterEach(mm.restore);
+
   describe('auth()', function () {
-    afterEach(mm.restore);
     it('should pass if no authorization', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
@@ -44,11 +45,12 @@ describe('middleware/auth.test.js', function () {
       .expect(200, done);
     });
 
-    it('should 401 with authorization and check fail', function (done) {
+    it('should pass with authorization and check fail', function (done) {
+      // npm install no need to check auth
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest10')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest').toString('base64'))
-      .expect(401, done);
+      .expect(200, done);
     });
 
     it('should 500 with authorization and mysql error', function (done) {

test/middleware/opensearch.test.js

@@ -30,8 +30,8 @@ describe('middleware/opensearch.test.js', function () {
     it('should get 200', function (done) {
       request(app)
       .get('/opensearch.xml')
-      .set('host', 'localhost:7002')
-      .expect(/http:\/\/localhost:7002/, done);
+      .set('host', 'localhost')
+      .expect(/http:\/\/localhost/, done);
     });
   });
 });

test/middleware/web_not_found.test.js

@@ -0,0 +1,63 @@
+/**!
+ * cnpmjs.org - test/middleware/web_not_found.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../servers/web');
+
+describe('middleware/web_not_found.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('web_not_found()', function () {
+    it('should redirect /byte to /package/byte', function (done) {
+      request(app)
+      .get('/byte')
+      .expect('Location', '/package/byte')
+      .expect(302, done);
+    });
+
+    it('should redirect /byte/ to /package/byte', function (done) {
+      request(app)
+      .get('/byte/')
+      .expect('Location', '/package/byte')
+      .expect(302, done);
+    });
+
+    it('should 404 /~byte', function (done) {
+      request(app)
+      .get('/~byte')
+      .expect(404, done);
+    });
+
+    it('should 200 /package/byte', function (done) {
+      request(app)
+      .get('/package/byte')
+      .expect(200, done);
+    });
+
+    it('should 404 /package/byte404', function (done) {
+      request(app)
+      .get('/package/byte404')
+      .expect(404, done);
+    });
+  });
+});

test/proxy/module.test.js

@@ -23,6 +23,7 @@ var Module = require('../../proxy/module');
 
 var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 
+var id;
 describe('proxy/module.test.js', function () {
   afterEach(mm.restore);
 
@@ -36,6 +37,79 @@ describe('proxy/module.test.js', function () {
     });
   });
 
+  describe('search()', function () {
+    it('should search modules', function (done) {
+      Module.search('as', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.searchMatchs.length.should.above(0);
+        data.searchMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search match keywords modules', function (done) {
+      Module.search('aa', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.keywordMatchs.length.should.above(0);
+        data.keywordMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      Module.search('emptyemptyemptyempty', function (err, data) {
+        should.not.exist(err);
+        data.should.eql({
+          keywordMatchs: [],
+          searchMatchs: []
+        });
+        done();
+      });
+    });
+  });
+
+  describe('addKeywords()', function () {
+    var mockName = 'aa' + Date.now();
+
+    after(function (done) {
+      mysql.query('DELETE FROM module_keyword WHERE name=?', [mockName], done);
+    });
+
+    it('should add diff keywords to module', function (done) {
+      Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        done();
+      });
+    });
+
+    it('should add same keywords to module', function (done) {
+      Module.addKeywords('aa', 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(0);
+        done();
+      });
+    });
+  });
+
+  describe('getKeywords()', function () {
+    it('should get aa module keywords', function (done) {
+      Module.getKeywords('aa', function (err, keywords) {
+        should.not.exist(err);
+        keywords.should.eql(['aa', 'bb', 'cc']);
+        done();
+      });
+    });
+  });
+
   describe('add()', function () {
     it('should success ad he@0.3.6', function (done) {
       var sourcePackage = require('../fixtures/0.3.6.json');
@@ -53,6 +127,7 @@ describe('proxy/module.test.js', function () {
       };
       mod.package.dist = dist;
       Module.add(mod, function (err, result) {
+        id = result.id;
         should.not.exist(err);
         Module.getById(result.id, function (err, row) {
           should.not.exist(err);
@@ -74,4 +149,17 @@ describe('proxy/module.test.js', function () {
       });
     });
   });
+
+  describe('updateReadme()', function () {
+    it('should update ok', function (done) {
+      Module.updateReadme(id, 'test', function (err, data) {
+        should.not.exist(err);
+        Module.getById(id, function (err, data) {
+          should.not.exist(err);
+          data.package.readme.should.equal('test');
+          done();
+        });
+      });
+    });
+  });
 });

test/proxy/module_deps.test.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - test/proxy/module_deps.test.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var pedding = require('pedding');
+var ModuleDeps = require('../../proxy/module_deps');
+
+describe('proxy/module_deps.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    ModuleDeps.remove('testmodule', 'foo', done);
+    ModuleDeps.remove('testmodule', 'bar', done);
+  });
+
+  describe('add()', function () {
+    it('should add foo, bar success', function (done) {
+      done = pedding(2, done);
+      ModuleDeps.add('testmodule', 'foo', function (err) {
+        should.not.exist(err);
+        // add again should work
+        ModuleDeps.add('testmodule', 'foo', function (err) {
+          should.not.exist(err);
+          done();
+        });
+      });
+
+      ModuleDeps.add('testmodule', 'bar', done);
+    });
+  });
+
+  describe('list()', function () {
+    it('should list testmodule deps', function (done) {
+      ModuleDeps.list('testmodule', function (err, rows) {
+        should.not.exist(err);
+        should.exist(rows);
+        rows.should.be.an.Array;
+        rows.should.length(2);
+        rows.forEach(function (row) {
+          row.should.have.property('deps');
+        });
+        done();
+      });
+    });
+  });
+});

test/proxy/module_star.test.js

@@ -0,0 +1,64 @@
+/**!
+ * cnpmjs.org - test/proxy/module_star.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var co = require('co');
+var Star = require('../../proxy/module_star');
+
+describe('proxy/module_star.test.js', function () {
+  before(function (done) {
+    co(function *() {
+      yield Star.remove('testmodule', 'fengmk2');
+      yield Star.remove('testmodule', 'mk1');
+      yield Star.remove('testmodule', 'mk2');
+      done();
+    })();
+  });
+
+  it('should add a star', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      // again should be ok
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule', 'fengmk2');
+      done();
+    })();
+  });
+
+  it('should get all star users', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule', 'mk1');
+      yield Star.add('testmodule', 'mk2');
+
+      var rows = yield Star.listUsers('testmodule');
+      rows.should.containDeep(['fengmk2', 'mk1', 'mk2']);
+      done();
+    })();
+  });
+
+  it('should get user all star modules', function (done) {
+    co(function *() {
+      yield Star.add('testmodule', 'fengmk2');
+      yield Star.add('testmodule1', 'fengmk2');
+      yield Star.add('testmodule2', 'fengmk2');
+
+      var rows = yield Star.listUserModules('fengmk2');
+      rows.should.containDeep(['testmodule', 'testmodule1', 'testmodule2']);
+      done();
+    })();
+  });
+});

test/proxy/npm.test.js

@@ -16,35 +16,56 @@
 
 var should = require('should');
 var mm = require('mm');
+var fs = require('fs');
+var path = require('path');
 var npm = require('../../proxy/npm');
+var co = require('co');
+
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 
 describe('proxy/npm.test.js', function () {
   afterEach(mm.restore);
 
   it('should return a module info from source npm', function (done) {
-    npm.get('pedding', function (err, data) {
-      should.not.exist(err);
-      should.exist(data);
+    co(function *() {
+      var data = yield npm.get('pedding');
       data.name.should.equal('pedding');
       done();
-    });
+    })();
   });
 
   it('should return null when module not exist', function (done) {
-    npm.get('pedding-not-exists', function (err, data) {
-      should.not.exist(err);
+    co(function *() {
+      var data = yield npm.get('pedding-not-exists');
       should.not.exist(data);
       done();
-    });
+    })();
   });
 
   it('should return error when http error', function (done) {
     mm.http.request(/\//, '{');
-    npm.get('pedding-not-exists', function (err, data) {
-      should.exist(err);
-      err.name.should.equal('JSONResponseFormatError');
-      should.not.exist(data);
-      done();
-    });
+    co(function *() {
+      try {
+        var data = yield npm.get('pedding-not-exists');
+      } catch (err) {
+        err.name.should.equal('JSONResponseFormatError');
+        done();
+      }
+    })();
+  });
+
+  it('should return ServerError when http 500 response', function (done) {
+    var content = fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8');
+    mm.http.request(/\//, content, { statusCode: 500 });
+    // http://registry.npmjs.org/octopie
+    co(function *() {
+      try {
+        var data = yield npm.get('octopie');
+      } catch (err) {
+        err.name.should.equal('NPMServerError');
+        err.message.should.equal('Status 500, ' + content);
+        done();
+      }
+    })();
   });
 });

test/sync.js

@@ -18,8 +18,8 @@ var SyncModuleWorker = require('../proxy/sync_module_worker');
 var mysql = require('../common/mysql');
 var Log = require('../proxy/module_log');
 
-var name = process.argv[2] || 'address,pedding';
-var names = name.split(',');
+var names = process.argv[2] || 'byte';
+names = names.split(',');
 
 Log.create({
   name: names[0],
@@ -30,6 +30,8 @@ Log.create({
     name: names,
     username: 'fengmk2',
     concurrency: names.length,
+    // noDep: true,
+    // publish: true,
   });
 
   worker.start();

test/sync/sync_all.test.js

@@ -19,6 +19,7 @@ var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
 var should = require('should');
 var Module = require('../../proxy/module');
+var co = require('co');
 
 describe('sync/sync_all.js', function () {
   describe('sync()', function () {
@@ -27,16 +28,14 @@ describe('sync/sync_all.js', function () {
     it('should sync first time ok', function (done) {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync;
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         mm.restore();
-        Total.getTotalInfo(function (err, result) {
-          should.not.exist(err);
-          result.last_sync_module.should.equal('cutter');
-          done();
-        });
-      });
+        var result = yield Total.getTotalInfo();
+        result.last_sync_module.should.equal('cutter');
+        done();
+      })();
     });
 
     it('should sync common ok', function (done) {
@@ -48,11 +47,12 @@ describe('sync/sync_all.js', function () {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
       mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpm', 'cnpmjs.org', 'cutter']);
+      co(function *() {
+        var data = yield sync;
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        mm.restore();
         done();
-      });
+      })();
     });
   });
 });

test/sync/sync_exist.test.js

@@ -18,6 +18,7 @@ var mm = require('mm');
 var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
 var should = require('should');
+var co = require('co');
 
 describe('sync/sync_exist.js', function () {
   describe('sync()', function () {
@@ -26,11 +27,11 @@ describe('sync/sync_exist.js', function () {
     it('should sync first time ok', function (done) {
       mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync();
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });
+      })();
     });
 
     it('should sync common ok', function (done) {
@@ -40,11 +41,11 @@ describe('sync/sync_exist.js', function () {
         cutter: {}
       });
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
-      sync(function (err, data) {
-        should.not.exist(err);
+      co(function *() {
+        var data = yield sync();
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });      
+      })();
     });
   });
 });

view/web/footer.html

@@ -0,0 +1,5 @@
+Copyright 2013 - 2014 © cnpmjs.org
+|
+<a href="/">Home</a>
+|
+<script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>

view/web/layout.html

@@ -29,6 +29,10 @@
       table.downloads td.count {
         text-align: right;
       }
+
+      .nav-tabs{margin:20px 0;}
+      .nav-cont{display:none;}
+      .nav-cont.active{display:block;}
     </style>
     <script>
       $(function () {
@@ -41,26 +45,32 @@
             return location.href = '/browse/keyword/' + val;
           }
         });
+
+        $(".nav-tabs li").each(function (index) {
+          $(this).data("index", index);
+        })
+        .on("click", function (e) {
+          e.preventDefault();
+          $(".nav-tabs li.active,.nav-cont.active").removeClass("active");
+          $(this).addClass("active");
+          $(".nav-cont").eq($(this).data("index")).addClass("active");
+        });
       });
     </script>
   </head>
   <body>
     <div class="container">
       <div id="content-header">
-        <a href="/"><img src="http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg"></a>
+        <a href="/"><img src="{{logoURL}}"></a>
         <input type="text" id="search-input" class="form-control" placeholder="Search Packages">
       </div>
       <div id="content-body"><%- locals.body %></div>
     <div class="bottom">
       <hr/>
       <p>
-        Copyright 2013 - 2014 &copy; cnpmjs.org
-        |
-        <a href="/">Home</a>
-        |
-        <script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>
+        {{footer}}
       </p>
-      <a href="https://github.com/fengmk2/cnpmjs.org" id="fork" target="_blank">
+      <a href="https://github.com/cnpm/cnpmjs.org" id="fork" target="_blank">
         <img alt="Fork me on GitHub" src="http://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
     </a>
     </div>

view/web/package.html

@@ -1,7 +1,13 @@
   <div id="package">
     <h1>
       <%= package.name %>
-      <small>(<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)</small>
+      <small>
+      <% if (package._publish_on_cnpm) { %>
+      (Private package)
+      <% } else { %>
+      (<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)
+      <% } %>
+      </small>
     </h1>
 
     <% if (package.deprecated) { %>
@@ -12,175 +18,231 @@
       <p class="description"><%= package.description %></p>
     <% } %>
 
-    <pre class="sh"><code>$ cnpm install <%= package.name %></code></pre>
-
-    <table class="downloads">
-      <tbody>
-        <tr>
-          <td class="count"><%- download.today %></td><td> downloads today</td>
-          <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
-          <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
-        </tr>
-        <tr>
-          <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
-          <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
-          <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
-        </tr>
-      </tbody>
-    </table>
-
-    <table class="metadata">
-      <% if (package.maintainers) { %>
-      <tr>
-        <th>Maintainers</th>
-        <td>
-        <% package.maintainers.forEach(function (m) { %>
-          <div class="user">
-            <a class="username" href="/~<%= m.name %>">
-              <% if (m.gravatar) { %>
-              <img src="<%- m.gravatar %>" class="avatar">
-              <% } %>
-              <%= m.name %>
-            </a>
-          </div>
-        <% }) %>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <th>Version</th>
-        <td>
-          <b>
-          <%= package.version %>
-          </b>
-          <% if (package.fromNow) { %>
-          last updated
-          <%= package.fromNow %>
-          <% } %>
-        </td>
-      </tr>
-      <% if (package.license) { %>
-      <tr>
-        <th>License</th>
-        <td>
-          <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
-        </td>
-      </tr>
-      <% } %>
-      <%
-      if (typeof package.keywords === 'string') {
-        package.keywords = package.keywords.split(/\s*,?\s*/)
-      }
-      if (Array.isArray(package.keywords)) {
-      %>
-      <tr>
-        <th>Keywords</th>
-        <td><%-
-          package.keywords.map(function (kw) {
-            kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
-            return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
-          }).join(', ')
-          %></td>
-      </tr>
-      <% } %>
-      <% if (package.repository && package.repository !== 'undefined') {
-      var gh = package.repository.url &&
-          package.repository.url.match(
-          /^(?:https?:\/\/|git(?::\/\/|@))(gist.github.com|github.com)[:\/](.*?)(?:.git)?$/)
-      if (gh) {
-        gh = 'https://' + gh[1] + '/' + gh[2]
-      }
-      %>
-      <tr>
-        <th>Repository</td>
-        <td>
-          <% if (gh) { %><a href="<%= gh %>" target="_blank"><% } %>
-          <%= package.repository.url %><% if (gh) { %></a><% } %>
-          (<%= package.repository.type %>)
-        </td>
-      </tr>
-      <% } %>
-      <% if (typeof package.homepage === 'string') { %>
-      <tr>
-        <th>Homepage</td>
-        <td>
-          <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
-        </td>
-      </tr>
-      <% } %>
-      <% if (package.bugs && package.bugs.url) { %>
-      <tr>
-        <th>Bugs</td>
-        <td>
-          <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <%
-          var deps = Object.keys(package.dependencies || {})
-          var l = deps.length
-        %>
-        <th>Dependencies<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          if (l === 0) {
-            %>None<%
-          } else {
-            var m = 200
-            if (l > m) deps = deps.slice(0, m)
-            deps.forEach(function(pkg, i) {
-              if (i > 0) { %>, <% }
-              %>
-              <a href="/package/<%= pkg %>"><%= pkg %></a><%
-            })
-            if (l > m) {
-              %>, and <%= l-m %> more<%
-            }
-          }
-          %>
-        </td>
-      </tr>
-
-      <% if (package.users) {
-        var starredBy = package.starredBy
-        var l = starredBy.length
-      %>
-      <tr>
-        <th>Starred by<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          var max = 20
-          if (l > max) {
-            starredBy = starredBy.sort(function (a, b) {
-              return Math.random() * 2 - 1
-            }).slice(0, max)
-          }
-          starredBy.forEach(function (user, i) {
-            if (i > 0) { %>, <% }
-            %><a href="/~<%= user %>"><%= user %></a><%
-          })
-          if (l > max) {
-            %><br><a href="/browse/star/<%= package.name %>">and
-            <%= (l-max) %> more</a><%
-          }
-          %>
-        </td>
-      </tr>
-      <% } %>
-    </table>
-
-    <div class="details">
-      <ul class="toc">
-        <% if (package.readme) { %>
-        <li><a href="#readme">Read Me</a></li>
-        <% } %>
-      </ul>
+    <pre class="sh"><code>$ <%- config.npmClientName %> install <%= package.name %> <% if (package.preferGlobal) { %>-g<% } %></code></pre>
 
+    <ul class="nav nav-tabs">
       <% if (package.readme) { %>
-      <section id="readme">
-      <%- package.readme %>
-      </section>
+        <li class="active"><a href="#readme">Readme.md</a></li>
+        <li><a href="#meta">package.json</a></li>
+      <% } else { %>
+        <li class="active"><a href="#meta">package.json</a></li>
       <% } %>
+    </ul>
+
+    <% if (package.readme) { %>
+    <div class="nav-cont active">
+      <div class="details">
+        <section id="readme">
+        <%- package.readme %>
+        </section>
+      </div>
+    </div>
+    <% } %>
+
+    <div class="nav-cont">
+      <table class="downloads">
+        <tbody>
+          <tr>
+            <td class="count"><%- download.today %></td><td> downloads today</td>
+            <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
+            <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
+          </tr>
+          <tr>
+            <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
+            <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
+            <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
+          </tr>
+        </tbody>
+      </table>
+
+      <table class="metadata">
+        <% if (Array.isArray(package.maintainers) && package.maintainers.length > 0) { %>
+        <tr>
+          <th>Maintainers</th>
+          <td>
+          <% package.maintainers.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" href="/~<%= m.name %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <th>Version</th>
+          <td>
+            <b>
+            <%= package.version %>
+            </b>
+            <% if (package.fromNow) { %>
+            last updated
+            <%= package.fromNow %>
+            <% } %>
+          </td>
+        </tr>
+        <% if (package.license) { %>
+        <tr>
+          <th>License</th>
+          <td>
+            <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
+          </td>
+        </tr>
+        <% } %>
+        <%
+        if (typeof package.keywords === 'string') {
+          package.keywords = package.keywords.split(/\s*,?\s*/)
+        }
+        if (Array.isArray(package.keywords)) {
+        %>
+        <tr>
+          <th>Keywords</th>
+          <td><%-
+            package.keywords.map(function (kw) {
+              kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
+              return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
+            }).join(', ')
+            %></td>
+        </tr>
+        <% } %>
+        <% if (package.repository && package.repository !== 'undefined') { %>
+        <tr>
+          <th>Repository</td>
+          <td>
+            <a href="<%= package.repository.weburl || package.repository.url %>" target="_blank">
+              <%= package.repository.url %>
+            </a>
+            (<%= package.repository.type %>)
+          </td>
+        </tr>
+        <% } %>
+        <% if (typeof package.homepage === 'string') { %>
+        <tr>
+          <th>Homepage</td>
+          <td>
+            <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
+          </td>
+        </tr>
+        <% } %>
+        <% if (package.bugs && package.bugs.url) { %>
+        <tr>
+          <th>Bugs</td>
+          <td>
+            <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <%
+            var deps = Object.keys(package.dependencies || {})
+            var l = deps.length
+          %>
+          <th>Dependencies<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <tr>
+          <%
+            var deps = package.dependents || [];
+            var l = deps.length
+          %>
+          <th>Dependents<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <% if (package.users.length > 0) {
+          var users = package.users
+          var l = users.length
+        %>
+        <tr>
+          <th>Starred by (<%= l %>)</th>
+          <td>
+            <%
+            var max = 20
+            if (l > max) {
+              users = users.sort(function (a, b) {
+                return Math.random() * 2 - 1
+              }).slice(0, max)
+            }
+            users.forEach(function (user, i) {
+              if (i > 0) { %>, <% }
+              %><a href="/~<%= user %>"><%= user %></a><%
+            })
+            if (l > max) {
+              %><br><a href="/browse/star/<%= package.name %>">and
+              <%= (l - max) %> more</a><%
+            }
+            %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (Array.isArray(package.contributors) && package.contributors.length > 0) { %>
+        <tr>
+          <th>Contributors</th>
+          <td>
+          <% package.contributors.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" target="_blank" href="<%= m.url %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (package.dist && package.dist.tarball) { %>
+        <tr>
+          <th>Download</th>
+          <td>
+            <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
+              <%= package.dist.tarball %>
+            </a>
+          </td>
+        </tr>
+        <% } %>
+      </table>
     </div>
   </div>

view/web/search.html

@@ -16,26 +16,43 @@
 <div id="search">
   <% if (!packages.length) { %>
     <div class="alert alert-warning">
-      Can not found package match <%= keyword %>. You can 
+      Can not found package match <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
     </div>
   <% } else if (packages[0].name !== keyword) { %>
     <div class="alert alert-info">
-      Can not found package <%= keyword %>. You can 
+      Can not found package <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
-    </div>    
+    </div>
   <% } %>
   <% if (packages.length) { %>
     <h1>
-      Packages match <span style="color: #09f;"><%= keyword %></span>
+      Packages match "<span style="color: #09f;"><%= keyword %></span>"
     </h1>
     <hr />
-    <% for (var i = 0; i < packages.length; i++) { %>
-      <div class="package <%= packages[i].name === keyword ? 'match' : '' %>">
-        <a href="/package/<%= packages[i].name %>" class="package-name"><%= packages[i].name %></a>
-        <span class="package-description"><%= packages[i].description %></span>
+    <% for (var i = 0; i < packages.length; i++) {
+      var item = packages[i];
+    %>
+      <div class="package <%= item.name === keyword ? 'match' : '' %>">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
+      </div>
+    <% } %>
+  <% } %>
+
+  <% if (keywords.length) { %>
+    <h1>
+      Keywords match "<span style="color: #09f;"><%= keyword %></span>"
+    </h1>
+    <hr />
+    <% for (var i = 0; i < keywords.length; i++) {
+      var item = keywords[i];
+    %>
+      <div class="package <%= item.name === keyword ? 'match' : '' %>">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
       </div>
     <% } %>
   <% } %>

view/web/sync.html

@@ -11,6 +11,7 @@
   var $notify = $('#sync-notify');
   var timer;
   var name = '<%= name %>';
+  var resourceURL = '/sync/' + name;
   $(function() {
     var checkLogId = location.hash.match(/logid=(\d+)/);
     var logid = checkLogId ? checkLogId[1] : '';
@@ -18,7 +19,7 @@
       return getSyncLog(logid);
     }
     $.ajax({
-      url: location.pathname,
+      url: resourceURL,
       type: 'PUT',
       dataType: 'json',
       success: handleSyncSucess,
@@ -53,7 +54,7 @@
   var hasFail = false;
   function getSyncLog(id) {
     $.ajax({
-      url: location.pathname + '/log/' + id + '?offset=' + offset,
+      url: resourceURL + '/log/' + id + '?offset=' + offset,
       type: 'GET',
       dataType: 'json',
       success: function (data) {

worker.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,7 +16,7 @@
  */
 
 var graceful = require('graceful');
-
+var logger = require('./common/logger');
 var config = require('./config');
 var registry = require('./servers/registry');
 var web = require('./servers/web');
@@ -34,6 +34,7 @@ graceful({
     if (err.message) {
       err.message += ' (uncaughtException throw ' + throwErrorCount + ' times on pid:' + process.pid + ')';
     }
-    console.error(err);
+    console.error(err.stack);
+    logger.error(err);
   }
 });