Release 0.4.0

Local mysql dev env

.gitignore

@@ -7,6 +7,7 @@ coverage.html
 *.pid
 *.gz
 dump.rdb
+.DS_Store
 
 pids
 logs
@@ -20,3 +21,6 @@ backup/*.json
 backup/*.gz
 docs/web/history.md
 view/web/_layout.html
+bin/mysql.js
+bin/test.sql
+coverage/

.jshintignore

@@ -0,0 +1,4 @@
+node_modules/
+coverage/
+.tmp/
+.git/

.jshintrc

@@ -0,0 +1,95 @@
+{
+    // JSHint Default Configuration File (as on JSHint website)
+    // See http://jshint.com/docs/ for more details
+
+    "maxerr"        : 50,       // {int} Maximum error before stopping
+
+    // Enforcing
+    "bitwise"       : true,     // true: Prohibit bitwise operators (&, |, ^, etc.)
+    "camelcase"     : false,    // true: Identifiers must be in camelCase
+    "curly"         : true,     // true: Require {} for every new block or scope
+    "eqeqeq"        : true,     // true: Require triple equals (===) for comparison
+    "forin"         : false,    // true: Require filtering for..in loops with obj.hasOwnProperty()
+    "immed"         : false,    // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
+    "indent"        : false,    // {int} Number of spaces to use for indentation
+    "latedef"       : false,    // true: Require variables/functions to be defined before being used
+    "newcap"        : false,    // true: Require capitalization of all constructor functions e.g. `new F()`
+    "noarg"         : true,     // true: Prohibit use of `arguments.caller` and `arguments.callee`
+    "noempty"       : true,     // true: Prohibit use of empty blocks
+    "nonew"         : false,    // true: Prohibit use of constructors for side-effects (without assignment)
+    "plusplus"      : false,    // true: Prohibit use of `++` & `--`
+    "quotmark"      : false,    // Quotation mark consistency:
+                                //   false    : do nothing (default)
+                                //   true     : ensure whatever is used is consistent
+                                //   "single" : require single quotes
+                                //   "double" : require double quotes
+    "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
+    "unused"        : false,    // true: Require all defined variables be used
+    "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
+    "trailing"      : false,    // true: Prohibit trailing whitespaces
+    "maxparams"     : false,    // {int} Max number of formal params allowed per function
+    "maxdepth"      : false,    // {int} Max depth of nested blocks (within functions)
+    "maxstatements" : false,    // {int} Max number statements per function
+    "maxcomplexity" : false,    // {int} Max cyclomatic complexity per function
+    "maxlen"        : false,    // {int} Max number of characters per line
+
+    // Relaxing
+    "asi"           : false,     // true: Tolerate Automatic Semicolon Insertion (no semicolons)
+    "boss"          : true,      // true: Tolerate assignments where comparisons would be expected
+    "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
+    "eqnull"        : false,     // true: Tolerate use of `== null`
+    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
+    "esnext"        : true,      // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
+                                 // (ex: `for each`, multiple try/catch, function expression…)
+    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`
+    "expr"          : true,      // true: Tolerate `ExpressionStatement` as Programs
+    "funcscope"     : false,     // true: Tolerate defining variables inside control statements"
+    "globalstrict"  : false,     // true: Allow global "use strict" (also enables 'strict')
+    "iterator"      : false,     // true: Tolerate using the `__iterator__` property
+    "lastsemic"     : false,     // true: Tolerate omitting a semicolon for the last statement of a 1-line block
+    "laxbreak"      : true,      // true: Tolerate possibly unsafe line breakings
+    "laxcomma"      : false,     // true: Tolerate comma-first style coding
+    "loopfunc"      : false,     // true: Tolerate functions being defined in loops
+    "multistr"      : true,      // true: Tolerate multi-line strings
+    "proto"         : false,     // true: Tolerate using the `__proto__` property
+    "scripturl"     : false,     // true: Tolerate script-targeted URLs
+    "smarttabs"     : false,     // true: Tolerate mixed tabs/spaces when used for alignment
+    "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
+    "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
+    "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
+    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+
+    // Environments
+    "browser"       : true,     // Web Browser (window, document, etc)
+    "couch"         : false,    // CouchDB
+    "devel"         : true,     // Development/debugging (alert, confirm, etc)
+    "dojo"          : false,    // Dojo Toolkit
+    "jquery"        : false,    // jQuery
+    "mootools"      : false,    // MooTools
+    "node"          : true,     // Node.js
+    "nonstandard"   : false,    // Widely adopted globals (escape, unescape, etc)
+    "prototypejs"   : false,    // Prototype and Scriptaculous
+    "rhino"         : false,    // Rhino
+    "worker"        : false,    // Web Workers
+    "wsh"           : false,    // Windows Scripting Host
+    "yui"           : false,    // Yahoo User Interface
+    "noyield"       : true,     // allow generators without a yield
+
+    // Legacy
+    "nomen"         : false,    // true: Prohibit dangling `_` in variables
+    "onevar"        : false,    // true: Allow only one `var` statement per function
+    "passfail"      : false,    // true: Stop on first error
+    "white"         : false,    // true: Check against strict whitespace and indentation rules
+
+    // Custom Globals
+    "globals"       : {         // additional predefined global variables
+      // mocha
+      "describe": true,
+      "it": true,
+      "before": true,
+      "afterEach": true,
+      "beforeEach": true,
+      "after": true
+    }
+}

.npmignore

@@ -9,3 +9,9 @@ public/dist/
 backup/*.json
 backup/*.gz
 view/web/_layout.html
+bin/mysql.js
+bin/test.sql
+coverage/
+.jshintrc
+.jshintignore
+.DS_Store

.travis.yml

@@ -1,5 +1,3 @@
 language: node_js
 node_js:
-  - '0.10'
-install: make install
-script: make test-coveralls
+  - '0.11'

AUTHORS

@@ -1,7 +1,8 @@
 # Ordered by date of first contribution.
-# Auto-generated by 'contributors' on Fri, 10 Jan 2014 02:56:32 GMT.
+# Auto-generated by 'contributors' on Mon, 03 Mar 2014 13:01:28 GMT.
 # https://github.com/xingrz/node-contributors
 
 fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)
-dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
-AlsoTang <alsotang@gmail.com> (https://github.com/alsotang)
+dead-horse <dead_horse@qq.com> (https://github.com/dead-horse)
+alsotang <alsotang@gmail.com> (https://github.com/alsotang)
+4simple <wondger@qq.com> (https://github.com/4simple)

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# How to contribute
+
+Third-party patches are essential for keeping `cnpmjs.org` great.
+We want to keep it as easy as possible to contribute changes that
+get things working in your environment. There are a few guidelines that we
+need contributors to follow so that we can have a chance of keeping on
+top of things.
+
+## Getting Started
+
+* Make sure you have a [GitHub account](https://github.com/signup/free)
+* Fork the repository on GitHub
+
+## Making Changes
+
+* Create a topic branch from where you want to base your work.
+  * This is usually the master branch.
+  * Only target release branches if you are certain your fix must be on that
+    branch.
+  * To quickly create a topic branch based on master.
+    Please avoid working directly on the `master` branch.
+* Make commits of logical units and including unit tests.
+* Check for unnecessary whitespace with `git diff --check` before committing.
+* Make sure your commit messages are in the proper format.
+* Make sure you have added the necessary tests for your changes.
+* Run _all_ the tests to assure nothing else was accidentally broken.
+* Follow [node style guide](https://github.com/felixge/node-style-guide)
+
+## Submitting Changes
+
+* Push your changes to a topic branch in your fork of the repository.
+* Submit a pull request.
+* Make sure travis-ci test pass.
+
+# Additional Resources
+
+* [General GitHub documentation](http://help.github.com/)
+* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
+* [cnpmjs.org](http://cnpmjs.org)

History.md

@@ -1,4 +1,214 @@
 
+0.4.0 / 2014-04-09
+==================
+
+ * fix test cases to run on local machine
+ * add contribute guidelines
+ * use local mysql for dev env. fix #308
+ * use copy to
+ * use koa-compress and koa-conditional-get
+ * maintainers is string, fix #301
+
+0.3.13 / 2014-03-27
+==================
+
+ * fix npm adduser update 409 bug
+ * fix multiline coverage
+ * show package engines. fixed #280
+ * dont sync local package field. fix #295
+
+0.3.12 / 2014-03-26
+==================
+
+ * fix result.successes not exist error
+ * fix search list
+ * add simple request for listall
+ * only return package name in /-/all and /-/all/since, fixed #291
+ * refine docs foloder
+ * use module gmt_modified as etag. fix #288
+ * fix typo, remove unused config in package.json
+ * web page only list cnpm registry related info
+ * use generator in qnfs
+
+0.3.11 / 2014-03-20 
+==================
+
+  * use common.isMaintainer, fixed #283
+  * update dependencies
+  * use co-mocha for test, fixed #279
+  * update thunkify-wrap, breaking change in thunkify-wrap
+  * refactor SQLs by using multiline
+  * use multiline to refactor sqls
+  * ignore contributors
+
+0.3.10 / 2014-03-16 
+==================
+
+  * Only /_session request send the authSession. fixed #223
+  * sync npm user info when maintainers and contributors not exists. fixed #82
+  * save npm user to mysql
+  * password salt always be randoms
+  * remove session access in /name and /name/version, fixed #274
+  * fix update maintainer session error
+  * update koa-middlewares
+  * fix test, fix sync_by_install
+  * use defer session
+  * Support npm owner|author add [name] [pkg]. fixed #271
+
+0.3.9 / 2014-03-14 
+==================
+
+  * custom user-agent
+  * use co-urllib instead of thunkify urllib; fix mock http.request test cases
+  * request limit custom message
+  * add config.redis check
+  * add koa-limit, fixed #267
+
+0.3.8 / 2014-03-11 
+==================
+
+  * update middlewares, fixed missing charset bug #264
+
+0.3.7 / 2014-03-11 
+==================
+
+  * show worker die date time
+  * update to koa@0.5.1
+  * hotfix for star user
+  * fix yield gather, sync missing deps even no missing versions
+  * fix return versions
+  * fix makefile, remove eventproxy
+  * refactor sync_module_worker
+  * add make test-dev, fixed #259
+  * change npm.js to generator
+  * update urllib, proxy/npm.js use generator
+  * sync_all and sync_exist to generator
+  * change function to generator
+  * need node >= v0.11.9
+
+0.3.6 / 2014-03-06 
+==================
+
+  * install missing package should sync it from source npm. fixed #252
+  * npm publish dont contains .jshint*
+  * npm test run jshint
+  * Add jshint check: $ make jshint
+  * use `yield* next` instead of `yield next`
+  * replace dist.u.qiniudn.com with cnpmjs.org/dist
+
+0.3.5 / 2014-03-05 
+==================
+
+  * redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
+  * redirect /name to /package/name when /name is 404. fixed #245
+  * Add missing properies and sync missing star users. fixed #235
+
+0.3.4 / 2014-03-04 
+==================
+
+  * add cov
+  * use istanbul run test coverage
+  * gzip support. fix #241
+  * readme spelling patch (@stanzheng)
+  * default readme to null, fixed #233
+  * remove readme in versions
+
+0.3.3 / 2014-02-28 
+==================
+
+  * Merge pull request #232 from cnpm/host-hotfix
+  * get request host from request.headers
+  * Merge pull request #231 from cnpm/bug-fix
+  * fix deps display bug#230 and nsf.url TypeError#229
+
+0.3.2 / 2014-02-28 
+==================
+
+  * update koa-sess and koa-redis
+  * fix sync all test
+  * remove nfs.downloadStream first, fix tmppath error
+  * fix fengmk2/giturl#1 bug
+
+0.3.1 / 2014-02-27 
+==================
+
+  * add etag fixed #224
+  * travis ci install on source npm
+
+0.3.0 / 2014-02-27 
+==================
+
+  * fix typo and dont sync not exists pkgs
+  * use koa-middlewares
+  * fix signed cookie not work on npm@1.3.25; node --harmony-generators
+  * fix opensearch test case
+  * update koa bodyparser
+  * logger.error(err) should send err stack email notice
+  * json body parse limit and bug fix.
+  * fix sync 404 reason not clear
+  * all controllers to koa
+  * controller/web/user.js to koa
+  * change web connect to koa
+  * use outputError
+  * use yield exports.addPackageAndDist.call(this, next);
+  * add end() when ws write end
+  * fix yield coWrite
+  * fix all the test of registry module.test.js
+  * convert registry/module.js to koa type
+  * fix auth middleware
+  * finish registry user controller koa and update mm to support thunkify. fixed #196
+  * change controllers/user.js to koa
+  * thunkify all proxy
+  * convert all middlewares to koa type
+  * change regsitry sync to koa
+  * addd koa-jsonp, koa-bodyparser, fix / controller
+  * first koa run registry home page /
+  * Merge pull request #212 from cnpm/fix-sync-404
+  * return friendly 404 reason
+  * Merge pull request #211 from cnpm/bug-fix
+  * override json limit to default 10mb. fixed #209
+  * fix #210 addPackageAndDist package version detect bug
+
+0.2.27 / 2014-02-19 
+==================
+
+  * support json result in search, fixed #189
+
+0.2.26 / 2014-02-19 
+==================
+
+  * npm publish also need to add deps
+
+0.2.25 / 2014-02-19 
+==================
+
+  * max handle number of package.json `dependencies` property
+  * Dependents support. fixed #190
+
+0.2.24 / 2014-02-13 
+==================
+
+  * fix if delete all the versions
+  * refactor remove module, fixed #186
+
+0.2.23 / 2014-01-26 
+==================
+
+  * system admin can add, publish, remove the packages. fixed #176
+
+0.2.22 / 2014-01-26 
+==================
+
+  * add keyword and search support keyword. #181
+
+0.2.21 / 2014-01-24 
+==================
+
+  * refactor code styles on package.html
+  * nav-tabs e.preventDefault
+  * Show registry server error response. fixed #178
+  * nav-tabs for package.html (@4simple)
+
 0.2.20 / 2014-01-23 
 ==================
 

Makefile

@@ -2,37 +2,51 @@ TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
 REPORTER = tap
 TIMEOUT = 30000
 MOCHA_OPTS =
+REGISTRY = --registry=http://r.cnpmjs.org
 
 install:
-	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm --disturl=http://dist.u.qiniudn.com
+	@npm install $(REGISTRY) \
+		--disturl=http://dist.cnpmjs.org
 
-test: install
-	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
+jshint: install
+	@-./node_modules/.bin/jshint ./
+
+pretest:
+	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
+	@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
+	@mysql -uroot 'cnpmjs_test' -e 'show tables;'
+
+test: install pretest
+	@NODE_ENV=test ./node_modules/.bin/mocha \
+		--harmony-generators \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
+		--require co-mocha \
+		--require ./test/init.js \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
-test-cov:
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
-
-test-cov-html:
-	@rm -f coverage.html
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
-	@ls -lh coverage.html
-
-test-coveralls: test
-	@echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
-	@-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
-
-test-all: test test-cov
+test-cov cov: install
+	@NODE_ENV=test node --harmony \
+		node_modules/.bin/istanbul cover --preserve-comments \
+		./node_modules/.bin/_mocha \
+		-- -u exports \
+		--reporter $(REPORTER) \
+		--timeout $(TIMEOUT) \
+		--require should \
+		--require co-mocha \
+		--require ./test/init.js \
+		$(MOCHA_OPTS) \
+		$(TESTS)
+	@./node_modules/.bin/cov coverage
 
 contributors: install
-	@./node_modules/contributors/bin/contributors -f plain -o AUTHORS
+	@./node_modules/.bin/contributors -f plain -o AUTHORS
 
 autod: install
-	@./node_modules/.bin/autod -w -e public,view,docs,backup
+	@./node_modules/.bin/autod -w -e public,view,docs,backup,coverage
 	@$(MAKE) install
 
 .PHONY: test

README.md

@@ -1,7 +1,7 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org)[![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
@@ -9,39 +9,103 @@ cnpmjs.org
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
+Private npm registry and web for Enterprise, base on [koa](http://koajs.com/),
+MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+Our goal is to provide a low cost maintenance and easy to use solution for private npm.
+
+## What can you do with `cnpmjs.org`
+
+* Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
+* Build a mirror NPM. (we use it to build a mirror in China: [cnpmjs.org](http://cnpmjs.org/))
+* Build a completely independent NPM registry to store whatever you like.
+
+### Features
+
+* **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
+You can get the source code through `npm` or `git`.
+* **Low cost and easy maintenance**: `package.json` info store in MySQL, tarball(tgz file) store in CDN or other store systems.
+* **Automatic synchronization**: automatic synchronization from any registry specified, support two sync modes:
+  - Sync all modules from a specified registry, like [npm registry](http://registry.npmjs.org).
+  - Only sync the modules that exists in your own registry.
+* **Manual synchronization**: automatic synchronization may has little delay, but you can syn immediately by manually.
+* **Customized client**: we provide a client [cnpm](https://github.com/cnpm/cnpm)
+to extend `npm` with more features(`sync` command, [gzip](https://github.com/npm/npm-registry-client/pull/40) support).
+And it easy to wrap for your own registry which build with `cnpmjs.org`.
+* **Compatible with NPM client**: you can use the origin NPM client with `cnpmjs.org`,
+only need to change the registry in config. Even include manual synchronization (through `install` command).
+
+## Getting Start
+
+* @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
+* install and deploy cnpmjs.org through npm: [examples](https://github.com/cnpm/custom-cnpm-example)
+* Mirror NPM in China: [cnpmjs.org](http://cnpmjs.org)
+* cnpm client: [cnpm](https://github.com/cnpm/cnpm), `npm install -g cnpm`
+* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
+* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
+## Develop on your local machine
 
-## Install
+### Dependencies
+
+* [node](http://nodejs.org) >=0.11.9
+* [mysql](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
+
+### Start MySQL
 
 ```bash
-$ npm install
+$ nohup mysqld &
 ```
 
-## Usage
+### Clone codes and run test
 
-```js
-$ node dispatch.js
+```bash
+# clone from git
+$ git clone https://github.com/cnpm/cnpmjs.org.git
+
+# install dependencies
+$ make install
+
+# test
+$ make test
+
+# coverage
+$ make test-cov
+
+# udpate dependencies
+$ make autod
+
+# start server
+$ node --harmony dispatch.js
 ```
 
+## How to contribute
+
+* Clone the project
+* Checkout a new branch
+* Add new features or fix bugs in the new branch
+* Make a pull request and we will review it ASAP
+
+Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
+
 ## Authors
 
 ```bash
 $ git summary
 
  project  : cnpmjs.org
- repo age : 5 weeks
- active   : 98 days
- commits  : 239
- files    : 85
+ repo age : 4 months ago
+ commits  : 472
+ active   : 167 days
+ files    : 104
  authors  :
-   140  fengmk2                 58.6%
-    98  dead_horse              41.0%
-     1  Alsotang                0.4%
+   272  fengmk2                 57.6%
+   195  dead_horse              41.3%
+     2  4simple                 0.4%
+     2  Stanley Zheng           0.4%
+     1  Alsotang                0.2%
 ```
 
 ## License

bin/nodejsctl

@@ -7,7 +7,7 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS=node
+NODEJS='node --harmony-generators'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
 STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\

bin/restore_module_deps.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - bin/restore_module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var Module = require('../proxy/module');
+var ModuleDeps = require('../proxy/module_deps');
+
+var addCount = 0;
+
+function restore(id, callback) {
+  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
+  mysql.query(sql, [id], function (err, rows) {
+    if (err) {
+      return callback(err);
+    }
+    if (rows.length === 0) {
+      return callback(null, []);
+    }
+
+    console.log('[%s] got %d rows', id, rows.length);
+
+    rows.forEach(function (r) {
+      Module.parseRow(r);
+      if (!r.package) {
+        return;
+      }
+      var deps = Object.keys(r.package.dependencies || {});
+      if (!Array.isArray(deps) || !deps.length) {
+        return;
+      }
+      deps.forEach(function (dep) {
+        ModuleDeps.add(dep, r.name, function (err) {
+          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
+        });
+      });
+      addCount += deps.length;
+    });
+    setTimeout(function () {
+      console.log('[%s] add %d relations', id, addCount);
+      callback(null, rows);
+    }, 1000);
+  });
+}
+
+var id = 0;
+function run() {
+  restore(id, function (err, rows) {
+    if (err) {
+      throw err;
+    }
+    if (rows.length === 0) {
+      console.log('finished, last id: %s, exit.', id);
+      process.exit(0);
+    }
+
+    id = rows[rows.length - 1].id;
+    run();
+  });
+}
+
+run();

common/logger.js

@@ -1,7 +1,12 @@
-/*!
+/**!
  * cnpmjs.org - common/logger.js
- * Copyright(c) 2013 
- * Author: dead_horse <undefined>
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
  */
 
 'use strict';
@@ -10,11 +15,12 @@
  * Module dependencies.
  */
 
-var config = require('../config');
 var util = require('util');
 var moment = require('moment');
 var logstream = require('logfilestream');
 var ms = require('ms');
+var config = require('../config');
+var mail = require('./mail');
 
 var isTEST = process.env.NODE_ENV === 'test';
 var ONE_DAY = ms('1d');
@@ -29,7 +35,9 @@ levels.forEach(function (catetory) {
   var stream = logstream(options);
   function write(msg) {
     var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
+    var subject = null;
     if (msg instanceof Error) {
+      subject = msg.name;
       var err = {
         name: msg.name,
         code: msg.code,
@@ -57,12 +65,21 @@ levels.forEach(function (catetory) {
     } else {
       msg = time + ' ' + util.format.apply(util, arguments) + '\n';
     }
+
     if (!isTEST) {
+      stream.write(msg);
       if (config.debug) {
         var level = catetory;
         console.log('[' + level + '] ' + msg);
       } else {
-        stream.write(msg);
+        if (catetory === 'error' && subject) {
+          // send error email
+          var to = [];
+          for (var name in config.admins) {
+            to.push(config.admins[name]);
+          }
+          mail.error(to, subject, msg);
+        }
       }
     }
   }

common/mail.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - common/mail.js
  *
  * Copyright(c) cnpmjs.org and other contributors.

common/mysql.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var ready = require('ready');
 var mysql = require('mysql');
 var config = require('../config');
@@ -35,7 +36,13 @@ var pool = mysql.createPool({
 exports.pool = pool;
 
 exports.query = function (sql, values, cb) {
-  pool.query(sql, values, cb);
+  if (typeof values === 'function') {
+    cb = values;
+    values = null;
+  }
+  pool.query(sql, values, function (err, rows) {
+    cb(err, rows);
+  });
 };
 
 exports.queryOne = function (sql, values, cb) {
@@ -57,6 +64,8 @@ exports.escape = function (val) {
 
 ready(exports);
 
+thunkify(exports);
+
 function init() {
   exports.query('show tables', function (err, rows) {
     if (err) {

common/qnfs.js

@@ -14,11 +14,13 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var qn = require('qn');
 var config = require('../config');
-
 var client = qn.create(config.qn);
 
+thunkify(client, ['delete', 'uploadFile', 'upload']);
+
 exports._client = client;
 
 /**
@@ -28,36 +30,45 @@ exports._client = client;
  * @param {Object} options
  *  - {String} key
  *  - {Number} size
- * @param {Function(err, result)} callback
- *  - {Object} result
- *   - {String} url
  */
-exports.upload = function (filepath, options, callback) {
-  client.delete(options.key, function (err, data) {
-    client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
-      if (err) {
-        return callback(err);
-      }
-      callback(null, {url: data.url});
-    });
+exports.upload = function *(filepath, options) {
+  try {
+    yield client.delete(options.key);
+  } catch (err) {
+    // ignore error here
+  }
+
+  var res = yield client.uploadFile(filepath, {
+    key: options.key,
+    size: options.size
   });
+  var url = res && res[0] ? res[0].url : '';
+  return { url: url };
 };
 
-exports.uploadBuffer = function (buf, options, callback) {
-  client.delete(options.key, function (err, data) {
-    client.upload(buf, {key: options.key}, function (err, data) {
-      if (err) {
-        return callback(err);
-      }
-      callback(null, {url: data.url});
-    });
-  });
+exports.uploadBuffer = function *(buf, options) {
+  try {
+    yield client.delete(options.key);
+  } catch (err) {
+    // ignore error here
+  }
+
+  var res = yield client.upload(buf, {key: options.key});
+  var url = res && res[0] ? res[0].url : '';
+  return { url: url };
 };
 
 exports.url = function (key) {
   return client.resourceURL(key);
 };
 
-exports.remove = function (key, callback) {
-  client.delete(key, callback);
+exports.remove = function *(key) {
+  try {
+    return yield client.delete(key);
+  } catch (err) {
+    if (err.name === 'QiniuFileNotExistsError') {
+      return;
+    }
+    throw err;
+  }
 };

common/redis.js

@@ -0,0 +1,38 @@
+/**!
+ * cnpmjs.org - common/redis.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+
+// close redis by set config.redis to `null` or `{}`
+if (!config.redis || !config.redis.host || !config.redis.port) {
+
+  var redis = require('redis');
+  var wrapper = require('co-redis');
+  var logger = require('./logger');
+
+  var _client = redis.createClient(config.redis);
+
+  _client.on('error', function (err) {
+    logger.error(err);
+  });
+
+  module.exports = wrapper(_client);
+
+} else {
+  console.warn('[%s] [worker:%s:common/redis.js] Redis config can not found',
+    Date(), process.pid);
+  module.exports = null;
+}

common/session.js

@@ -15,27 +15,20 @@
  * Module dependencies.
  */
 
-var connect = require('connect');
-var RedisStore = require('connect-redis')(connect);
+var middlewares = require('koa-middlewares');
 var config = require('../config');
 
-var session;
 var key = 'AuthSession';
-var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 30 };
+var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 365, signed: false };
+var options = {
+  key: key,
+  cookie: cookie,
+  defer: true,
+  rolling: false
+};
 
-if (config.debug) {
-  session = connect.cookieSession({
-    secret: config.sessionSecret,
-    key: key,
-    cookie: cookie
-  });
-} else {
-  session = connect.session({
-    key: key,
-    secret: config.sessionSecret,
-    store: config.sessionStore || new RedisStore(config.redis),
-    cookie: cookie,
-  });
+if (!config.debug) {
+  options.store = config.sessionStore || middlewares.RedisStore(config.redis);
 }
 
-module.exports = session;
+module.exports = middlewares.session(options);

config/index.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - config/index.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,31 +17,34 @@
 
 var path = require('path');
 var fs = require('fs');
+var os = require('os');
 var mkdirp = require('mkdirp');
+var copy = require('copy-to');
 
 fs.existsSync = fs.existsSync || path.existsSync;
-var pkg = require('../package.json');
+var version = require('../package.json').version;
 
 var root = path.dirname(__dirname);
 
 var config = {
-  version: pkg.version,
+  version: version,
   registryPort: 7001,
   webPort: 7002,
   enableCluster: false,
+  numCPUs: os.cpus().length,
   debug: true, // if debug
   logdir: path.join(root, '.tmp', 'logs'),
   viewCache: false,
   // mysql config
   mysqlServers: [
     {
-      host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
+      host: '127.0.0.1',
       port: 3306,
-      user: 'cnpmjs',
-      password: 'cnpmjs123'
+      user: 'root',
+      password: ''
     }
   ],
-  mysqlDatabase: 'cnpmjstest',
+  mysqlDatabase: 'cnpmjs_test',
   mysqlMaxConnections: 4,
   mysqlQueryTimeout: 5000,
 
@@ -51,6 +54,7 @@ var config = {
     port: 19533,
     pass: 'cnpmjs_dev'
   },
+  jsonLimit: '10mb', // max request json body size
   uploadDir: path.join(root, 'public', 'dist'),
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
   qn: {
@@ -71,6 +75,7 @@ var config = {
     debug: false
   },
 
+  disturl: 'http://dist.u.qiniudn.com',
   logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
   registryHost: 'r.cnpmjs.org',
   customFooter: '', // you can add copyright and site total script html here
@@ -79,8 +84,8 @@ var config = {
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
-    admin: 'admin@cnpmjs.org',
     fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
     dead_horse: 'dead_horse@qq.com',
     cnpmjstest10: 'cnpmjstest10@cnpmjs.org',
   },
@@ -88,15 +93,23 @@ var config = {
   backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
   syncModel: 'none', // 'none', 'all', 'exist'
   syncConcurrency: 1,
+  maxDependencies: 200, // max handle number of package.json `dependencies` property
+
+  limit: {
+    enable: false,
+    token: 'koa-limit:download',
+    limit: 1000,
+    interval: 1000 * 60 * 60 * 24,
+    whiteList: [],
+    blackList: [],
+    message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
+  }
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js
 var customConfig = path.join(root, 'config/config.js');
 if (fs.existsSync(customConfig)) {
-  var options = require(customConfig);
-  for (var k in options) {
-    config[k] = options[k];
-  }
+  copy(require(customConfig)).toCover(config);
 }
 
 mkdirp.sync(config.logdir);
@@ -108,7 +121,5 @@ config.loadConfig = function (customConfig) {
   if (!customConfig) {
     return;
   }
-  for (var key in customConfig) {
-    config[key] = customConfig[key];
-  }
+  copy(customConfig).toCover(config);
 };

controllers/download.js

@@ -14,10 +14,15 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var moment = require('moment');
 var DownloadTotal = require('../proxy/download');
 
 exports.total = function (name, callback) {
+  if (typeof name === 'function') {
+    callback = name;
+    name = null;
+  }
   var end = moment();
   var start = end.clone().subtract('months', 1).startOf('month');
   var lastday = end.clone().subtract('days', 1).format('YYYY-MM-DD');
@@ -74,3 +79,5 @@ exports.total = function (name, callback) {
 
   DownloadTotal[method].apply(DownloadTotal, args);
 };
+
+thunkify(exports);

controllers/registry/user.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - controllers/registry/user.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -15,148 +15,199 @@
  * Module dependencies.
  */
 
-var debug = require('debug')('cnpmjs.org:controllers:registry');
-var logger = require('../../common/logger');
+var debug = require('debug')('cnpmjs.org:controllers:registry:user');
+var utility = require('utility');
+var crypto = require('crypto');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
+var config = require('../../config');
 
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  User.get(name, function (err, row) {
-    if (err) {
-      return next(err);
-    }
-    if (!row) {
-      return next();
-    }
+exports.show = function *(next) {
+  var name = this.params.name;
+  var user = yield User.get(name);
+  if (!user) {
+    return yield *next;
+  }
 
-    res.setHeader('etag', '"' + row.rev + '"');
-    var data = {
-      _id: 'org.couchdb.user:' + row.name,
-      _rev: row.rev,
-      name: row.name,
-      email: row.email,
+  var data = user.json;
+  if (!data) {
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
       type: 'user',
       roles: [],
-      date: row.gmt_modified,
+      date: user.gmt_modified,
     };
-    res.json(data);
-  });
+  }
+  data._cnpm_meta = {
+    id: user.id,
+    npm_user: user.npm_user,
+    gmt_create: user.gmt_create,
+    gmt_modified: user.gmt_modified,
+    admin: !!config.admins[user.name],
+  };
+
+  this.body = data;
 };
 
+function ensurePasswordSalt(user, body) {
+  if (!user.password_sha && body.password) {
+    // create password_sha on server
+    user.salt = crypto.randomBytes(30).toString('hex');
+    user.password_sha = utility.sha1(body.password + user.salt);
+  }
+}
+
+// npm 1.4.4
+// add new user first
+// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'accept-encoding': 'gzip',
+//   'user-agent': 'node/v0.11.12 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '150',
+//   connection: 'close' } { name: 'mk2',
+//   password: '123456',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:33:19.465Z' }
+
+// old npm flow
 // json:
-//  { name: 'fengmk2',
-//    salt: 'xxxx',
-//    password_sha: 'xxxxxx',
-//    email: 'fengmk2@gmail.com',
-//    _id: 'org.couchdb.user:fengmk2',
-//    type: 'user',
-//    roles: [],
-//    date: '2013-12-04T12:56:13.714Z' } }
-exports.add = function (req, res, next) {
-  var name = req.params.name;
-  var body = req.body || {};
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'user-agent': 'node/v0.8.26 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '258',
+//   connection: 'keep-alive' }
+// { name: 'mk2',
+//   salt: '18d8d51936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
+//   password_sha: '8f4408912a6db1d96b132a90856d99db029cef3d',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:39:25.696Z' }
+exports.add = function *() {
+  var name = this.params.name;
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     // roles: body.roles || [],
   };
+
+  ensurePasswordSalt(user, body);
+
   if (!user.name || !user.salt || !user.password_sha || !user.email) {
-    return res.json(422, {
+    this.status = 422;
+    this.body = {
       error: 'paramError',
-      reason: 'params missing'
-    });
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
   }
   debug('add user: %j', user);
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  User.get(name, ep.doneLater(function (row) {
-    if (row) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    User.add(user, ep.done('add'));
-  }));
+  var existUser = yield User.get(name);
+  if (existUser) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'User ' + name + ' already exists.'
+    };
+    return;
+  }
 
-  ep.once('add', function (result) {
-    res.setHeader('etag', '"' + result.rev + '"');
-    // location: 'http://registry.npmjs.org/_users/org.couchdb.user:cnpmjstest1',
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.add(user);
+  this.etag = '"' + result.rev + '"';
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
 };
 
-exports.authSession = function (req, res, next) {
+exports.authSession = function *() {
   // body: {"name":"foo","password":"****"}
-  var body = req.body || {};
+  var body = this.request.body || {};
   var name = body.name;
   var password = body.password;
-  User.auth(name, password, function (err, user) {
-    debug('authSession %s: %j', name, user);
-    if (err) {
-      return next(err);
-    }
-    if (!user) {
-      return res.json(401, {ok: false, name: null, roles: []});
-    }
+  var user = yield User.auth(name, password);
+  debug('authSession %s: %j', name, user);
 
-    req.session.name = user.name;
-    res.json(200, {ok: true, name: user.name, roles: []});
-  });
+  if (!user) {
+    this.status = 401;
+    this.body = {ok: false, name: null, roles: []};
+    return;
+  }
+  var session = yield *this.session;
+  session.name = user.name;
+  this.body = {ok: true, name: user.name, roles: []};
 };
 
-exports.update = function (req, res, next) {
-  var name = req.params.name;
-  var rev = req.params.rev;
+exports.update = function *(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
   if (!name || !rev) {
-    return next();
+    return yield* next;
   }
+  debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
 
-  debug('update: %s, rev: %s, session.name: %s', name, rev, req.session.name);
-
-  if (name !== req.session.name) {
-    // must authSession first
-    res.statusCode = 401;
-    return res.json({
+  if (name !== this.user.name) {
+    // must auth user first
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Name is incorrect.'
-    });
+    };
+    return;
   }
 
-  var body = req.body || {};
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     rev: body.rev || body._rev,
     // roles: body.roles || [],
   };
-  User.update(user, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    //check rev error
-    if (!result) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + user.name,
-      rev: result.rev
-    });
-  });
+
+  ensurePasswordSalt(user, body);
+
+  if (!user.name || !user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    this.body = {
+      error: 'paramError',
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
+  }
+
+  var result = yield User.update(user);
+  if (!result) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
 };

controllers/sync.js

@@ -17,48 +17,59 @@
 var Log = require('../proxy/module_log');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 
-exports.sync = function (req, res, next) {
-  var username = req.session.name || 'anonymous';
-  var name = req.params.name;
-  var publish = req.query.publish === 'true';
-  var noDep = req.query.nodeps === 'true';
-  if (publish && !req.session.isAdmin) {
-    return res.json(403, {
+exports.sync = function *() {
+  var username = this.user.name || 'anonymous';
+  var name = this.params.name;
+  var publish = this.query.publish === 'true';
+  var noDep = this.query.nodeps === 'true';
+  if (publish && !this.user.isAdmin) {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Only admin can publish'
-    });
+    };
+    return;
   }
 
   var options = {
     publish: publish,
     noDep: noDep,
   };
-  SyncModuleWorker.sync(name, username, options, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    if (!result.ok) {
-      return res.json(result.statusCode, result.pkg);
-    }
-    res.json(201, {
-      ok: true,
-      logId: result.logId
-    });
-  });
+
+  var result = yield SyncModuleWorker.sync(name, username, options);
+
+  // friendly 404 reason info
+  if (result.staticCache === 404) {
+    this.status = 404;
+    this.body = {
+      ok: false,
+      reason: 'can not found ' + name + ' in the source registry'
+    };
+    return;
+  }
+  if (!result.ok) {
+    this.status = result.statusCode;
+    this.body = result.pkg;
+    return;
+  }
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logId: result.logId
+  };
 };
 
-exports.getSyncLog = function (req, res, next) {
-  var logId = req.params.id;
-  var name = req.params.name;
-  var offset = Number(req.query.offset) || 0;
-  Log.get(logId, function (err, row) {
-    if (err || !row) {
-      return next(err);
-    }
-    var log = row.log.trim();
-    if (offset > 0) {
-      log = log.split('\n').slice(offset).join('\n');
-    }
-    res.json(200, {ok: true, log: log});
-  });
+exports.getSyncLog = function *(next) {
+  var logId = this.params.id;
+  var offset = Number(this.query.offset) || 0;
+  var row = yield Log.get(logId);
+  if (!row) {
+    return yield* next;
+  }
+
+  var log = row.log.trim();
+  if (offset > 0) {
+    log = log.split('\n').slice(offset).join('\n');
+  }
+  this.body = {ok: true, log: log};
 };

controllers/total.js

@@ -16,31 +16,25 @@
  */
 
 var microtime = require('microtime');
-var eventproxy = require('eventproxy');
 var Total = require('../proxy/total');
-var down = require('./download');
+var Download = require('./download');
 var version = require('../package.json').version;
 var config = require('../config');
 
 var startTime = '' + microtime.now();
 
-exports.show = function (req, res, next) {
-  var ep = eventproxy.create();
-  ep.fail(next);
+exports.show = function *() {
+  var r = yield [Total.get(), Download.total()];
+  var total = r[0];
+  var download = r[1];
 
-  Total.get(ep.done('total'));
-  down.total(null, ep.done('download'));
-  ep.all('total', 'download', function (total, download) {
-    total.download = download;
-    total.db_name = 'registry';
-    total.instance_start_time = startTime;
-    total.node_version = process.version;
-    total.app_version = version;
-    total.donate = 'https://me.alipay.com/imk2';
-    total.sync_model = config.syncModel;
-    if (req.query.callback) {
-      return res.jsonp(total, req.query.callback);
-    }
-    res.json(total);
-  });
+  total.download = download;
+  total.db_name = 'registry';
+  total.instance_start_time = startTime;
+  total.node_version = process.version;
+  total.app_version = version;
+  total.donate = 'https://me.alipay.com/imk2';
+  total.sync_model = config.syncModel;
+
+  this.body = total;
 };

controllers/web/dist.js

@@ -0,0 +1,23 @@
+/**!
+ * cnpmjs.org - controllers/web/dist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../../config');
+
+exports.redirect = function *(next) {
+  var params = this.params;
+  var url = config.disturl + (params[0] || '/');
+  this.redirect(url);
+};

controllers/web/package.js

@@ -26,133 +26,163 @@ var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var ModuleDeps = require('../../proxy/module_deps');
 var setDownloadURL = require('../../lib/common').setDownloadURL;
+var ModuleStar = require('../../proxy/module_star');
 
-exports.display = function (req, res, next) {
-  var params = req.params;
+exports.display = function *(next) {
+  var params = this.params;
   var name = params.name;
   var tag = params.version;
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  if (tag) {
-    var version = semver.valid(tag);
-    if (version) {
-      Module.get(name, version, ep.done('pkg'));
-    } else {
-      Module.getByTag(name, tag, ep.done('pkg'));
-    }
+  var getPackageMethod;
+  var getPackageArgs;
+  var version = semver.valid(tag || '');
+  if (version) {
+    getPackageMethod = 'get';
+    getPackageArgs = [name, version];
   } else {
-    Module.getByTag(name, 'latest', ep.done('pkg'));
+    getPackageMethod = 'getByTag';
+    getPackageArgs = [name, tag || 'latest'];
+  }
+  var r = yield [
+    Module[getPackageMethod].apply(Module, getPackageArgs),
+    down.total(name),
+    ModuleDeps.list(name),
+    ModuleStar.listUsers(name),
+  ];
+  var pkg = r[0];
+  var download = r[1];
+  var dependents = (r[2] || []).map(function (item) {
+    return item.deps;
+  });
+  var users = r[3];
+
+  if (!pkg || !pkg.package) {
+    return yield* next;
   }
 
-  down.total(name, ep.done('download'));
+  pkg.package.fromNow = moment(pkg.publish_time).fromNow();
+  pkg = pkg.package;
+  pkg.users = users;
+  pkg.readme = marked(pkg.readme || '');
+  if (!pkg.readme) {
+    pkg.readme = pkg.description || '';
+  }
 
-  ep.all('pkg', 'download', function (pkg, download) {
-    if (!pkg || !pkg.package) {
-      return next();
-    }
-
-    pkg.package.fromNow = moment(pkg.publish_time).fromNow();
-    pkg = pkg.package;
-    pkg.readme = marked(pkg.readme || '');
-    if (pkg.maintainers) {
-      for (var i = 0; i < pkg.maintainers.length; i++) {
-        var maintainer = pkg.maintainers[i];
-        if (maintainer.email) {
-          maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
-        }
+  if (pkg.maintainers) {
+    for (var i = 0; i < pkg.maintainers.length; i++) {
+      var maintainer = pkg.maintainers[i];
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
       }
     }
+  }
 
-    if (pkg.contributors) {
-      // registry.cnpmjs.org/compressible
-      if (!Array.isArray(pkg.contributors)) {
-        pkg.contributors = [pkg.contributors];
+  if (pkg.contributors) {
+    // registry.cnpmjs.org/compressible
+    if (!Array.isArray(pkg.contributors)) {
+      pkg.contributors = [pkg.contributors];
+    }
+    for (var i = 0; i < pkg.contributors.length; i++) {
+      var contributor = pkg.contributors[i];
+      if (contributor.email) {
+        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
       }
-      for (var i = 0; i < pkg.contributors.length; i++) {
-        var contributor = pkg.contributors[i];
-        if (contributor.email) {
-          contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
-        }
-        if (config.packagePageContributorSearch || !contributor.url) {
-          contributor.url = '/~' + encodeURIComponent(contributor.name);
-        }
+      if (config.packagePageContributorSearch || !contributor.url) {
+        contributor.url = '/~' + encodeURIComponent(contributor.name);
       }
     }
+  }
 
-    if (pkg.repository && pkg.repository.url) {
-      pkg.repository.weburl = giturl.parse(pkg.repository.url);
-    }
+  if (pkg.repository && pkg.repository.url) {
+    pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
+  }
 
-    setLicense(pkg);
+  setLicense(pkg);
 
-    for (var k in download) {
-      download[k] = humanize(download[k]);
-    }
+  for (var k in download) {
+    download[k] = humanize(download[k]);
+  }
+  setDownloadURL(pkg, this, config.registryHost);
 
-    setDownloadURL(pkg, req, config.registryHost);
+  pkg.dependents = dependents;
 
-    res.render('package', {
-      title: 'Package - ' + pkg.name,
-      package: pkg,
-      download: download
-    });
+  yield this.render('package', {
+    title: 'Package - ' + pkg.name,
+    package: pkg,
+    download: download
   });
 };
 
-exports.search = function (req, res, next) {
-  var params = req.params;
-  var word = req.params.word;
-  Module.search(word, function (err, packages) {
-    if (err) {
-      return next(err);
-    }
+exports.search = function *(next) {
+  var params = this.params;
+  var word = params.word;
+  var result = yield Module.search(word);
 
-    res.render('search', {
-      title: 'Keyword - ' + word,
+  var match = null;
+  for (var i = 0; i < result.searchMatchs.length; i++) {
+    var p = result.searchMatchs[i];
+    if (p.name === word) {
+      match = p;
+      break;
+    }
+  }
+
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.body = {
       keyword: word,
-      packages: packages || []
-    });
+      match: match,
+      packages: result.searchMatchs,
+      keywords: result.keywordMatchs,
+    };
+    this.type = 'application/json; charset=utf-8';
+    return;
+  }
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    match: match,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
   });
 };
 
-exports.rangeSearch = function (req, res, next) {
-  var startKey = req.query.startkey || '';
+exports.rangeSearch = function *(next) {
+  var startKey = this.query.startkey || '';
   if (startKey[0] === '"') {
     startKey = startKey.substring(1);
   }
   if (startKey[startKey.length - 1] === '"') {
     startKey = startKey.substring(0, startKey.length - 1);
   }
-  var limit = Number(req.query.limit) || 20;
-  Module.search(startKey, {limit: limit}, function (err, packages) {
-    if (err) {
-      return next(err);
-    }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield Module.search(startKey, {limit: limit});
 
-    var rows = [];
-    for (var i = 0; i < packages.length; i++) {
-      var p = packages[i];
-      var row = {
-        key: p.name,
-        count: 1,
-        value: {
-          name: p.name,
-          description: p.description,
-        }
-      };
-      rows.push(row);
-    }
-    res.json({
-      rows: rows
-    });
-  });
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
+
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
 };
 
-exports.displaySync = function (req, res, next) {
-  var name = req.params.name || req.query.name;
-  res.render('sync', {
+exports.displaySync = function *(next) {
+  var name = this.params.name || this.query.name;
+  yield this.render('sync', {
     name: name,
     title: 'Sync - ' + name
   });

controllers/web/user.js

@@ -15,31 +15,24 @@
  */
 var Module = require('../../proxy/module');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.display = function (req, res, next) {
-  var name = req.params.name;
+exports.display = function *(next) {
+  var name = this.params.name;
 
-  var ep = eventproxy.create();
-  ep.fail(next);
-  Module.listByAuthor(name, ep.done('packages'));
-  User.get(name, ep.done('user'));
+  var r = yield [Module.listByAuthor(name), User.get(name)];
+  var packages = r[0];
+  var user = r[1];
+  if (!user && !packages.length) {
+    return yield* next;
+  }
+  user = {
+    name: name,
+    email: user && user.email
+  };
 
-  ep.all('packages', 'user', function (packages, user) {
-    //because of sync, maybe no this user in database,
-    //but his packages in this registry
-    if (!user && !packages.length) {
-      return next();
-    }
-    user = {
-      name: name,
-      email: user && user.email
-    };
-
-    return res.render('profile', {
-      title: 'User - ' + name,
-      packages: packages || [],
-      user: user
-    });
+  yield this.render('profile', {
+    title: 'User - ' + name,
+    packages: packages || [],
+    user: user
   });
 };

dispatch.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - dispatch.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,7 +17,6 @@
 
 var path = require('path');
 var util = require('util');
-var fs = require('fs');
 var cluster = require('cluster');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
@@ -30,25 +29,25 @@ if (config.enableCluster) {
   });
 
   cluster.on('fork', function (worker) {
-    console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
+    console.log('[%s] [worker:%d] new worker start', Date(), worker.process.pid);
   });
 
   cluster.on('disconnect', function (worker) {
     var w = cluster.fork();
-    console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
-      new Date(), process.pid, worker.process.pid, w.process.pid);
+    console.error('[%s] [master:%s] wroker:%s disconnect, suicide: %s, state: %s. New worker:%s fork',
+      Date(), process.pid, worker.process.pid, worker.suicide, worker.state, w.process.pid);
   });
 
   cluster.on('exit', function (worker, code, signal) {
     var exitCode = worker.process.exitCode;
-    var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
+    var err = new Error(util.format('worker %s died (code: %s, signal: %s, suicide: %s, state: %s)',
+      worker.process.pid, exitCode, signal, worker.suicide, worker.state));
     err.name = 'WorkerDiedError';
-    console.error(err);
+    console.error('[%s] [master:%s] wroker exit: %s', Date(), process.pid, err.stack);
   });
 
-  var numCPUs = require('os').cpus().length;
   // Fork workers.
-  for (var i = 0; i < numCPUs; i++) {
+  for (var i = 0; i < config.numCPUs; i++) {
     cluster.fork();
   }
 

docs/db.sql

@@ -9,10 +9,36 @@ CREATE TABLE `user` (
  `roles` varchar(200) NOT NULL DEFAULT '[]',
  `rev` varchar(40) NOT NULL,
  `email` varchar(400) NOT NULL,
+ `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+ `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
  PRIMARY KEY (`id`),
  UNIQUE KEY `name` (`name`),
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
+-- ALTER TABLE `user`
+--   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+--   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
+
+CREATE TABLE `module_keyword` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `description` longtext,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `keyword_module_name` (`keyword`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
+
+CREATE TABLE `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
 
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
@@ -101,3 +127,13 @@ CREATE TABLE `download_total` (
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
 -- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

docs/flow/publish.md

@@ -1,5 +1,24 @@
 # npm publish flow
 
+## Flows
+
+1. try to put package.json and tgz, maybe base64 tgz body
+2. if new version not exists, publish success
+3. if new version exists, 409, try to get full package info with ?write=true
+4. if new version had publish, show: "Update the 'version' field in package.json and try again."
+
+```bash
+$ cnpm publish
+npm http PUT http://r.cnpmjs.org/cnpmjs.org
+npm http 409 http://r.cnpmjs.org/cnpmjs.org
+npm http GET http://r.cnpmjs.org/cnpmjs.org?write=true
+npm http 200 http://r.cnpmjs.org/cnpmjs.org?write=true
+npm ERR! publish fail Cannot publish over existing version.
+npm ERR! publish fail Update the 'version' field in package.json and try again.
+```
+
+## Details
+
 code: https://github.com/isaacs/npm-registry-client/blob/master/lib/publish.js
 
 * couch login if token not exists: [couch-login](https://github.com/isaacs/couch-login)

docs/web/install.md

@@ -103,13 +103,13 @@ npm install -g cnpm
 alias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
- --disturl=http://dist.u.qiniudn.com\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'
 
  #or put this in .zshrc or .bashrc
  echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
- --disturl=http://dist.u.qiniudn.com\
+ --disturl=http://cnpmjs.org/dist\
  --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
 ```

docs/web/readme.md

@@ -1,19 +1,5 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
-
-[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
-
-## What is this?
-
-> Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
-
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
-
-## Install your private npm registry
-
-@see [Install and Get Started](/install).
-
 ## Registry
 
 * Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
@@ -114,32 +100,32 @@ $(function () {
 });
 </script>
 
-## cnpm cli
+## Usage
 
-alias it:
+use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
+
+```
+npm install -g cnpm --registry=http://r.cnpmjs.org
+```
+
+Or you can alias NPM to use it:
 
 ```bash
 alias cnpm="npm --registry=http://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=http://dist.u.qiniudn.com \
+--disturl=http://cnpmjs.org/dist \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
 $ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=http://dist.u.qiniudn.com \
+  --disturl=http://cnpmjs.org/dist \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
-Or you can just use our `cnpm` cli:
-
-```bash
-$ npm install cnpm -g
-```
-
 ### install
 
-Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
 ```
 $ cnpm install [name]
@@ -188,14 +174,16 @@ Release [History](/history).
 $ git summary
 
  project  : cnpmjs.org
- repo age : 5 weeks
- active   : 98 days
- commits  : 239
- files    : 85
+ repo age : 4 months ago
+ commits  : 472
+ active   : 167 days
+ files    : 104
  authors  :
-   140  fengmk2                 58.6%
-    98  dead_horse              41.0%
-     1  Alsotang                0.4%
+   272  fengmk2                 57.6%
+   195  dead_horse              41.3%
+     2  4simple                 0.4%
+     2  Stanley Zheng           0.4%
+     1  Alsotang                0.2%
 ```
 
 ## npm and cnpm relation

index.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - index.js
  *
  * Copyright(c) cnpmjs.org and other contributors.

lib/common.js

@@ -29,11 +29,11 @@ exports.getCDNKey = function (name, filename) {
   return '/' + name + '/-/' + filename;
 };
 
-exports.setDownloadURL = function (pkg, req, host) {
+exports.setDownloadURL = function (pkg, ctx, host) {
   if (pkg.dist) {
-    host = host || req.headers.host;
+    host = host || ctx.host;
     pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
-      req.connection.encrypted ? 'https' : 'http',
+      ctx.protocol,
       host, pkg.name, pkg.name, pkg.version);
   }
 };
@@ -41,3 +41,17 @@ exports.setDownloadURL = function (pkg, req, host) {
 exports.isAdmin = function (username) {
   return typeof config.admins[username] === 'string';
 };
+
+exports.isMaintainer = function (user, maintainers) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  var username = user.name;
+  maintainers = maintainers || [];
+  var match = maintainers.filter(function (item) {
+    return item.name === username;
+  });
+
+  return match.length > 0;
+};

middleware/auth.js

@@ -20,50 +20,41 @@ var config = require('../config');
 var common = require('../lib/common');
 
 module.exports = function (options) {
-  return function auth(req, res, next) {
-    if (!req.session) {
-      // redis crash
-      req.session = {};
-      return next();
+  return function *auth(next) {
+    var session = yield *this.session;
+    debug('%s, %s, %j', this.url, this.sessionId, session);
+    this.user = {};
+
+    if (session.name) {
+      this.user.name = session.name;
+      this.user.isAdmin = common.isAdmin(session.name);
+      debug('auth exists user: %j, headers: %j', this.user, this.header);
+      return yield *next;
     }
-    req.session.onlySync = config.enablePrivate ? true : false;
-    if (req.session.name) {
-      req.session.isAdmin = common.isAdmin(req.session.name);
-      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
-        req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
-      return next();
-    }
-    var authorization = (req.headers.authorization || '').split(' ')[1] || '';
+
+    var authorization = (this.get('authorization') || '').split(' ')[1] || '';
     authorization = authorization.trim();
     if (!authorization) {
-      return next();
+      return yield *next;
     }
 
     authorization = new Buffer(authorization, 'base64').toString().split(':');
     if (authorization.length !== 2) {
-      return next();
+      return yield *next;
     }
 
     var username = authorization[0];
     var password = authorization[1];
 
-    User.auth(username, password, function (err, row) {
-      if (err) {
-        return next(err);
-      }
+    var row = yield User.auth(username, password);
+    if (!row) {
+      debug('auth fail user: %j, headers: %j', row, this.header);
+      return yield *next;
+    }
 
-      if (!row) {
-        debug('auth fail user: %j, headers: %j', row, req.headers);
-        req.session.name = null;
-        req.session.isAdmin = false;
-        return next();
-      }
-
-      req.session.name = row.name;
-      req.session.isAdmin = common.isAdmin(req.session.name);
-      debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
-        row, req.session.onlySync, req.session.isAdmin, req.headers);
-      next();
-    });
+    this.user.name = row.name;
+    this.user.isAdmin = common.isAdmin(row.name);
+    debug('auth pass user: %j, headers: %j', this.user, this.header);
+    yield *next;
   };
 };

middleware/limit.js

@@ -0,0 +1,34 @@
+/**!
+ * cnpmjs.org - middleware/limit.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+var limit = require('koa-limit');
+var store = require('../common/redis');
+
+var limitConfig = config.limit;
+
+if (!limitConfig.enable) {
+  module.exports = function *ignoreLimit(next) {
+    yield *next;
+  };
+} else {
+
+  if (!config.debug) {
+    limitConfig.store = store;
+  }
+
+  module.exports = limit(limitConfig);
+}

middleware/login.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
-module.exports = function login(req, res, next) {
-  if (!req.session.name) {
-    return res.json(401, {
+module.exports = function *login(next) {
+  if (!this.user.name) {
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Login first.'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/opensearch.js

@@ -24,8 +24,11 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
 
 var lastModifyDate = new Date();
 
-module.exports = function publishable(req, res, next) {
-  res.charset = res.charset || 'utf-8';
-  res.setHeader('Content-Type', 'text/xml');
-  res.send(template.replace('${host}', req.headers.host));
+module.exports = function *publishable(next) {
+  if (this.path === '/opensearch.xml') {
+    this.type = 'text/xml';
+    this.charset = 'utf-8';
+    this.body = template.replace('${host}', this.host);
+  }
+  yield *next;
 };

middleware/publishable.js

@@ -14,13 +14,17 @@
  * Module dependencies.
  */
 
-module.exports = function publishable(req, res, next) {
-  if (req.session.onlySync && !req.session.isAdmin) {
+var config = require('../config');
+
+module.exports = function *publishable(next) {
+  if (config.enablePrivate && !this.user.isAdmin) {
     // private mode, only admin user can publish
-    return res.json(403, {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Private mode enable, only admin can publish this module'
-    });
+    };
+    return;
   }
-  next();
+  yield *next;
 };

middleware/registry_not_found.js

@@ -0,0 +1,28 @@
+/**!
+ * cnpmjs.org - middleware/registry_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+  this.status = 404;
+  this.body = {
+    error: 'not_found',
+    reason: 'document not found'
+  };
+};

middleware/sync_by_install.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - middleware/sync_by_install.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -14,32 +14,27 @@
  * Module dependencies.
  */
 
-var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
 var config = require('../config');
 
 /**
- * req.session.allowSync  -  allow sync triggle by cnpm install
+ * this.allowSync  -  allow sync triggle by cnpm install
  */
 
-module.exports = function (req, res, next) {
+module.exports = function *syncByInstall(next) {
   if (!config.syncByInstall || !config.enablePrivate) {
     // only config.enablePrivate should enable sync on install
-    return next();
+    return yield *next;
   }
   // request not by node, consider it request from web
-  if (req.headers['user-agent'] && req.headers['user-agent'].indexOf('node') !== 0) {
-    return next();
+  var ua = this.get('user-agent');
+  if (!ua || ua.indexOf('node') < 0) {
+    return yield *next;
   }
 
-  req.session.allowSync = true;
-  if (req.session.isAdmin) {
-    // if current user is admin, should not enable auto sync on install, because it would be unpublish
-    req.session.allowSync = false;
+  if (this.query.write) {
+    return yield *next;
   }
 
-  // TODO: allow sync will let publish sync package...
-  req.session.allowSync = false;
-
-  debug('%s allowSync: %s', req.session.name, req.session.allowSync);
-  next();
+  this.allowSync = true;
+  yield *next;
 };

middleware/web_not_found.js

@@ -0,0 +1,33 @@
+/**!
+ * cnpmjs.org - middleware/web_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield *next;
+
+  if (this.status) {
+    return;
+  }
+
+  var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  if (m) {
+    return this.redirect('/package/' + m[1]);
+  }
+
+  this.status = 404;
+  this.type = 'text/html';
+  this.charset = 'utf-8';
+  this.body = 'Cannot GET ' + this.path;
+};

package.json

@@ -1,69 +1,62 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.2.20",
+  "version": "0.4.0",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
-    "test": "make test-all",
+    "test": "make install && make jshint && make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
     "stop": "./bin/nodejsctl stop"
   },
-  "config": {
-    "blanket": {
-      "pattern": "//^((?!(node_modules|test|common)).)*$/",
-      "data-cover-flags": {
-        "debug": false
-      }
-    },
-    "travis-cov": {
-      "threshold": 90
-    }
-  },
   "dependencies": {
-    "bagpipe": "0.3.5",
-    "connect": "2.12.0",
-    "connect-markdown": "0.0.3",
-    "connect-redis": "1.4.6",
-    "connect-render": "0.3.2",
-    "connect-rt": "0.0.2",
-    "debug": "0.7.4",
-    "eventproxy": "0.2.6",
-    "forward": "0.0.4",
-    "giturl": "0.0.1",
-    "graceful": "0.0.5",
+    "co": "3.0.5",
+    "co-gather": "0.0.1",
+    "co-read": "0.0.2",
+    "co-redis": "1.1.0",
+    "co-urllib": "0.2.0",
+    "co-write": "0.3.0",
+    "copy-to": "0.0.3",
+    "debug": "0.8.0",
+    "eventproxy": "0.3.1",
+    "giturl": "0.0.2",
+    "graceful": "0.0.6",
     "gravatar": "1.0.6",
     "humanize-number": "0.0.2",
+    "koa": "0.5.2",
+    "koa-limit": "1.0.0",
+    "koa-markdown": "0.0.3",
+    "koa-middlewares": "0.1.3",
     "logfilestream": "0.1.0",
-    "marked": "0.3.0",
+    "marked": "0.3.2",
     "microtime": "0.5.1",
     "mime": "1.2.11",
     "mkdirp": "0.3.5",
     "moment": "2.5.1",
     "ms": "0.6.2",
-    "mysql": "2.0.1",
-    "nodemailer": "0.6.0",
-    "qn": "0.2.0",
+    "multiline": "0.3.2",
+    "mysql": "2.1.1",
+    "nodemailer": "0.6.1",
+    "qn": "0.2.1",
     "ready": "0.1.1",
-    "response-cookie": "0.0.2",
-    "response-patch": "0.1.1",
+    "redis": "0.10.1",
     "semver": "2.2.1",
-    "urllib": "0.5.5",
-    "urlrouter": "0.5.4",
-    "utility": "0.1.10"
+    "thunkify-wrap": "0.1.1",
+    "utility": "0.1.12"
   },
   "devDependencies": {
     "autod": ">=0.0.13",
-    "blanket": "*",
+    "chunkstream": "0.0.1",
+    "co-mocha": "0.0.2",
     "contributors": "*",
-    "coveralls": "*",
-    "mm": "0.1.8",
+    "cov": "*",
+    "istanbul-harmony": "*",
+    "jshint": "*",
+    "mm": "0.2.1",
     "mocha": "*",
-    "mocha-lcov-reporter": "*",
     "pedding": "0.0.3",
-    "should": "3.0.1",
-    "supertest": "0.9.0",
-    "travis-cov": "*"
+    "should": "3.3.0",
+    "supertest": "0.10.0"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
@@ -76,10 +69,14 @@
     "email": "fengmk2@gmail.com"
   },
   "keywords": [
-    "cnpmjs.org", "npm", "npmjs", "npmjs.org", "registry"
+    "cnpmjs.org",
+    "npm",
+    "npmjs",
+    "npmjs.org",
+    "registry"
   ],
   "engines": {
-    "node": ">= 0.8.0"
+    "node": ">= 0.11.9"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

proxy/download.js

@@ -14,30 +14,49 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var mysql = require('../common/mysql');
+var multiline = require('multiline');
 
-var PLUS_SQL = 'INSERT INTO download_total(gmt_create, gmt_modified, \
-    date, name, count) \
-  VALUES(now(), now(), ?, ?, ?) \
-  ON DUPLICATE KEY UPDATE gmt_modified=now(), \
-    count=count + VALUES(count), name=VALUES(name), date=VALUES(date);';
-
+var PLUS_SQL = multiline(function () {;/*
+  INSERT INTO
+    download_total(gmt_create, gmt_modified, date, name, count)
+  VALUES
+    (now(), now(), ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    count=count + VALUES(count),
+    name=VALUES(name),
+    date=VALUES(date);
+*/});
 exports.plusTotal = function (data, callback) {
   mysql.query(PLUS_SQL, [data.date, data.name, data.count], callback);
 };
 
-var SELECT_ONE_TOTAL_SQL = 'SELECT date, count FROM download_total WHERE date>=? AND date<=? AND name=?;';
-
+var SELECT_ONE_TOTAL_SQL = multiline(function () {;/*
+  SELECT
+    date, count
+  FROM
+    download_total
+  WHERE
+    date>=? AND date<=? AND name=?;
+*/});
 exports.getModuleTotal = function (name, start, end, callback) {
   mysql.query(SELECT_ONE_TOTAL_SQL, [start, end, name], callback);
 };
 
-var SELECT_ALL_TOTAL_SQL = 'SELECT date, sum(count) AS count \
-  FROM download_total \
-  WHERE date>=? AND date<=? \
-  GROUP BY date;';
-
+var SELECT_ALL_TOTAL_SQL = multiline(function () {;/*
+  SELECT
+    date, sum(count) AS count
+  FROM
+    download_total
+  WHERE
+    date>=? AND date<=?
+  GROUP BY
+    date;
+*/});
 exports.getTotal = function (start, end, callback) {
   mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
 };
+
+thunkify(exports);

proxy/module.js

@@ -14,21 +14,37 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
+var multiline = require('multiline');
 
-var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, version, description, package, dist_tarball, dist_shasum, dist_size';
+var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, \
+  version, description, package, dist_tarball, dist_shasum, dist_size';
 
-var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
-  publish_time, author, name, version, package, dist_tarball, dist_shasum, dist_size, description) \
-  VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?) \
-  ON DUPLICATE KEY UPDATE gmt_modified=now(), publish_time=VALUES(publish_time), description=VALUES(description), \
-    author=VALUES(author), name=VALUES(name), version=VALUES(version), package=VALUES(package), \
-    dist_tarball=VALUES(dist_tarball), dist_shasum=VALUES(dist_shasum), dist_size=VALUES(dist_size);';
+var INSERT_MODULE_SQL = multiline(function () {;/*
+  INSERT INTO
+    module(gmt_create, gmt_modified, publish_time, author, name, version,
+      package, dist_tarball, dist_shasum, dist_size, description)
+  VALUES
+    (now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    gmt_modified=now(),
+    publish_time=VALUES(publish_time),
+    description=VALUES(description),
+    author=VALUES(author),
+    name=VALUES(name),
+    version=VALUES(version),
+    package=VALUES(package),
+    dist_tarball=VALUES(dist_tarball),
+    dist_shasum=VALUES(dist_shasum),
+    dist_size=VALUES(dist_size);
+*/});
 
 exports.add = function (mod, callback) {
+  var keywords = mod.package.keywords;
   var pkg;
   try {
     pkg = stringifyPackage(mod.package);
@@ -51,15 +67,111 @@ exports.add = function (mod, callback) {
       return callback(err);
     }
     callback(null, {id: result.insertId, gmt_modified: new Date()});
+
+    if (typeof keywords === 'string') {
+      keywords = [keywords];
+    }
+
+    if (!Array.isArray(keywords)) {
+      return;
+    }
+
+    var words = [];
+    for (var i = 0; i < keywords.length; i++) {
+      var w = keywords[i];
+      if (typeof w === 'string') {
+        w = w.trim();
+        if (w) {
+          words.push(w);
+        }
+      }
+    }
+
+    if (words.length === 0) {
+      return;
+    }
+
+    // add keywords
+    exports.addKeywords(mod, description, words, utility.noop);
   });
 };
 
-var UPDATE_DESC_SQL = 'UPDATE module SET description=? WHERE id=?;';
+var GET_KEYWORD_SQL = multiline(function () {;/*
+  SELECT
+    keyword
+  FROM
+    module_keyword
+  WHERE
+    name = ?
+  ORDER BY
+    keyword;
+*/});
+
+exports.getKeywords = function (name, callback) {
+  mysql.query(GET_KEYWORD_SQL, [name], function (err, rows) {
+    var keywords = [];
+    if (rows && rows.length) {
+      keywords = rows.map(function (r) {
+        return r.keyword;
+      });
+    }
+    callback(err, keywords);
+  });
+};
+
+var ADD_KEYWORD_SQL = multiline(function () {;/*
+  INSERT INTO
+    module_keyword(gmt_create, keyword, name, description)
+  VALUES
+    (now(), ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    description=VALUES(description);
+*/});
+
+exports.addKeywords = function (name, description, keywords, callback) {
+  var sql = '';
+  var values = [];
+  for (var i = 0; i < keywords.length; i++) {
+    sql += ADD_KEYWORD_SQL;
+    values.push(keywords[i]);
+    values.push(name);
+    values.push(description);
+  }
+  mysql.query(sql, values, function (err, results) {
+    if (err) {
+      return callback(err);
+    }
+    var ids = [];
+    for (var i = 0; i < results.length; i++) {
+      var r = results[i];
+      if (r.insertId) {
+        ids.push(r.insertId);
+      }
+    }
+    callback(null, ids);
+  });
+};
+
+var UPDATE_DESC_SQL = multiline(function () {;/*
+  UPDATE
+    module
+  SET
+    description=?
+  WHERE
+    id=?;
+*/});
 exports.updateDescription = function (id, description, callback) {
   mysql.query(UPDATE_DESC_SQL, [description, id], callback);
 };
 
-var UPDATE_PACKAGE_SQL = 'UPDATE module SET package=? WHERE id=?;';
+var UPDATE_PACKAGE_SQL = multiline(function () {;/*
+  UPDATE
+    module
+  SET
+    package=?
+  WHERE
+    id=?;
+*/});
 exports.updateReadme = function (id, readme, callback) {
   exports.getById(id, function (err, data) {
     if (err) {
@@ -72,9 +184,19 @@ exports.updateReadme = function (id, readme, callback) {
   });
 };
 
-var UPDATE_DIST_SQL = 'UPDATE module SET publish_time=?, version=?, package=?, \
-  dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
-
+var UPDATE_DIST_SQL = multiline(function () {;/*
+  UPDATE
+    module
+  SET
+    publish_time=?,
+    version=?,
+    package=?,
+    dist_tarball=?,
+    dist_shasum=?,
+    dist_size=?
+  WHERE
+    id=?;
+*/});
 exports.update = function (mod, callback) {
   var pkg;
   try {
@@ -106,13 +228,22 @@ function parseRow(row) {
     }
   }
 }
+exports.parseRow = parseRow;
 
 function stringifyPackage(pkg) {
   return encodeURIComponent(JSON.stringify(pkg));
 }
 
-var SELECT_MODULE_BY_ID_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE id=?;';
 
+var SELECT_MODULE_BY_ID_SQL = multiline(function () {;/*
+  SELECT
+    id, publish_time, gmt_create, gmt_modified, author, name,
+    version, description, package, dist_tarball, dist_shasum, dist_size
+  FROM
+    module
+  WHERE
+    id=?;
+*/});
 exports.getById = function (id, callback) {
   id = Number(id);
   mysql.queryOne(SELECT_MODULE_BY_ID_SQL, [id], function (err, row) {
@@ -129,8 +260,15 @@ exports.getById = function (id, callback) {
   });
 };
 
-var SELECT_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE name=? AND version=?;';
-
+var SELECT_MODULE_SQL = multiline(function () {;/*
+  SELECT
+    id, publish_time, gmt_create, gmt_modified, author, name,
+    version, description, package, dist_tarball, dist_shasum, dist_size
+  FROM
+    module
+  WHERE
+    name=? AND version=?;
+*/});
 exports.get = function (name, version, callback) {
   mysql.queryOne(SELECT_MODULE_SQL, [name, version], function (err, row) {
     if (err || !row) {
@@ -146,14 +284,26 @@ exports.get = function (name, version, callback) {
   });
 };
 
-var INSERT_TAG_SQL = 'INSERT INTO tag(gmt_create, gmt_modified, \
-  name, tag, version, module_id) \
-  VALUES(now(), now(), ?, ?, ?, ?) \
-  ON DUPLICATE KEY UPDATE gmt_modified=now(), module_id=VALUES(module_id), \
-    name=VALUES(name), tag=VALUES(tag), version=VALUES(version);';
-
-var SELECT_MODULE_ID_SQL = 'SELECT id FROM module WHERE name=? AND version=?;';
-
+var SELECT_MODULE_ID_SQL = multiline(function () {;/*
+  SELECT
+    id
+  FROM
+    module
+  WHERE
+    name=? AND version=?;
+*/});
+var INSERT_TAG_SQL = multiline(function () {;/*
+  INSERT INTO
+    tag(gmt_create, gmt_modified, name, tag, version, module_id)
+  VALUES
+    (now(), now(), ?, ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    gmt_modified=now(),
+    module_id=VALUES(module_id),
+    name=VALUES(name),
+    tag=VALUES(tag),
+    version=VALUES(version);
+*/});
 exports.addTag = function (name, tag, version, callback) {
   mysql.queryOne(SELECT_MODULE_ID_SQL, [name, version], function (err, row) {
     if (err) {
@@ -169,8 +319,14 @@ exports.addTag = function (name, tag, version, callback) {
   });
 };
 
-var SELECT_TAG_SQL = 'SELECT tag, version, gmt_modified, module_id FROM tag WHERE name=? AND tag=?;';
-
+var SELECT_TAG_SQL = multiline(function () {;/*
+  SELECT
+    tag, version, gmt_modified, module_id
+  FROM
+    tag
+  WHERE
+    name=? AND tag=?;
+*/});
 exports.getByTag = function (name, tag, callback) {
   mysql.queryOne(SELECT_TAG_SQL, [name, tag], function (err, row) {
     if (err || !row) {
@@ -180,21 +336,51 @@ exports.getByTag = function (name, tag, callback) {
   });
 };
 
-var DELETE_TAGS_SQL = 'DELETE FROM tag WHERE name=?;';
-
+var DELETE_TAGS_SQL = multiline(function () {;/*
+  DELETE FROM
+    tag
+  WHERE
+    name=?;
+*/});
 exports.removeTags = function (name, callback) {
   mysql.query(DELETE_TAGS_SQL, [name], callback);
 };
 
-var SELECT_ALL_TAGS_SQL = 'SELECT tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
+var DELETE_TAGS_BY_IDS_SQL = multiline(function () {;/*
+  DELETE FROM
+    tag
+  WHERE
+    id in (?);
+*/});
+exports.removeTagsByIds = function (ids, callback) {
+  mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
+};
 
+var SELECT_ALL_TAGS_SQL = multiline(function () {;/*
+  SELECT
+    id, tag, version, gmt_modified, module_id
+  FROM
+    tag
+  WHERE
+    name=?;
+*/});
 exports.listTags = function (name, callback) {
   mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
 };
 
-var SELECT_LATEST_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS +
-  ' FROM module WHERE name=? AND version <> "next" ORDER BY publish_time DESC LIMIT 1;';
-
+var SELECT_LATEST_MODULE_SQL = multiline(function () {;/*
+  SELECT
+    id, publish_time, gmt_create, gmt_modified, author, name,
+    version, description, package, dist_tarball, dist_shasum, dist_size
+  FROM
+    module
+  WHERE
+    name=? AND version <> "next"
+  ORDER BY
+    publish_time DESC
+  LIMIT
+    1;
+*/});
 exports.getLatest = function (name, callback) {
   exports.getByTag(name, 'latest', function (err, row) {
     if (err || row) {
@@ -217,8 +403,17 @@ exports.getLatest = function (name, callback) {
   });
 };
 
-var LIST_MODULE_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE name=? ORDER BY id DESC;';
-
+var LIST_MODULE_SQL = multiline(function () {;/*
+  SELECT
+    id, publish_time, gmt_create, gmt_modified, author, name,
+    version, description, package, dist_tarball, dist_shasum, dist_size
+  FROM
+    module
+  WHERE
+    name=?
+  ORDER BY
+    id DESC;
+*/});
 exports.listByName = function (name, callback) {
   mysql.query(LIST_MODULE_SQL, [name], function (err, rows) {
     if (err) {
@@ -238,10 +433,23 @@ exports.listByName = function (name, callback) {
   });
 };
 
-var LIST_SINCE_SQLS = [
-  'SELECT module_id FROM tag WHERE tag="latest" AND gmt_modified>?',
-  'SELECT name, package FROM module WHERE id IN (?);'
-];
+var LIST_SINCE_SQLS = [];
+LIST_SINCE_SQLS.push(multiline(function () {;/*
+  SELECT
+    module_id
+  FROM
+    tag
+  WHERE
+    tag="latest" AND gmt_modified>?;
+*/}));
+LIST_SINCE_SQLS.push(multiline(function () {;/*
+  SELECT
+    distinct(name)
+  FROM
+    module
+  WHERE
+    id IN (?);
+*/}));
 exports.listSince = function (start, callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
@@ -259,31 +467,91 @@ exports.listSince = function (start, callback) {
   });
 };
 
-var LIST_SHORT_SQL = 'SELECT distinct(name) FROM tag ORDER BY name';
+var LIST_ALL_NAME_SQL = multiline(function () {;/*
+  SELECT
+    distinct(name)
+  FROM
+    module;
+*/});
+exports.listAllNames = function (callback) {
+  mysql.query(LIST_ALL_NAME_SQL, [], callback);
+};
+
+var LIST_SHORT_SQL = multiline(function () {;/*
+  SELECT
+    distinct(name)
+  FROM
+    tag
+  ORDER BY
+    name;
+*/});
 exports.listShort = function (callback) {
   mysql.query(LIST_SHORT_SQL, callback);
 };
 
-var LIST_ALL_MODULE_NAMES_SQL = 'SELECT distinct(name) FROM module ORDER BY name';
+var LIST_ALL_MODULE_NAMES_SQL = multiline(function () {;/*
+  SELECT
+    distinct(name)
+  FROM
+    module
+  ORDER BY
+    name;
+*/});
 exports.listAllModuleNames = function (callback) {
   mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
 };
 
-var DELETE_MODULE_BY_NAME_SQL = 'DELETE FROM module WHERE name=?;';
+var DELETE_MODULE_BY_NAME_SQL = multiline(function () {;/*
+  DELETE FROM
+    module
+  WHERE
+    name=?;
+*/});
 exports.removeByName = function (name, callback) {
   mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
 };
 
-var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = 'DELETE FROM module WHERE name=? AND version IN(?);';
+var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = multiline(function () {;/*
+  DELETE FROM
+    module
+  WHERE
+    name=? AND version in(?);
+*/});
 exports.removeByNameAndVersions = function (name, versions, callback) {
   mysql.query(DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL, [name, versions], callback);
 };
 
-var LIST_BY_AUTH_SQLS = [
-  'SELECT distinct(name) AS name FROM module WHERE author = ? ORDER BY publish_time DESC LIMIT 100;',
-  'SELECT module_id FROM tag WHERE tag="latest" AND name IN (?)',
-  'SELECT name, description FROM module WHERE id IN (?) ORDER BY publish_time DESC'
-];
+var LIST_BY_AUTH_SQLS = [];
+LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
+  SELECT
+    distinct(name) AS name
+  FROM
+    module
+  WHERE
+    author=?
+  ORDER BY
+    publish_time DESC
+  LIMIT
+    100;
+*/}));
+LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
+  SELECT
+    module_id
+  FROM
+    tag
+  WHERE
+    tag="latest" AND name IN (?);
+*/}));
+LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
+  SELECT
+    name, description
+  FROM
+    module
+  WHERE
+    id IN (?)
+  ORDER BY
+    publish_time DESC;
+*/}));
 exports.listByAuthor = function (author, callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
@@ -310,10 +578,40 @@ exports.listByAuthor = function (author, callback) {
   });
 };
 
-var SEARCH_SQLS = [
-  'SELECT module_id FROM tag WHERE name LIKE ? AND  tag="latest" ORDER BY name LIMIT ?;',
-  'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;'
-];
+var SEARCH_MODULES_SQL = multiline(function () {;/*
+  SELECT
+    module_id
+  FROM
+    tag
+  WHERE
+    name LIKE ? AND tag="latest"
+  ORDER BY
+    name
+  LIMIT
+    ?;
+*/});
+var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {;/*
+  SELECT
+    name, description
+  FROM
+    module_keyword
+  WHERE
+    keyword=?
+  ORDER BY
+    id DESC
+  LIMIT
+    ?;
+*/});
+var QUERY_MODULES_BY_ID_SQL = multiline(function () {;/*
+  SELECT
+    name, description
+  FROM
+    module
+  WHERE
+    id IN (?)
+  ORDER BY
+    name;
+*/});
 exports.search = function (word, options, callback) {
   if (typeof options === 'function') {
     callback = options;
@@ -321,21 +619,80 @@ exports.search = function (word, options, callback) {
   }
   options = options || {};
   var limit = options.limit || 100;
-  word = word.replace(/^%/, '') + '%'; //ignore prefix %
+  word = word.replace(/^%/, ''); //ignore prefix %
   var ep = eventproxy.create();
   ep.fail(callback);
-  mysql.query(SEARCH_SQLS[0], [word, limit], ep.done(function (rows) {
-    if (!rows || rows.length === 0) {
-      return callback(null, []);
+
+  // search flows:
+  // 1. prefix search by name
+  // 2. like search by name
+  // 3. keyword equal search
+  var ids = {};
+
+  mysql.query(SEARCH_MODULES_SQL, [word + '%', limit ], ep.done(function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
     }
-    ep.emit('ids', rows.map(function (r) {
-      return r.module_id;
-    }));
+    if (rows.length >= 20) {
+      return ep.emit('ids', Object.keys(ids));
+    }
+
+    mysql.query(SEARCH_MODULES_SQL, [ '%' + word + '%', limit ], ep.done('likeSearch'));
   }));
 
-  ep.on('ids', function (ids) {
-    mysql.query(SEARCH_SQLS[1], [ids], ep.done(function (modules) {
-      callback(null, modules);
+  mysql.query(SEARCH_MODULES_BY_KEYWORD_SQL, [ word, limit ], ep.done('keywordRows'));
+
+  ep.on('likeSearch', function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
+    }
+
+    ep.emit('ids', Object.keys(ids));
+  });
+
+  ep.all('ids', 'keywordRows', function (ids, keywordRows) {
+    keywordRows = keywordRows || [];
+    var data = {
+      keywordMatchs: keywordRows,
+      searchMatchs: []
+    };
+    if (ids.length === 0) {
+      return callback(null, data);
+    }
+
+    mysql.query(QUERY_MODULES_BY_ID_SQL, [ids], ep.done(function (modules) {
+      data.searchMatchs = modules;
+      callback(null, data);
     }));
   });
 };
+
+thunkify(exports);
+
+exports.updateMaintainers = function *(id, maintainers) {
+  var mod = yield exports.getById(id);
+  mod.package.maintainers = maintainers;
+  var pkg = stringifyPackage(mod.package);
+  return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
+};
+
+var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
+  SELECT
+    id, gmt_modified
+  FROM
+    module
+  WHERE
+    name=?
+  ORDER BY
+    gmt_modified DESC;
+*/});
+exports.getLastModified = function *(name) {
+  var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
+  return row && row.gmt_modified;
+};

proxy/module_deps.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - proxy/module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var thunkify = require('thunkify-wrap');
+var mysql = require('../common/mysql');
+var multiline = require('multiline');
+
+var LIST_DEPS_SQL = multiline(function () {;/*
+  SELECT
+    deps
+  FROM
+    module_deps
+  WHERE
+    name=?;
+*/});
+exports.list = function (name, callback) {
+  mysql.query(LIST_DEPS_SQL, [name], callback);
+};
+
+var INSERT_DEPS_SQL = multiline(function () {;/*
+  INSERT INTO
+    module_deps(gmt_create, name, deps)
+  VALUES
+    (now(), ?, ?);
+*/});
+exports.add = function (name, deps, callback) {
+  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
+    if (err && err.code === 'ER_DUP_ENTRY') {
+      err = null;
+    }
+    callback(err);
+  });
+};
+
+var DELETE_DEPS_SQL = multiline(function () {;/*
+  DELETE FROM
+    module_deps
+  WHERE
+    name=? AND deps=?;
+*/});
+exports.remove = function (name, deps, callback) {
+  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
+};
+
+thunkify(exports);

proxy/module_log.js

@@ -14,11 +14,16 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var mysql = require('../common/mysql');
+var multiline = require('multiline');
 
-var INSERT_LOG_SQL = 'INSERT INTO module_log(gmt_create, gmt_modified, name, username, log) \
-  VALUES(now(), now(), ?, ?, "");';
-
+var INSERT_LOG_SQL = multiline(function () {;/*
+  INSERT INTO
+    module_log(gmt_create, gmt_modified, name, username, log)
+  VALUES
+    (now(), now(), ?, ?, "");
+*/});
 exports.create = function (data, callback) {
   mysql.query(INSERT_LOG_SQL, [data.name, data.username], function (err, result) {
     if (err) {
@@ -28,7 +33,15 @@ exports.create = function (data, callback) {
   });
 };
 
-var APPEND_SQL = 'UPDATE module_log SET log=CONCAT(log, ?), gmt_modified=now() WHERE id=?;';
+var APPEND_SQL = multiline(function () {;/*
+  UPDATE
+    module_log
+  SET
+    log=CONCAT(log, ?),
+    gmt_modified=now()
+  WHERE
+    id=?;
+*/});
 exports.append = function (id, log, callback) {
   log = '\n' + log;
   mysql.query(APPEND_SQL, [log, id], function (err) {
@@ -36,7 +49,16 @@ exports.append = function (id, log, callback) {
   });
 };
 
-var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?;';
+var SELECT_SQL = multiline(function () {;/*
+  SELECT
+    *
+  FROM
+    module_log
+  WHERE
+    id=?;
+*/});
 exports.get = function (id, callback) {
   mysql.queryOne(SELECT_SQL, [id], callback);
 };
+
+thunkify(exports);

proxy/module_star.js

@@ -0,0 +1,73 @@
+/**!
+ * cnpmjs.org - proxy/module_star.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var multiline = require('multiline');
+
+var ADD_SQL = multiline(function () {;/*
+  INSERT iNTO
+    module_star(name, user)
+  VALUES
+    (?, ?);
+*/});
+exports.add = function *add(name, user) {
+  try {
+    yield mysql.query(ADD_SQL, [name, user]);
+  } catch (err) {
+    if (err.code !== 'ER_DUP_ENTRY') {
+      throw err;
+    }
+  }
+};
+
+var REMOVE_SQL = multiline(function () {;/*
+  DELETE FROM
+    module_star
+  WHERE
+    name = ? AND user = ?;
+*/});
+exports.remove = function *(name, user) {
+  return yield mysql.query(REMOVE_SQL, [name, user]);
+};
+
+var LIST_USERS_SQL = multiline(function () {;/*
+  SELECT
+    user
+  FROM
+    module_star
+  WHERE
+    name = ?;
+*/});
+exports.listUsers = function *(name) {
+  var rows = yield mysql.query(LIST_USERS_SQL, [name]);
+  return rows.map(function (r) {
+    return r.user;
+  });
+};
+
+var LIST_USER_MODULES_SQL = multiline(function () {;/*
+  SELECT
+    name
+  FROM
+    module_star
+  WHERE
+    user = ?;
+*/});
+exports.listUserModules = function *(user) {
+  return (yield mysql.query(LIST_USER_MODULES_SQL, [user])).map(function (r) {
+    return r.name;
+  });
+};

proxy/npm.js

@@ -14,44 +14,63 @@
  * Module dependencies.
  */
 
-var urllib = require('urllib');
+var urllib = require('co-urllib');
 var config = require('../config');
 
-function request(url, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = null;
-  }
+var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+
+function *request(url, options) {
   options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
+  options.headers = {
+    'user-agent': USER_AGENT
+  };
   url = config.sourceNpmRegistry + url;
-  urllib.request(url, options, callback);
+  var r;
+  try {
+    r = yield *urllib.request(url, options);
+  } catch (err) {
+    var statusCode = err.status || -1;
+    var data = err.data || '[empty]';
+    if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
+      err.name = 'NPMServerError';
+      err.message = 'Status ' + statusCode + ', ' + data.toString();
+    }
+    throw err;
+  }
+  return r;
 }
 
-exports.get = function (name, callback) {
-  request('/' + name, function (err, data, res) {
-    if (err) {
-      return callback(err);
-    }
-    res = res || {};
-    if (res.statusCode === 404) {
-      data = null;
-    }
-    callback(null, data, res);
+exports.getUser = function *(name) {
+  var url = '/-/user/org.couchdb.user:' + name;
+  var r = yield *request(url);
+  var data = r.data;
+  if (data && !data.name) {
+    data = null;
+  }
+  return data;
+};
+
+exports.get = function *(name) {
+  var r = yield *request('/' + name);
+  var data = r.data;
+  if (r.status === 404) {
+    data = null;
+  }
+  return data;
+};
+
+exports.getAllSince = function *(startkey) {
+  var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, {
+    timeout: 300000
   });
+  return r.data;
 };
 
-exports.getAllSince = function (startkey, callback) {
-  request('/-/all/since?stale=update_after&startkey=' + startkey, {
-    dataType: 'json',
+exports.getShort = function *() {
+  var r = yield *request('/-/short', {
     timeout: 300000
-  }, callback);
-};
-
-exports.getShort = function (callback) {
-  request('/-/short', {
-    dataType: 'json',
-    timeout: 300000
-  }, callback);
+  });
+  return r.data;
 };

proxy/sync_module_worker.js

@@ -15,21 +15,29 @@
  * Module dependencies.
  */
 
+var co = require('co');
+var gather = require('co-gather');
+var thunkify = require('thunkify-wrap');
 var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
 var util = require('util');
 var fs = require('fs');
 var path = require('path');
 var crypto = require('crypto');
-var eventproxy = require('eventproxy');
-var urllib = require('urllib');
+var urllib = require('co-urllib');
 var utility = require('utility');
+var ms = require('ms');
 var nfs = require('../common/nfs');
 var npm = require('./npm');
 var common = require('../lib/common');
 var Module = require('./module');
+var ModuleDeps = require('./module_deps');
 var Log = require('./module_log');
-var ms = require('ms');
+var config = require('../config');
+var ModuleStar = require('./module_star');
+var User = require('./user');
+
+var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -86,9 +94,24 @@ SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
 SyncModuleWorker.prototype.start = function () {
   this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
     this.username, this.names[0], this.concurrency, this.noDep, this._publish);
-  for (var i = 0; i < this.concurrency; i++) {
-    this.next(i);
-  }
+  var self = this;
+  co(function *() {
+    var arr = [];
+    for (var i = 0; i < self.concurrency; i++) {
+      arr.push(self.next(i));
+    }
+    yield arr;
+  })();
+};
+
+SyncModuleWorker.prototype.pushSuccess = function (name) {
+  this.successes.push(name);
+  this.emit('success', name);
+};
+
+SyncModuleWorker.prototype.pushFail = function (name) {
+  this.fails.push(name);
+  this.emit('fail', name);
 };
 
 SyncModuleWorker.prototype.add = function (name) {
@@ -97,10 +120,11 @@ SyncModuleWorker.prototype.add = function (name) {
   }
   this.nameMap[name] = true;
   this.names.push(name);
+  this.emit('add', name);
   this.log('    add dependencies: %s', name);
 };
 
-SyncModuleWorker.prototype.next = function (concurrencyId) {
+SyncModuleWorker.prototype.next = function *(concurrencyId) {
   var name = this.names.shift();
   if (!name) {
     return process.nextTick(this.finish.bind(this));
@@ -108,369 +132,514 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
 
   var that = this;
   that.syncingNames[name] = true;
-  npm.get(name, function (err, pkg, response) {
-    var statusCode = response && response.statusCode || -1;
-    if (!err && !pkg) {
-      err = new Error('Module ' + name + ' not exists, http status ' + statusCode);
-      err.name = 'NpmModuleNotExsitsError';
-    }
-    if (err) {
-      if (statusCode === 404) {
-        that.successes.push(name);
-      } else {
-        that.fails.push(name);
-      }
-      that.log('[error] [%s] get package error: %s', name, err.stack);
-      delete that.syncingNames[name];
-      return that.next(concurrencyId);
+  var pkg;
+  // get from npm
+  try {
+    pkg = yield npm.get(name);
+  } catch (err) {
+    // if 404
+    if (err.res && err.res.statusCode === 404) {
+      that.pushSuccess(name);
+    } else {
+      that.pushFail(name);
     }
+    that.log('[error] [%s] get package error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  if (!pkg) {
+    that.log('[error] [%s] get package error: package not exist', name);
+    delete that.syncingNames[name];
+    yield that.next(concurrencyId);
+    return;
+  }
 
-    that.log('[c#%d] [%s] start...', concurrencyId, name);
-    that._sync(name, pkg, function (err, versions) {
-      delete that.syncingNames[name];
-      if (err) {
-        that.fails.push(name);
-        that.log('[error] [%s] sync error: %s', name, err.stack);
-        return that.next(concurrencyId);
-      }
-      that.log('[%s] synced success, %d versions: %s',
-        name, versions.length, versions.join(', '));
-      that.successes.push(name);
-      that.emit('success', name);
-      that.next(concurrencyId);
-    });
-  });
+  that.log('[c#%d] [%s] start...', concurrencyId, name);
+  var versions;
+  try {
+    versions = yield that._sync(name, pkg);
+  } catch (err) {
+    that.pushFail(name);
+    that.log('[error] [%s] sync error: %s', name, err.stack);
+    delete that.syncingNames[name];
+    yield *that.next(concurrencyId);
+    return;
+  }
+  that.log('[%s] synced success, %d versions: %s',
+    name, versions.length, versions.join(', '));
+  that.pushSuccess(name);
+  delete that.syncingNames[name];
+  yield that.next(concurrencyId);
 };
 
-SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
+function *_listStarUsers(modName) {
+  var users = yield ModuleStar.listUsers(modName);
+  var userMap = {};
+  users.forEach(function (user) {
+    userMap[user] = true;
+  });
+  return userMap;
+}
+
+function *_addStar(modName, username) {
+  yield ModuleStar.add(modName, username);
+}
+
+function *_saveNpmUser(username) {
+  var user = yield *npm.getUser(username);
+  if (!user) {
+    return;
+  }
+  yield User.saveNpmUser(user);
+}
+
+SyncModuleWorker.prototype._sync = function *(name, pkg) {
   var username = this.username;
   var that = this;
-  var ep = eventproxy.create();
-  ep.fail(callback);
 
   var hasModules = false;
-  Module.listByName(name, ep.done(function (rows) {
-    hasModules = rows.length > 0;
-    var map = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.package || !r.package.dist) {
-        // package json parse error
-        continue;
-      }
+  var result = yield [
+    Module.listByName(name),
+    Module.listTags(name),
+    _listStarUsers(name)
+  ];
+  var moduleRows = result[0];
+  var tagRows = result[1];
+  var existsStarUsers = result[2];
 
-      if (r.package && r.package._publish_on_cnpm) {
-        // publish on cnpm, dont sync this version package
-        that.log('  [%s] publish on local cnpm, don\'t sync', name);
-        ep.unbind();
-        callback(null, []);
-        return;
-      }
-
-      if (r.version === 'next') {
-        continue;
-      }
-      if (!map.latest) {
-        map.latest = r;
-      }
-      map[r.version] = r;
+  hasModules = moduleRows.length > 0;
+  var map = {};
+  for (var i = 0; i < moduleRows.length; i++) {
+    var r = moduleRows[i];
+    if (!r.package || !r.package.dist) {
+      // package json parse error
+      continue;
     }
-    ep.emit('existsMap', map);
-  }));
 
-  Module.listTags(name, ep.done(function (rows) {
-    var tags = {};
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (!r.module_id) {
-        // no module_id, need to sync tags
-        continue;
-      }
-      tags[r.tag] = r.version;
+    if (r.package && r.package._publish_on_cnpm) {
+      // publish on cnpm, dont sync this version package
+      that.log('  [%s] publish on local cnpm, don\'t sync', name);
+      return [];
     }
-    ep.emit('existsTags', tags);
-  }));
+
+    if (r.version === 'next') {
+      continue;
+    }
+    if (!map.latest) {
+      map.latest = r;
+    }
+    map[r.version] = r;
+  }
+
+  var tags = {};
+  for (var i = 0; i < tagRows.length; i++) {
+    var r = tagRows[i];
+    if (!r.module_id) {
+      // no module_id, need to sync tags
+      continue;
+    }
+    tags[r.tag] = r.version;
+  }
 
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
   var missingReadmes = [];
+  var missingStarUsers = [];
+  var npmUsernames = {};
 
-  ep.all('existsMap', 'existsTags', function (map, tags) {
-    var times = pkg.time || {};
-    pkg.versions = pkg.versions || {};
-    var versionNames = Object.keys(times);
-    if (versionNames.length === 0) {
-      versionNames = Object.keys(pkg.versions);
+  // find out all user names
+  for (var v in pkg.versions) {
+    var p = pkg.versions[v];
+
+    var maintainers = p.maintainers || [];
+    if (maintainers && !Array.isArray(maintainers)) {
+      // http://r.cnpmjs.org/jasmine-node
+      // TODO: "maintainers": "Martin Häger <martin.haeger@gmail.com>",
+      maintainers = [maintainers];
     }
-    if (versionNames.length === 0) {
-      that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
-      if (!hasModules) {
-        // save a next module
-        var maintainer = pkg.maintainers && pkg.maintainers[0];
-        if (maintainer && maintainer.name) {
-          maintainer = maintainer.name;
-        }
-        if (!maintainer) {
-          maintainer = '-';
-        }
-        var nextMod = {
+
+    maintainers.forEach(function (m) {
+      if (m.name) {
+        npmUsernames[m.name.toLowerCase()] = 1;
+      }
+    });
+  }
+
+  // get the missing star users
+  var starUsers = pkg.users || {};
+  for (var k in starUsers) {
+    if (!existsStarUsers[k]) {
+      missingStarUsers.push(k);
+    }
+    npmUsernames[k.toLowerCase()] = 1;
+  }
+  that.log('  [%s] found %d missing star users', name, missingStarUsers.length);
+
+  var times = pkg.time || {};
+  pkg.versions = pkg.versions || {};
+  var versionNames = Object.keys(times);
+  if (versionNames.length === 0) {
+    versionNames = Object.keys(pkg.versions);
+  }
+  if (versionNames.length === 0) {
+    that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
+    if (!hasModules) {
+      // save a next module
+      var maintainer = pkg.maintainers && pkg.maintainers[0];
+      if (maintainer && maintainer.name) {
+        maintainer = maintainer.name;
+      }
+      if (!maintainer) {
+        maintainer = '-';
+      }
+      var nextMod = {
+        name: name,
+        version: 'next',
+        author: maintainer,
+        package: {
           name: name,
           version: 'next',
-          author: maintainer,
-          package: {
-            name: name,
-            version: 'next',
-            description: pkg.description || '',
-            readme: pkg.readme || '',
-            maintainers: pkg.maintainers || {
-              name: maintainer
-            },
+          description: pkg.description || '',
+          readme: pkg.readme || '',
+          maintainers: pkg.maintainers || {
+            name: maintainer
           },
-        };
-        Module.add(nextMod, function (err, result) {
-          that.log('  [%s] save next module, %j, error: %s', name, result, err);
-        });
+        },
+      };
+      try {
+        var result = yield Module.add(nextMod);
+        that.log('  [%s] save next module, %j', name, result);
+      } catch (err) {
+        that.log('  [%s] save next module error %s', err.message);
       }
     }
-    var versions = [];
-    for (var i = 0; i < versionNames.length; i++) {
-      var v = versionNames[i];
-      var exists = map[v] || {};
-      var version = pkg.versions[v];
-      if (!version || !version.dist) {
+  }
+
+  var versions = [];
+  for (var i = 0; i < versionNames.length; i++) {
+    var v = versionNames[i];
+    var exists = map[v] || {};
+    var version = pkg.versions[v];
+    if (!version || !version.dist || !version.dist.tarball) {
+      continue;
+    }
+    //patch for readme
+    if (!version.readme) {
+      version.readme = pkg.readme;
+    }
+    var publish_time = times[v];
+    version.publish_time = publish_time ? Date.parse(publish_time) : null;
+    if (!version.maintainers || !version.maintainers[0]) {
+      version.maintainers = pkg.maintainers;
+    }
+    var sourceAuthor = version.maintainers && version.maintainers[0] &&
+      version.maintainers[0].name || exists.author;
+
+    if (exists.package && exists.package.dist.shasum === version.dist.shasum &&
+      exists.author === sourceAuthor) {
+      // * author make sure equal
+      // * shasum make sure equal
+      if ((version.publish_time === exists.publish_time) ||
+          (!version.publish_time && exists.publish_time)) {
+        // debug('  [%s] %s publish_time equal: %s, %s',
+        //   name, version.version, version.publish_time, exists.publish_time);
+        // * publish_time make sure equal
+        if (exists.description === null && version.description) {
+          // * make sure description exists
+          missingDescriptions.push({
+            id: exists.id,
+            description: version.description
+          });
+        }
+
+        if (!exists.package.readme && version.readme) {
+          // * make sure readme exists
+          missingReadmes.push({
+            id: exists.id,
+            readme: version.readme
+          });
+        }
         continue;
       }
-      //patch for readme
-      if (!version.readme) {
-        version.readme = pkg.readme;
-      }
-      var publish_time = times[v];
-      version.publish_time = publish_time ? Date.parse(publish_time) : null;
-      if (!version.maintainers || !version.maintainers[0]) {
-        version.maintainers = pkg.maintainers;
-      }
-      var sourceAuthor = version.maintainers && version.maintainers[0] && version.maintainers[0].name || exists.author;
-
-      if (exists.package && exists.package.dist.shasum === version.dist.shasum && exists.author === sourceAuthor) {
-        // * author make sure equal
-        // * shasum make sure equal
-        if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
-          debug('  [%s] %s publish_time equal: %s, %s',
-            name, version.version, version.publish_time, exists.publish_time);
-          // * publish_time make sure equal
-          if (exists.description === null && version.description) {
-            // * make sure description exists
-            missingDescriptions.push({
-              id: exists.id,
-              description: version.description
-            });
-          }
-
-          if (!exists.package.readme && version.readme) {
-            // * make sure readme exists
-            missingReadmes.push({
-              id: exists.id,
-              readme: version.readme
-            });
-          }
-          continue;
-        }
-      }
-      versions.push(version);
     }
+    versions.push(version);
+  }
 
-    var sourceTags = pkg['dist-tags'] || {};
-    for (var t in sourceTags) {
-      var sourceTagVersion = sourceTags[t];
-      if (sourceTagVersion && tags[t] !== sourceTagVersion) {
-        missingTags.push([t, sourceTagVersion]);
-      }
-    }
-
-    if (versions.length === 0) {
-      that.log('  [%s] all versions are exists', name);
-      return ep.emit('syncDone');
+  var sourceTags = pkg['dist-tags'] || {};
+  for (var t in sourceTags) {
+    var sourceTagVersion = sourceTags[t];
+    if (sourceTagVersion && tags[t] !== sourceTagVersion) {
+      missingTags.push([t, sourceTagVersion]);
     }
+  }
 
+  if (versions.length === 0) {
+    that.log('  [%s] all versions are exists', name);
+  } else {
     versions.sort(function (a, b) {
       return a.publish_time - b.publish_time;
     });
-    missingVersions = versions;
-    that.log('  [%s] %d versions', name, versions.length);
-    ep.emit('syncModule', missingVersions.shift());
-  });
+    that.log('  [%s] %d versions need to sync', name, versions.length);
+  }
 
+  missingVersions = versions;
   var versionNames = [];
   var syncIndex = 0;
-  ep.on('syncModule', function (syncModule) {
+
+  // sync missing versions
+  while (missingVersions.length) {
     var index = syncIndex++;
-    that._syncOneVersion(index, syncModule, function (err, result) {
-      if (err) {
-        that.log('    [%s:%d] error, version: %s, %s: %s',
-          syncModule.name, index, syncModule.version, err.name, err.message);
-      } else {
-        versionNames.push(syncModule.version);
-      }
+    var syncModule = missingVersions.shift();
+    if (!syncModule.dist.tarball) {
+      continue;
+    }
+    try {
+      var result = yield that._syncOneVersion(index, syncModule);
+      versionNames.push(syncModule.version);
+    } catch (err) {
+      that.log('    [%s:%d] error, version: %s, %s: %s',
+        syncModule.name, index, syncModule.version, err.name, err.message);
+    }
+  }
 
-      var nextVersion = missingVersions.shift();
-      if (!nextVersion) {
-        return ep.emit('syncDone', result);
-      }
-      ep.emit('syncModule', nextVersion);
-    });
-  });
-
-  ep.on('syncDone', function () {
+  // sync missing descriptions
+  function *syncDes() {
     if (missingDescriptions.length === 0) {
-      return ep.emit('descriptionDone');
+      return;
     }
-
     that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
-    missingDescriptions.forEach(function (item) {
-      Module.updateDescription(item.id, item.description, function (err, result) {
-        if (err) {
-          that.log('    save error, id： %s, description: %s, error: %s', item.id, item.description, err);
-        } else {
-          that.log('    saved, id: %s, description length: %d', item.id, item.description.length);
-        }
-        ep.emitLater('saveDescription');
-      });
-    });
+    var res = yield gather(missingDescriptions.map(function (item) {
+      return Module.updateDescription(item.id, item.description);
+    }));
 
-    ep.after('saveDescription', missingDescriptions.length, function () {
-      ep.emit('descriptionDone');
-    });
-  });
-
-  ep.on('syncDone', function () {
-    if (missingTags.length === 0) {
-      return ep.emit('tagDone');
+    for (var i = 0; i < res.length; i++) {
+      var item = missingDescriptions[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, description: %s, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    saved, id: %s, description length: %d',
+          item.id, item.description.length);
+      }
     }
+  }
 
+  // sync missing tags
+  function *syncTag() {
+    if (missingTags.length === 0) {
+      return;
+    }
     that.log('  [%s] adding %d tags', name, missingTags.length);
     // sync tags
-    missingTags.forEach(function (item) {
-      Module.addTag(name, item[0], item[1], ep.done(function (result) {
-        that.log('    added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
-        ep.emitLater('addTag');
-      }));
-    });
 
-    ep.after('addTag', missingTags.length, function () {
-      ep.emit('tagDone');
-    });
-  });
+    var res = yield gather(missingTags.map(function (item) {
+      return Module.addTag(name, item[0], item[1]);
+    }));
 
-  ep.once('syncDone', function () {
+    for (var i = 0; i < res.length; i++) {
+      var item = missingTags[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    add tag %s:%s error, error: %s',
+          item.id, item.description, r.error.message);
+      } else {
+        that.log('    added tag %s:%s, module_id: %s',
+          item[0], item[1], r.value && r.value.module_id);
+      }
+    }
+  }
+
+  // sycn missing readme
+  function *syncReadme() {
     if (missingReadmes.length === 0) {
-      return ep.emit('readmeDone');
+      return;
+    }
+    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
+
+    var res = yield gather(missingReadmes.map(function (item) {
+      return Module.updateReadme(item.id, item.readme);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var item = missingReadmes[i];
+      var r = res[i];
+      if (r.error) {
+        that.log('    save error, id: %s, error: %s', item.id, r.error.message);
+      } else {
+        that.log('    saved, id: %s', item.id);
+      }
+    }
+  }
+
+  function *syncMissingUsers() {
+    var missingUsers = [];
+    var names = Object.keys(npmUsernames);
+    if (names.length === 0) {
+      return;
+    }
+    var rows = yield *User.listByNames(names);
+    var map = {};
+    rows.forEach(function (r) {
+      map[r.name] = r;
+    });
+    names.forEach(function (username) {
+      var r = map[username];
+      if (!r || !r.json) {
+        missingUsers.push(username);
+      }
+    });
+
+    if (missingUsers.length === 0) {
+      that.log('  [%s] all %d npm users exists', name, names.length);
+      return;
     }
 
-    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
-    missingReadmes.forEach(function (item) {
-      Module.updateReadme(item.id, item.readme, function (err, result) {
-        if (err) {
-          that.log('    save error, id: %s, error: %s', item.id, err);
-        } else {
-          that.log('    saved, id: %s', item.id);
-        }
-        ep.emitLater('saveReadme');
-      });
-    });
+    that.log('  [%s] saving %d/%d missing npm users: %j',
+      name, missingUsers.length, names.length, missingUsers);
+    var res = yield gather(missingUsers.map(function (username) {
+      return _saveNpmUser(username);
+    }));
 
-    ep.after('saveReadme', missingReadmes.length, function () {
-      ep.emit('readmeDone');
-    });
-  });
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    save npm user error, %s', r.error.message);
+      }
+    }
+  }
 
-  ep.all('tagDone', 'descriptionDone', 'readmeDone', function () {
-    // TODO: set latest version
-    callback(null, versionNames);
-  });
+  // sync missing star users
+  function *syncMissingStarUsers() {
+    if (missingStarUsers.length === 0) {
+      return;
+    }
+
+    that.log('  [%s] saving %d star users', name, missingStarUsers.length);
+    var res = yield gather(missingStarUsers.map(function (username) {
+      return _addStar(name, username);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    add star user error, %s', r.error.message);
+      }
+    }
+  }
+
+  yield [syncDes(), syncTag(), syncReadme(), syncMissingStarUsers(), syncMissingUsers()];
+  return versionNames;
 };
 
-SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePackage, callback) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
   var that = this;
   var username = this.username;
   var downurl = sourcePackage.dist.tarball;
   var filename = path.basename(downurl);
   var filepath = common.getTarballFilepath(filename);
   var ws = fs.createWriteStream(filepath);
+
   var options = {
     writeStream: ws,
     followRedirect: true,
     timeout: 600000, // 10 minutes download
+    headers: {
+      'user-agent': USER_AGENT
+    }
   };
-  var ep = eventproxy.create();
-  ep.fail(function (err) {
-    // remove tmp file whatever
-    fs.unlink(filepath, utility.noop);
-    callback(err);
-  });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s',
+  var dependencies = Object.keys(sourcePackage.dependencies || {});
+  var devDependencies = Object.keys(sourcePackage.devDependencies || {});
+
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, ' +
+   'publish on cnpm: %s, dependencies: %d, devDependencies: %d',
     sourcePackage.name, versionIndex, sourcePackage.version,
-    sourcePackage.dist, that.noDep, that._publish);
+    sourcePackage.dist, that.noDep, that._publish,
+    dependencies.length, devDependencies.length);
+
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
+  }
+
+  if (devDependencies.length > config.maxDependencies) {
+    devDependencies = devDependencies.slice(0, config.maxDependencies);
+  }
+
   if (!that.noDep) {
-    for (var k in sourcePackage.dependencies) {
-      that.add(k);
+    for (var i = 0; i < dependencies.length; i++) {
+      that.add(dependencies[i]);
     }
 
-    for (var k in sourcePackage.devDependencies) {
-      that.add(k);
+    for (var i = 0; i < devDependencies.length; i++) {
+      that.add(devDependencies[i]);
     }
   }
 
+  // add module dependence
+  try {
+    yield dependencies.map(function (depName) {
+      return ModuleDeps.add(depName, sourcePackage.name);
+    });
+  } catch (err) {
+    // ignore
+  }
+
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
-  urllib.request(downurl, options, ep.done(function (_, response) {
-    var statusCode = response && response.statusCode || -1;
+
+  try {
+    // get tarball
+    var r = yield *urllib.request(downurl, options);
+    var statusCode = r.status || -1;
     if (statusCode === 404) {
-      // just copy source dist
       shasum = sourcePackage.dist.shasum;
-      return ep.emit('uploadResult', {
+      return yield afterUpload({
         url: downurl
       });
     }
-
     if (statusCode !== 200) {
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
       err.data = sourcePackage;
-      return ep.emit('error', err);
+      throw err;
     }
 
+    // read and check
     var rs = fs.createReadStream(filepath);
-    rs.once('error', function (err) {
-      ep.emit('error', err);
-    });
     rs.on('data', function (data) {
       shasum.update(data);
       dataSize += data.length;
     });
-    rs.on('end', function () {
-      shasum = shasum.digest('hex');
-      if (shasum !== sourcePackage.dist.shasum) {
-        var err = new Error('Download ' + downurl + ' shasum:' + shasum + ' not match ' + sourcePackage.dist.shasum);
-        err.name = 'DownloadTarballShasumError';
-        err.data = sourcePackage;
-        return ep.emit('error', err);
-      }
+    var end = thunkify.event(rs);
+    yield end(); // after end event emit
 
-      var options = {
-        key: common.getCDNKey(sourcePackage.name, filename),
-        size: dataSize,
-        shasum: shasum
-      };
-      nfs.upload(filepath, options, ep.done('uploadResult'));
-    });
-  }));
+    // check shasum
+    shasum = shasum.digest('hex');
+    if (shasum !== sourcePackage.dist.shasum) {
+      var err = new Error('Download ' + downurl + ' shasum:' + shasum +
+        ' not match ' + sourcePackage.dist.shasum);
+      err.name = 'DownloadTarballShasumError';
+      err.data = sourcePackage;
+      throw err;
+    }
 
-  ep.on('uploadResult', function (result) {
+    options = {
+      key: common.getCDNKey(sourcePackage.name, filename),
+      size: dataSize,
+      shasum: shasum
+    };
+    // upload to NFS
+    var result = yield nfs.upload(filepath, options);
+    return yield afterUpload(result);
+  } finally {
     // remove tmp file whatever
     fs.unlink(filepath, utility.noop);
+  }
 
+  function *afterUpload(result) {
     //make sure sync module have the correct author info
     //only if can not get maintainers, use the username
     var author = username;
@@ -486,6 +655,8 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
       publish_time: sourcePackage.publish_time,
     };
 
+    // delete _publish_on_cnpm, because other cnpm maybe sync from current cnpm
+    delete mod.package._publish_on_cnpm;
     if (that._publish) {
       // sync as publish
       mod.package._publish_on_cnpm = true;
@@ -505,52 +676,41 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     }
 
     mod.package.dist = dist;
-    Module.add(mod, ep.done(function (result) {
-      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j, publish on cnpm: %s',
-        sourcePackage.name, versionIndex,
-        result.id,
-        author, mod.version, dataSize,
-        new Date(mod.publish_time),
-        that._publish);
-      callback(null, result);
-    }));
-  });
+    var r = yield Module.add(mod);
+
+    that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, ' +
+    'size: %d, publish_time: %j, publish on cnpm: %s',
+    sourcePackage.name, versionIndex,
+    r.id,
+    author, mod.version, dataSize,
+    new Date(mod.publish_time),
+    that._publish);
+
+    return r;
+  }
 };
 
-SyncModuleWorker.sync = function (name, username, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = null;
-  }
+SyncModuleWorker.sync = function *(name, username, options) {
   options = options || {};
-  npm.get(name, function (err, pkg, response) {
-    if (err) {
-      return callback(err);
-    }
-    if (!pkg || !pkg._rev) {
-      return callback(null, {
-        ok: false,
-        statusCode: response.statusCode,
-        pkg: pkg
-      });
-    }
-    Log.create({name: name, username: username}, function (err, result) {
-      if (err) {
-        return callback(err);
-      }
-      var worker = new SyncModuleWorker({
-        logId: result.id,
-        name: name,
-        username: username,
-        noDep: options.noDep,
-        publish: options.publish,
-      });
-      worker.start();
-      callback(null, {
-        ok: true,
-        logId: result.id,
-        pkg: pkg
-      });
-    });
+  var pkg = yield npm.get(name);
+  if (!pkg || !pkg._rev) {
+    return {
+      ok: false,
+      pkg: pkg
+    };
+  }
+  var result = yield Log.create({name: name, username: username});
+  var worker = new SyncModuleWorker({
+    logId: result.id,
+    name: name,
+    username: username,
+    noDep: options.noDep,
+    publish: options.publish,
   });
+  worker.start();
+  return {
+    ok: true,
+    logId: result.id,
+    pkg: pkg
+  };
 };

proxy/total.js

@@ -14,16 +14,26 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
+var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var eventproxy = require('eventproxy');
+var multiline = require('multiline');
 
-var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
-  FROM information_schema.tables \
-  WHERE TABLE_SCHEMA = ? \
-  GROUP BY TABLE_NAME \
-  ORDER BY data_length DESC \
-  LIMIT 0 , 200';
+var DB_SIZE_SQL = multiline(function () {;/*
+  SELECT
+    TABLE_NAME AS name, data_length, index_length
+  FROM
+    information_schema.tables
+  WHERE
+    TABLE_SCHEMA = ?
+  GROUP BY
+    TABLE_NAME
+  ORDER BY
+    data_length DESC
+  LIMIT
+    0, 200;
+*/});
 
 var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM module;';
 var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM module;';
@@ -83,7 +93,14 @@ exports.get = function (callback) {
   });
 };
 
-var PLUS_DELETE_MODULE_SQL = 'UPDATE total SET module_delete=module_delete+1 WHERE name="total";';
+var PLUS_DELETE_MODULE_SQL = multiline(function () {;/*
+  UPDATE
+    total
+  SET
+    module_delete=module_delete+1
+  WHERE
+    name="total";
+*/});
 exports.plusDeleteModule = function (callback) {
   mysql.query(PLUS_DELETE_MODULE_SQL, callback);
 };
@@ -92,24 +109,55 @@ exports.getTotalInfo = function (callback) {
   mysql.queryOne(TOTAL_INFO_SQL, callback);
 };
 
-var SET_LAST_SYNC_TIME_SQL = 'UPDATE total SET last_sync_time=? WHERE name="total";';
+var SET_LAST_SYNC_TIME_SQL = multiline(function () {;/*
+  UPDATE
+    total
+  SET
+    last_sync_time=?
+  WHERE
+    name="total";
+*/});
 exports.setLastSyncTime = function (time, callback) {
   mysql.query(SET_LAST_SYNC_TIME_SQL, Number(time), callback);
 };
 
-var SET_LAST_EXIST_SYNC_TIME_SQL = 'UPDATE total SET last_exist_sync_time=? WHERE name="total";';
+var SET_LAST_EXIST_SYNC_TIME_SQL = multiline(function () {;/*
+  UPDATE
+    total
+  SET
+    last_exist_sync_time=?
+  WHERE
+    name="total";
+*/});
 exports.setLastExistSyncTime = function (time, callback) {
   mysql.query(SET_LAST_EXIST_SYNC_TIME_SQL, Number(time), callback);
 };
 
-var UPDATE_SYNC_STATUS_SQL = 'UPDATE total SET sync_status = ? WHERE name="total";';
+var UPDATE_SYNC_STATUS_SQL = multiline(function () {;/*
+  UPDATE
+    total
+  SET
+    sync_status=?
+  WHERE
+    name="total";
+*/});
 exports.updateSyncStatus = function (status, callback) {
   mysql.query(UPDATE_SYNC_STATUS_SQL, [status], callback);
 };
 
-var UPDATE_SYNC_NUM_SQL = 'UPDATE total SET sync_status = ?, need_sync_num = ?, \
-  success_sync_num = ?, fail_sync_num = ?, left_sync_num = ?, last_sync_module=? \
-  WHERE name="total";';
+var UPDATE_SYNC_NUM_SQL = multiline(function () {;/*
+  UPDATE
+    total
+  SET
+    sync_status = ?,
+    need_sync_num = ?,
+    success_sync_num = ?,
+    fail_sync_num = ?,
+    left_sync_num = ?,
+    last_sync_module = ?
+  WHERE
+    name="total";
+*/});
 exports.updateSyncNum = function (params, callback) {
   var query = [
     params.syncStatus, params.need || 0,
@@ -118,3 +166,5 @@ exports.updateSyncNum = function (params, callback) {
   ];
   mysql.query(UPDATE_SYNC_NUM_SQL, query, callback);
 };
+
+thunkify(exports);

proxy/user.js

@@ -14,36 +14,43 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var crypto = require('crypto');
-
-var COLUMNS = 'id, rev, name, email, salt, password_sha, ip, roles, gmt_create, gmt_modified';
-var SELECT_USER_SQL = 'SELECT ' + COLUMNS + ' FROM user WHERE name=?;';
-
-function sha1(s) {
-  return crypto.createHash("sha1").update(s).digest("hex");
-}
-
-function passwordSha(password, salt) {
-  return sha1(password + salt);
-}
-exports.passwordSha = passwordSha;
+var multiline = require('multiline');
 
+var SELECT_USER_SQL = multiline(function () {;/*
+  SELECT
+    id, rev, name, email, salt, password_sha, ip,
+    roles, json, npm_user, gmt_create, gmt_modified
+  FROM
+    user
+  WHERE
+    name=?;
+*/});
 exports.get = function (name, callback) {
   mysql.queryOne(SELECT_USER_SQL, [name], function (err, row) {
     if (row) {
       try {
-        row.roles = JSON.parse(row.roles);
+        row.roles = row.roles ? JSON.parse(row.roles) : [];
       } catch (e) {
         row.roles = [];
       }
+      try {
+        row.json = row.json ? JSON.parse(row.json) : null;
+      } catch (e) {
+        row.json = null;
+      }
     }
     callback(err, row);
   });
 };
 
+function passwordSha(password, salt) {
+  return utility.sha1(password + salt);
+}
+
 exports.auth = function (name, password, callback) {
   exports.get(name, function (err, row) {
     if (err || !row) {
@@ -58,9 +65,14 @@ exports.auth = function (name, password, callback) {
   });
 };
 
-var INSERT_USER_SQL = 'INSERT INTO user(rev, name, email, salt, password_sha, ip, roles, gmt_create, gmt_modified) \
-  VALUES(?, ?, ?, ?, ?, ?, ?, now(), now())';
 
+var INSERT_USER_SQL = multiline(function () {;/*
+  INSERT INTO
+    user(rev, name, email, salt, password_sha,
+    ip, roles, gmt_create, gmt_modified)
+  VALUES
+    (?, ?, ?, ?, ?, ?, ?, now(), now());
+*/});
 exports.add = function (user, callback) {
   var roles = user.roles || [];
   try {
@@ -75,9 +87,20 @@ exports.add = function (user, callback) {
   });
 };
 
-var UPDATE_USER_SQL = 'UPDATE user SET rev=?, email=?, salt=?, password_sha=?, ip=?, roles=?, gmt_modified=now() \
-  WHERE name=? AND rev=?;';
-
+var UPDATE_USER_SQL = multiline(function () {;/*
+  UPDATE
+    user
+  SET
+    rev=?,
+    email=?,
+    salt=?,
+    password_sha=?,
+    ip=?,
+    roles=?,
+    gmt_modified=now()
+  WHERE
+    name=? AND rev=?;
+*/});
 exports.update = function (user, callback) {
   var rev = user.rev || user._rev;
   var revNo = Number(rev.split('-', 1));
@@ -98,10 +121,41 @@ exports.update = function (user, callback) {
 
   var values = [newRev, user.email, user.salt, user.password_sha, user.ip, roles, user.name, rev];
   mysql.query(UPDATE_USER_SQL, values, function (err, data) {
-    if (err || !data.affectedRows) {
+    if (err) {
       return callback(err);
     }
-    callback(null, {rev: newRev});
+    callback(null, {rev: newRev, result: data});
   });
 };
 
+thunkify(exports);
+
+exports.passwordSha = passwordSha;
+
+exports.saveNpmUser = function *(user) {
+  var sql = 'SELECT id, json FROM user WHERE name=?;';
+  var row = yield mysql.queryOne(sql, [user.name]);
+  if (!row) {
+    sql = 'INSERT INTO user(npm_user, json, rev, name, email, salt, password_sha, ip, gmt_create, gmt_modified) \
+      VALUES(1, ?, ?, ?, ?, "0", "0", "0", now(), now());';
+    yield mysql.query(sql, [JSON.stringify(user), user._rev, user.name, user.email]);
+  } else {
+    sql = 'UPDATE user SET json=?, rev=? WHERE id=?;';
+    yield mysql.query(sql, [JSON.stringify(user), user._rev, row.id]);
+  }
+};
+
+var LIST_BY_NAMES_SQL = multiline(function () {;/*
+  SELECT
+    id, name, email, json
+  FROM
+    user
+  WHERE
+    name in (?);
+*/});
+exports.listByNames = function *(names) {
+  if (names.length === 0) {
+    return [];
+  }
+  return yield mysql.query(LIST_BY_NAMES_SQL, [names]);
+};

routes/registry.js

@@ -15,6 +15,8 @@
  * Module dependencies.
  */
 
+var middlewares = require('koa-middlewares');
+var limit = require('../middleware/limit');
 var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
@@ -24,7 +26,7 @@ var user = require('../controllers/registry/user');
 var sync = require('../controllers/sync');
 
 function routes(app) {
-  app.get('/', total.show);
+  app.get('/', middlewares.jsonp(), total.show);
 
   //before /:name/:version
   //get all modules, for npm search
@@ -34,36 +36,36 @@ function routes(app) {
   app.get('/-/short', mod.listAllModuleNames);
 
   // module
-  app.get('/:name', [syncByInstall], mod.show);
-  app.get('/:name/:version', [syncByInstall], mod.get);
+  app.get('/:name', syncByInstall, mod.show);
+  app.get('/:name/:version', syncByInstall, mod.get);
   // try to add module
-  app.put('/:name', [login, publishable], mod.add);
+  app.put('/:name', login, publishable, mod.add);
 
   // sync from source npm
   app.put('/:name/sync', sync.sync);
   app.get('/:name/sync/log/:id', sync.getSyncLog);
 
-  app.get('/:name/download/:filename', mod.download);
+  // need limit by ip
+  app.get('/:name/download/:filename', limit, mod.download);
 
   // put tarball
   // https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
-  app.put('/:name/-/:filename/-rev/:rev', [login, publishable], mod.upload);
+  app.put('/:name/-/:filename/-rev/:rev', login, publishable, mod.upload);
   // delete tarball
-  app.delete('/:name/download/:filename/-rev/:rev', [login, publishable], mod.removeTar);
+  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
 
   // put package.json to module
-  app.put('/:name/:version/-tag/latest', [login, publishable], mod.updateLatest);
+  app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
 
   // update module, unpublish will PUT this
-  app.put('/:name/-rev/:rev', [login, publishable], mod.removeWithVersions);
-  app.delete('/:name/-rev/:rev', [login, publishable], mod.removeAll);
+  app.put('/:name/-rev/:rev', login, publishable, mod.updateOrRemove);
+  app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
 
   // try to create a new user
   // https://registry.npmjs.org/-/user/org.couchdb.user:fengmk2
   app.put('/-/user/org.couchdb.user::name', user.add);
   app.get('/-/user/org.couchdb.user::name', user.show);
-  app.put('/-/user/org.couchdb.user::name/-rev/:rev', [login], user.update);
-
+  app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, user.update);
   // _session
   app.post('/_session', user.authSession);
 }

routes/web.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - routes/web.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -19,6 +19,7 @@ var pkg = require('../controllers/web/package');
 var user = require('../controllers/web/user');
 var sync = require('../controllers/sync');
 var total = require('../controllers/total');
+var dist = require('../controllers/web/dist');
 
 function routes(app) {
   app.get('/total', total.show);
@@ -34,6 +35,8 @@ function routes(app) {
   app.get('/sync', pkg.displaySync);
 
   app.get('/_list/search/search', pkg.rangeSearch);
+
+  app.get(/^\/dist(\/.+)?/, dist.redirect);
 }
 
 module.exports = routes;

servers/registry.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - servers/registry.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -15,69 +15,54 @@
  * Module dependencies.
  */
 
-require('response-patch');
+var koa = require('koa');
+var app = module.exports = koa();
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var responseCookie = require('response-cookie');
-var urlrouter = require('urlrouter');
-var forward = require('forward');
-var path = require('path');
+var microtime = require('microtime');
+var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
+var notFound = require('../middleware/registry_not_found');
 
-var rootdir = path.dirname(__dirname);
-var app = connect();
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
+app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
 
-app.use('/favicon.ico', forward(path.join(rootdir, 'public', 'favicon.png')));
-
-app.use('/dist', connect.static(config.uploadDir));
-
-app.use(responseCookie());
-app.use(connect.cookieParser());
-app.use(connect.query());
-app.use(connect.json());
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
+app.proxy = true;
 app.use(session);
+app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.compress({threshold: 150}));
+app.use(middlewares.conditional());
+app.use(middlewares.etag());
 
 /**
  * Routes
  */
 
-app.use(urlrouter(routes));
-
-app.use(function (req, res, next) {
-  res.json(404, {error: 'not_found', reason: 'document not found'});
-});
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.json(500, {
-    error: err.name,
-    reason: err.message
-  });
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.registryPort);
+}
 
 module.exports = app;

servers/web.js

@@ -15,37 +15,43 @@
  * Module dependencies.
  */
 
-require('response-patch');
 var path = require('path');
 var http = require('http');
 var fs = require('fs');
-var connect = require('connect');
-var rt = require('connect-rt');
-var urlrouter = require('urlrouter');
-var connectMarkdown = require('connect-markdown');
+var microtime = require('microtime');
+var koa = require('koa');
+var middlewares = require('koa-middlewares');
+var markdown = require('koa-markdown');
+var session = require('../common/session');
+var opensearch = require('../middleware/opensearch');
+var notFound = require('../middleware/web_not_found');
+var auth = require('../middleware/auth');
 var routes = require('../routes/web');
 var logger = require('../common/logger');
 var config = require('../config');
-var session = require('../common/session');
-var render = require('connect-render');
-var opensearch = require('../middleware/opensearch');
-var app = connect();
+
+var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
-
-app.use('/public', connect.static(path.join(rootdir, 'public')));
-app.use('/opensearch.xml', opensearch);
-
-app.use(connect.cookieParser());
+app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
+  buffer: !config.debug,
+  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+  dir: path.join(rootdir, 'public')
+}));
+app.use(opensearch);
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
+app.proxy = true;
 app.use(session);
-app.use(connect.query());
-app.use(connect.json());
+app.use(middlewares.bodyParser());
+app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.compress({threshold: 150}));
+app.use(middlewares.conditional());
+app.use(middlewares.etag());
 
 var viewDir = path.join(rootdir, 'view', 'web');
 var docDir = path.join(rootdir, 'docs', 'web');
@@ -57,7 +63,8 @@ var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
   .replace('{{logoURL}}', config.logoURL);
 fs.writeFileSync(layoutFile, layout);
 
-app.use('/', connectMarkdown({
+app.use(markdown({
+  baseUrl: '/',
   root: docDir,
   layout: layoutFile,
   titleHolder: '<%- locals.title %>',
@@ -65,41 +72,38 @@ app.use('/', connectMarkdown({
   indexName: 'readme'
 }));
 
-var helpers = {
+var locals = {
   config: config
 };
 
-app.use(render({
+middlewares.render(app, {
   root: viewDir,
-  viewExt: '.html',
+  viewExt: 'html',
   layout: '_layout',
   cache: config.viewCache,
-  helpers: helpers
-}));
+  debug: config.debug,
+  locals: locals
+});
 
 /**
  * Routes
  */
-
-app.use(urlrouter(routes));
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.statusCode = 500;
-  res.send('Server has some problems. :(');
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.webPort);
+}
 
 module.exports = app;

sync/index.js

@@ -22,8 +22,9 @@ var utility = require('utility');
 var debug = require('debug')('cnpmjs.org:sync:index');
 var Total = require('../proxy/total');
 var logger = require('../common/logger');
+var co = require('co');
 
-var sync;
+var sync = null;
 
 switch (config.syncModel) {
 case 'all':
@@ -34,28 +35,41 @@ case 'exist':
   break;
 }
 
+if (!sync && config.enableCluster) {
+  console.log('[%s] [sync_worker:%s] no need to sync, exit now', Date(), process.pid);
+  process.exit(0);
+}
+
+console.log('[%s] [sync_worker:%s] syncing with %s mode',
+  Date(), process.pid, config.syncModel);
+
 //set sync_status = 0 at first
 Total.updateSyncStatus(0, utility.noop);
 
 // the same time only sync once
 var syncing = false;
 
-function handleSync() {
+var handleSync = co(function *() {
   debug('mode: %s, syncing: %s', config.syncModel, syncing);
-  // check sync every one 30 minutes
   if (!syncing) {
     syncing = true;
     debug('start syncing');
-    sync(function (err, data) {
-      if (config.debug) {
-        console.log(err, data);
-      } else {
-        sendMailToAdmin(err, data, new Date());
-      }
-      syncing = false;
-    });
+    var data;
+    var error;
+    try {
+      var data = yield *sync();
+    } catch (err) {
+      error = err;
+    }
+    if (config.debug) {
+      error && console.error(error.stack);
+      data && console.log(data);
+    } else {
+      sendMailToAdmin(error, data, new Date());
+    }
+    syncing = false;
   }
-}
+});
 
 if (sync) {
   handleSync();
@@ -85,7 +99,7 @@ function sendMailToAdmin(err, result, syncTime) {
       'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
       syncTime, result.fails.length, result.fails.slice(0, 10),
       result.successes.length, result.successes.slice(0, 10));
-  } else {
+  } else if (result.successes && result.successes.length) {
     subject = 'Sync Finished';
     type = 'log';
     html = util.format('Sync packages from official registry finished.\n' +
@@ -93,7 +107,7 @@ function sendMailToAdmin(err, result, syncTime) {
       syncTime, result.successes.length, result.successes.slice(0, 10));
   }
   debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
-  if (type !== 'log') {
+  if (type && type !== 'log') {
     mail[type](to, subject, html, function (err) {
       if (err) {
         logger.info('send email with type: %s, subject: %s, html: %s', type, subject, html);

sync/status.js

@@ -19,18 +19,20 @@ var utility = require('utility');
 var Total = require('../proxy/total');
 
 function Status(options) {
-  this.worker = options.worker;
   this.need = options.need;
   this.lastSyncModule = '';
+  this.successes = 0;
+  this.fails = 0;
+  this.left = options.need;
 }
 
 Status.prototype.log = function (syncDone) {
   var params = {
     syncStatus: syncDone ? 0 : 1,
     need: this.need,
-    success: this.worker.successes.length,
-    fail: this.worker.fails.length,
-    left: this.worker.names.length,
+    success: this.successes,
+    fail: this.fails,
+    left: this.left,
     lastSyncModule: this.lastSyncModule,
   };
   Total.updateSyncNum(params, utility.noop);
@@ -43,19 +45,30 @@ Status.prototype.start = function () {
   this.started = true;
   //every 30s log it into mysql
   this.timer = setInterval(this.log.bind(this), 30000);
-  this.worker.on('success', function (moduleName) {
-    debug('sync [%s] success', moduleName);
-    this.lastSyncModule = moduleName;
-  }.bind(this));
-  this.worker.on('end', function () {
-    this.started = false;
-    this.log(true);
-    clearInterval(this.timer);
-  }.bind(this));
 };
 
-Status.init = function (options) {
-  return new Status(options);
+Status.init = function (options, worker) {
+  var status = new Status(options);
+  status.start();
+  worker.on('success', function (moduleName) {
+    debug('sync [%s] success', moduleName);
+    status.lastSyncModule = moduleName;
+    status.successes++;
+    status.left--;
+  });
+  worker.on('fail', function () {
+    status.fails++;
+    status.left--;
+  });
+  worker.on('add', function () {
+    status.left++;
+  });
+
+  worker.on('end', function () {
+    status.started = false;
+    status.log(true);
+    clearInterval(status.timer);
+  });
 };
 
 module.exports = Status;

sync/sync_all.js

@@ -24,6 +24,8 @@ var Npm = require('../proxy/npm');
 var Total = require('../proxy/total');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var Module = require('../proxy/module');
+var co = require('co');
+var thunkify = require('thunkify-wrap');
 
 function subtract(subtracter, minuend) {
   subtracter = subtracter || [];
@@ -55,109 +57,87 @@ function union(arrOne, arrTwo) {
  * when sync from official at the first time
  * get all packages by short and restart from last synced module
  * @param {String} lastSyncModule
- * @param {Function} callback
  */
-function getFirstSyncPackages(lastSyncModule, callback) {
-  Npm.getShort(function (err, pkgs) {
-    if (err || !lastSyncModule) {
-      return callback(err, pkgs);
-    }
-    // start from last success
-    var lastIndex = pkgs.indexOf(lastSyncModule);
-    if (lastIndex > 0) {
-      pkgs = pkgs.slice(lastIndex);
-    }
-    return callback(null, pkgs);
-  });
+function *getFirstSyncPackages(lastSyncModule) {
+  var pkgs = yield Npm.getShort();
+  if (!lastSyncModule) {
+    return pkgs;
+  }
+  // start from last success
+  var lastIndex = pkgs.indexOf(lastSyncModule);
+  if (lastIndex > 0) {
+    return pkgs.slice(lastIndex);
+  }
 }
 
 /**
  * get all the packages that update time > lastSyncTime
  * @param {Number} lastSyncTime
- * @param {Function} callback
  */
-function getCommonSyncPackages(lastSyncTime, callback) {
-  Npm.getAllSince(lastSyncTime, function (err, data) {
-    if (err || !data) {
-      return callback(err, []);
-    }
-    delete data._updated;
-    return callback(null, Object.keys(data));
-  });
+function *getCommonSyncPackages(lastSyncTime) {
+  var data = yield Npm.getAllSince(lastSyncTime);
+  if (!data) {
+    return [];
+  }
+  delete data._updated;
+  return Object.keys(data);
 }
 
 /**
  * get all the missing packages
  * @param {Function} callback
  */
-function getMissPackages(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
-  Npm.getShort(ep.doneLater('allPackages'));
-  Module.listAllModuleNames(ep.doneLater(function (rows) {
-    var existPackages = rows.map(function (row) {
-      return row.name;
-    });
-    ep.emit('existPackages', existPackages);
-  }));
-  ep.all('allPackages', 'existPackages', function (allPackages, existPackages) {
-    callback(null, subtract(allPackages, existPackages));
+function *getMissPackages(callback) {
+  var r = yield [Npm.getShort(), Module.listAllModuleNames];
+  var allPackages = r[0];
+  var existPackages = r[1].map(function (row) {
+    return row.name;
   });
+  return subtract(allPackages, existPackages);
 }
 
-//only sync not exist once
-var syncNotExist = true;
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
+  var info = yield Total.getTotalInfo();
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    debug('Last sync time %s', new Date(info.last_sync_time));
-    // TODO: 记录上次同步的最后一个模块名称
-    if (!info.last_sync_time) {
-      debug('First time sync all packages from official registry');
-      return getFirstSyncPackages(info.last_sync_module, ep.done('syncPackages'));
-    }
-    if (syncNotExist) {
-      getMissPackages(ep.done('missPackages'));
-      syncNotExist = false;
-    } else {
-      ep.emitLater('missPackages', []);
-    }
-    getCommonSyncPackages(info.last_sync_time - ms('10m'), ep.doneLater('newestPackages'));
-    ep.all('missPackages', 'newestPackages', function (missPackages, newestPackages) {
-      ep.emit('syncPackages', union(missPackages, newestPackages));
-    });
-  });
+  var packages;
+  debug('Last sync time %s', new Date(info.last_sync_time));
+  if (!info.last_sync_time) {
+    debug('First time sync all packages from official registry');
+    packages = yield getFirstSyncPackages(info.last_sync_module);
+  } else {
+    packages = yield getCommonSyncPackages(info.last_sync_time - ms('10m'));
+  }
 
-  ep.once('syncPackages', function (packages) {
-    packages = packages || [];
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      noDep: true,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      //only when all succss, set last sync time
-      !worker.fails.length && Total.setLastSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  packages = packages || [];
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    concurrency: config.syncConcurrency,
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  debug('All packages sync done, successes %d, fails %d',
+      worker.successes.length, worker.fails.length);
+  //only when all succss, set last sync time
+  if (!worker.fails.length) {
+    Total.setLastSyncTime(syncTime, utility.noop);
+  }
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

sync/sync_exist.js

@@ -24,6 +24,7 @@ var debug = require('debug')('cnpmjs.org:sync:sync_hot');
 var utility = require('utility');
 var Status = require('./status');
 var ms = require('ms');
+var thunkify = require('thunkify-wrap');
 
 function intersection(arrOne, arrTwo) {
   arrOne = arrOne || [];
@@ -39,71 +40,67 @@ function intersection(arrOne, arrTwo) {
   return results;
 }
 
-module.exports = function sync(callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
+module.exports = function *sync() {
   var syncTime = Date.now();
 
-  Module.listShort(ep.doneLater(function (packages) {
-    packages = packages.map(function (p) {
-      return p.name;
-    });
-    ep.emit('existPackages', packages);
-  }));
-  Total.getTotalInfo(ep.doneLater('totalInfo'));
-
-  ep.once('totalInfo', function (info) {
-    if (!info) {
-      return callback(new Error('can not found total info'));
-    }
-    if (!info.last_exist_sync_time) {
-      debug('First time sync all packages from official registry');
-      return Npm.getShort(ep.done(function (pkgs) {
-        if (!info.last_sync_module) {
-          return ep.emit('allPackages', pkgs);
-        }
-        // start from last success
-        var lastIndex = pkgs.indexOf(info.last_sync_module);
-        if (lastIndex > 0) {
-          pkgs = pkgs.slice(lastIndex);
-        }
-        ep.emit('allPackages', pkgs);
-      }));
-    }
-    Npm.getAllSince(info.last_exist_sync_time - ms('10m'), ep.done(function (data) {
-      if (!data) {
-        return ep.emit('allPackages', []);
-      }
-      if (data._updated) {
-        syncTime = data._updated;
-        delete data._updated;
-      }
-
-      return ep.emit('allPackages', Object.keys(data));
-    }));
+  var r = yield [Module.listShort(), Total.getTotalInfo()];
+  var existPackages = r[0].map(function (p) {
+    return p.name;
   });
+  var info = r[1];
+  if (!info) {
+    throw new Error('can not found total info');
+  }
 
-  ep.all('existPackages', 'allPackages', function (existPackages, allPackages) {
-    var packages = intersection(existPackages, allPackages);
-    debug('Total %d packages to sync', packages.length);
-    var worker = new SyncModuleWorker({
-      username: 'admin',
-      name: packages,
-      concurrency: config.syncConcurrency,
-    });
-    Status.init({
-      worker: worker,
-      need: packages.length
-    }).start();
-    worker.start();
-    worker.once('end', function () {
-      debug('All packages sync done, successes %d, fails %d',
-        worker.successes.length, worker.fails.length);
-      Total.setLastExistSyncTime(syncTime, utility.noop);
-      callback(null, {
-        successes: worker.successes,
-        fails: worker.fails
-      });
-    });
+  var allPackages;
+  if (!info.last_exist_sync_time) {
+    debug('First time sync all packages from official registry');
+    var pkgs = yield Npm.getShort();
+
+    if (info.last_sync_module) {
+      // start from last success
+      var lastIndex = pkgs.indexOf(info.last_sync_module);
+      if (lastIndex > 0) {
+        pkgs = pkgs.slice(lastIndex);
+      }
+    }
+    allPackages = pkgs;
+  } else {
+    debug('sync new module from last exist sync time: %s', info.last_sync_time);
+    var data = yield Npm.getAllSince(info.last_exist_sync_time - ms('10m'));
+    if (!data) {
+      allPackages = [];
+    }
+    if (data._updated) {
+      syncTime = data._updated;
+      delete data._updated;
+    }
+    allPackages = Object.keys(data);
+  }
+
+  var packages = intersection(existPackages, allPackages);
+  if (!packages.length) {
+    debug('no packages need be sync');
+    return;
+  }
+  debug('Total %d packages to sync', packages.length);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    concurrency: config.syncConcurrency
   });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  debug('All packages sync done, successes %d, fails %d',
+    worker.successes.length, worker.fails.length);
+
+  Total.setLastExistSyncTime(syncTime, utility.noop);
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
 };

test/controllers/registry/module.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/registry/module.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,6 +17,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var thunkify = require('thunkify-wrap');
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -25,19 +26,27 @@ var app = require('../../../servers/registry');
 var Module = require('../../../proxy/module');
 var Npm = require('../../../proxy/npm');
 var controller = require('../../../controllers/registry/module');
+var ModuleDeps = require('../../../proxy/module_deps');
 
 var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
 
 describe('controllers/registry/module.test.js', function () {
+  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
+  var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
+
   before(function (done) {
-    app.listen(0, done);
+    app.listen(0, function () {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(201, done);
+    });
   });
 
   afterEach(mm.restore);
 
-  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
-  var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
-
   describe('sync source npm package', function () {
     var logId;
     it('should put /:name/sync success', function (done) {
@@ -64,38 +73,89 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('GET /:name', function () {
-    it('should return module info', function (done) {
+  describe('GET /:name get module package info', function () {
+    var etag;
+
+    it('should return module info and etag', function (done) {
       request(app)
-      .get('/cnpmjs.org')
+      .get('/mk2testmodule')
+      .expect('content-type', 'application/json')
       .expect(200, function (err, res) {
         should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
+        etag.should.match(/^"\d{13}"$/);
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
-          'time', 'author', 'repository', '_attachments');
-        // res.body.author.should.eql({
-        //   "name": "fengmk2",
-        //   "email": "fengmk2@gmail.com",
-        //   "url": "http://fengmk2.github.com"
-        // });
-        res.body.name.should.equal('cnpmjs.org');
-        res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
+          'time', 'author',
+          // 'repository',
+          '_attachments',
+          'users',
+          // 'readmeFilename',
+          // 'homepage',
+          // 'bugs',
+          'license');
+        res.body.name.should.equal('mk2testmodule');
+        res.body.versions[Object.keys(res.body.versions)[0]]
+          .dist.tarball.should.include('/mk2testmodule/download');
+        res.body.time.should.have.property('modified');
+        res.body.time.modified.should.be.a.String;
+        res.body.time.should.have.property('created');
+        res.body.time.created.should.be.a.String;
+        // should not contains authSession cookie
+        should.not.exist(res.headers['set-cookie']);
         done();
       });
     });
+
+    it('should return module info and gzip when accept-encoding=gzip', function (done) {
+      request(app)
+      .get('/mk2testmodule')
+      .set('accept-encoding', 'gzip')
+      .expect('content-encoding', 'gzip')
+      .expect(200, function (err, res) {
+        // console.log(res.headers)
+        should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
+        res.body.should.have.keys('_id', '_rev', 'name', 'description',
+          'versions', 'dist-tags', 'readme', 'maintainers',
+          'time', 'author',
+          // 'repository',
+          '_attachments',
+          'users',
+          // 'readmeFilename',
+          // 'homepage', 'bugs',
+          'license');
+        res.body.name.should.equal('mk2testmodule');
+        res.body.versions[Object.keys(res.body.versions)[0]]
+          .dist.tarball.should.include('/mk2testmodule/download');
+        should.not.exist(res.headers['set-cookie']);
+        done();
+      });
+    });
+
+    it('should 304 when etag match', function (done) {
+      request(app)
+      .get('/mk2testmodule')
+      .set('If-None-Match', etag)
+      .expect(304, done);
+    });
   });
 
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.2.1')
+      .get('/mk2testmodule/0.0.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
-        body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.2.1');
-        body._id.should.equal('cnpmjs.org@0.2.1');
-        body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
+        body.name.should.equal('mk2testmodule');
+        body.version.should.match(/\d+\.\d+\.\d+/);
+        body._id.should.match(/mk2testmodule@\d+\.\d+\.\d+/);
+        body.dist.tarball.should.match(/mk2testmodule\-\d+\.\d+\.\d+\.tgz/);
         body.should.have.property('_cnpm_publish_time');
         body._cnpm_publish_time.should.be.a.Number;
         body.should.have.property('_publish_on_cnpm', true);
@@ -105,29 +165,93 @@ describe('controllers/registry/module.test.js', function () {
 
     it('should return module@tag info', function (done) {
       request(app)
-      .get('/cutter/latest')
+      .get('/mk2testmodule/latest')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
-        body.name.should.equal('cutter');
-        body.version.should.equal('0.0.3');
-        body._id.should.equal('cutter@0.0.3');
-        body.dist.tarball.should.include('/cutter/download/cutter-0.0.3.tgz');
+        body.name.should.equal('mk2testmodule');
+        body.version.should.equal('0.0.1');
+        body._id.should.equal('mk2testmodule@0.0.1');
+        body.dist.tarball.should.include('/mk2testmodule/download/mk2testmodule-0.0.1.tgz');
         done();
       });
     });
+  });
 
-    it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
+  describe('PUT /:name/-rev/id update maintainers', function () {
+    before(function (done) {
       request(app)
-      .get('/cnpmjs.org/0.2.1')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        var body = res.body;
-        body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.2.1');
-        body._publish_on_cnpm.should.equal(true);
-        done();
-      });
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should add new maintainers', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }, {
+          name: 'fengmk2',
+          email: 'fengmk2@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should add again new maintainers', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }, {
+          name: 'fengmk2',
+          email: 'fengmk2@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should rm maintainers', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
+    });
+
+    it('should rm again maintainers', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauth)
+      .expect(201)
+      .expect('content-type', 'application/json', done);
     });
   });
 
@@ -140,6 +264,13 @@ describe('controllers/registry/module.test.js', function () {
         name: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org'
       }],
+      keywords: [
+        'testputmodule', 'test', 'cnpmjstest'
+      ],
+      dependencies: {
+        'foo-testputmodule': "*",
+        'bar-testputmodule': '*'
+      }
     };
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
@@ -178,7 +309,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, done);
     });
 
-    it('should try to add return 403 when not module user and only next module exists',
+    it.skip('should try to add return 403 when not module user and only next module exists',
     function (done) {
       mm(config, 'enablePrivate', false);
       request(app)
@@ -201,7 +332,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('_id', '_rev', 'name', 'description', 'versions', 'dist-tags',
-          'readme', 'maintainers', 'time', '_attachments');
+          'readme', 'maintainers', 'time', '_attachments', 'users');
         res.body.versions.should.eql({});
         res.body.time.should.eql({});
         res.body['dist-tags'].should.eql({});
@@ -287,6 +418,7 @@ describe('controllers/registry/module.test.js', function () {
       var pkg = require(path.join(fixtures, 'testputmodule.json')).versions['0.1.8'];
       pkg.name = 'testputmodule';
       pkg.version = '0.1.9';
+      pkg.dependencies['foo-testputmodule'] = '*';
       request(app)
       .put('/' + pkg.name + '/' + pkg.version + '/-tag/latest')
       .set('authorization', baseauth)
@@ -294,7 +426,16 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('ok', 'rev');
-        done();
+        // should get deps foo-testputmodule contains 'testputmodule'
+        ModuleDeps.list('foo-testputmodule', function (err, rows) {
+          should.not.exist(err);
+          var exists = rows.filter(function (r) {
+            return r.deps === 'testputmodule';
+          });
+          exists.should.length(1);
+          exists[0].deps.should.equal('testputmodule');
+          done();
+        });
       });
     });
 
@@ -342,6 +483,60 @@ describe('controllers/registry/module.test.js', function () {
         done();
       });
     });
+
+    it('should publish with tgz base64, addPackageAndDist()', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      // delete first
+      request(app)
+      .del('/' + pkg.name + '/-rev/1')
+      .set('authorization', baseauth)
+      .expect({ok: true})
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+
+        request(app)
+        .put('/' + pkg.name)
+        .set('authorization', baseauth)
+        .send(pkg)
+        .expect(201, function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'rev');
+          res.body.ok.should.equal(true);
+
+          // upload again should 409
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', baseauth)
+          .send(pkg)
+          .expect(409, function (err, res) {
+            should.not.exist(err);
+            res.body.should.eql({
+              error: 'conflict',
+              reason: 'Document update conflict.'
+            });
+            done();
+          });
+
+        });
+      });
+    });
+
+    it('should version_error when versions missing', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      delete pkg.versions;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({
+          error: 'version_error',
+          reason: 'filename or version not found, filename: mk2testmodule-0.0.1.tgz, version: undefined'
+        });
+        done();
+      });
+    });
   });
 
   describe('GET /-/all', function () {
@@ -366,7 +561,6 @@ describe('controllers/registry/module.test.js', function () {
         res.body._updated.should.be.a.Number;
         var keys = Object.keys(res.body);
         keys.length.should.be.above(1);
-        res.body[keys[1]].dist.tarball.should.include('/download/');
         done();
       });
     });
@@ -438,6 +632,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should remove version ok', function (done) {
       //do not really remove it here
       mm.empty(Module, 'removeByNameAndVersions');
+      mm.empty(Module, 'removeTagsByIds');
       request(app)
       .put('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
@@ -454,35 +649,7 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .get('/cutter/download/cutter-0.0.2.tgz')
       .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
-      .expect(302, done)
-    });
-
-    it('should download a file direct from nfs stream', function (done) {
-      var nfs = require('../../../common/nfs');
-      mm(nfs, 'downloadStream', function (key, writeStream, options, callback) {
-        options.timeout.should.equal(600000);
-        nfs._client.download(key, {writeStream: writeStream, timeout: options.timeout}, callback);
-      });
-      Module.__get__ = Module.get;
-      mm(Module, 'get', function (name, version, callback) {
-        Module.__get__(name, version, function (err, info) {
-          info.package.dist.key = 'cutter/-/cutter-0.0.2.tgz';
-          callback(err, info);
-        });
-      });
-      request(app)
-      .get('/cutter/download/cutter-0.0.2.tgz')
-      .expect('ETag', 'c61fde5e8c26d053574d0c722097029fd1bc963a')
-      .expect('Content-Type', 'application/octet-stream')
-      .expect('Content-Length', '3139')
-      .expect('Content-Disposition', 'attachment; filename="cutter-0.0.2.tgz"')
-      .expect(200)
-      .end(function (err, res) {
-        should.not.exist(err);
-        // TODO: why supertest change buffer to text?
-        // res.text.length.should.equal(3139);
-        done();
-      });
+      .expect(302, done);
     });
   });
 
@@ -550,7 +717,11 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .del('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
-      .expect(200, done);
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        should.not.exist(res.headers['set-cookie']);
+        done();
+      });
     });
   });
 });

test/controllers/registry/user.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/registry/user.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,26 +16,45 @@
 
 var should = require('should');
 var request = require('supertest');
+var mm = require('mm');
 var app = require('../../../servers/registry');
 var user = require('../../../proxy/user');
-var mm = require('mm');
+var mysql = require('../../../common/mysql');
 
 describe('controllers/registry/user.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
+
   afterEach(mm.restore);
+
   describe('GET /-/user/org.couchdb.user:name', function () {
     it('should return user info', function (done) {
       request(app)
-      .get('/-/user/org.couchdb.user:cnpmjstest1')
+      .get('/-/user/org.couchdb.user:cnpmjstest10')
       .expect(200, function (err, res) {
         should.not.exist(err);
-        res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type', 'roles', 'date');
-        res.body.name.should.equal('cnpmjstest1');
+        res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type',
+          '_cnpm_meta', 'roles', 'date');
+        res.body.name.should.equal('cnpmjstest10');
+        done();
+      });
+    });
+
+    it.skip('should return npm user info', function (done) {
+      request(app)
+      .get('/-/user/org.couchdb.user:fengmk2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.name.should.equal('fengmk2');
+        res.body.github.should.equal('fengmk2');
+        res.body._cnpm_meta.should.have.keys('id', 'npm_user', 'gmt_create',
+          'gmt_modified', 'admin');
+        res.body._cnpm_meta.admin.should.equal(true);
         done();
       });
     });
@@ -47,7 +66,7 @@ describe('controllers/registry/user.test.js', function () {
     });
 
     it('should return 500 when mysql error', function (done) {
-      mm.error(user, 'get', 'mock error');
+      mm.error(mysql, 'query', 'mock mysql error');
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
       .expect(500, done);
@@ -91,7 +110,7 @@ describe('controllers/registry/user.test.js', function () {
         password_sha: 'password_sha',
         email: 'email'
       })
-      .expect(500, done);      
+      .expect(500, done);
     });
 
     it('should 201 when user.add ok', function (done) {
@@ -145,7 +164,12 @@ describe('controllers/registry/user.test.js', function () {
         ok: true,
         name: 'name',
         roles: []
-      }, done);
+      }, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.headers['set-cookie']);
+        res.headers['set-cookie'].join(';').should.include('AuthSession=');
+        done();
+      });
     });
   });
 
@@ -167,19 +191,26 @@ describe('controllers/registry/user.test.js', function () {
       mm.error(user, 'update', 'mock error');
       request(app)
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
+      .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
+        email: 'cnpmjstest10@cnpmjs.org'
+      })
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .expect(500, done);
     });
 
-    it('should 409 when req.body.rev error', function (done) {
+    it('should 201 when req.body.rev error', function (done) {
       request(app)
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org',
         rev: '1-123'
       })
-      .expect(409, done);
+      .expect(201, done);
     });
 
     it('should 201 update ok', function (done) {
@@ -188,6 +219,8 @@ describe('controllers/registry/user.test.js', function () {
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
       .send({
+        name: 'cnpmjstest10',
+        password: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org',
         rev: '1-123'
       })

test/controllers/sync.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/sync.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,12 +17,12 @@
 
 var request = require('supertest');
 var should = require('should');
-var registryApp = require('../../servers/registry');
-var webApp = require('../../servers/web');
 var pedding = require('pedding');
 var mm = require('mm');
-var Npm = require('../../proxy/npm');
 var path = require('path');
+var Npm = require('../../proxy/npm');
+var registryApp = require('../../servers/registry');
+var webApp = require('../../servers/web');
 
 describe('controllers/sync.test.js', function () {
   before(function (done) {
@@ -45,8 +45,6 @@ describe('controllers/sync.test.js', function () {
       request(registryApp)
       .del('/utility/-rev/123')
       .set('authorization', baseauth)
-      // .expect(200)
-      // .expect({ok: true})
       .end(function (err, res) {
         should.not.exist(err);
 
@@ -59,20 +57,6 @@ describe('controllers/sync.test.js', function () {
           res.body.should.have.keys('ok', 'logId');
           logIdRegistry = res.body.logId;
           done();
-          // setTimeout(function () {
-          //   request(registryApp)
-          //   .get('/utility')
-          //   .expect(200)
-          //   .end(function (err, res) {
-          //     should.not.exist(err);
-          //     Object.keys(res.body.versions).length.should.above(0);
-          //     for (var v in res.body.versions) {
-          //       var pkg = res.body.versions[v];
-          //       pkg.should.have.property('_publish_on_cnpm', true);
-          //     }
-          //     done();
-          //   });
-          // }, 5000);
         });
       });
     });
@@ -88,9 +72,20 @@ describe('controllers/sync.test.js', function () {
       }, done);
     });
 
-    it('should sync success', function (done) {
+    it('should sync through web success', function (done) {
+      mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+      request(webApp)
+      .put('/sync/utility')
+      .end(function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('ok', 'logId');
+        logIdWeb = res.body.logId;
+        done();
+      });
+    });
+
+    it('should sync through registry success', function (done) {
       mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
-      done = pedding(2, done);
       request(registryApp)
       .put('/utility/sync')
       .set('authorization', baseauth)
@@ -100,14 +95,6 @@ describe('controllers/sync.test.js', function () {
         logIdRegistry = res.body.logId;
         done();
       });
-      request(webApp)
-      .put('/sync/utility')
-      .end(function (err, res) {
-        should.not.exist(err);
-        res.body.should.have.keys('ok', 'logId');
-        logIdWeb = res.body.logId;
-        done();
-      });
     });
 
     it('should get sync log', function (done) {

test/controllers/total.test.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - test/controllers/total.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,9 +17,9 @@
 
 var should = require('should');
 var request = require('supertest');
+var pedding = require('pedding');
 var registryApp = require('../../servers/registry');
 var webApp = require('../../servers/web');
-var pedding = require('pedding');
 
 describe('controllers/total.test.js', function () {
   before(function (done) {
@@ -44,6 +44,7 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
+
     it('should return total info by jsonp', function (done) {
       request(registryApp)
       .get('?callback=totalCallback')
@@ -51,7 +52,8 @@ describe('controllers/total.test.js', function () {
       .expect(/totalCallback\({.*}\)/, done);
     });
   });
-  describe('GET /total in web', function () {
+
+  describe.skip('GET /total in web', function () {
     it('should return total info', function (done) {
       request(webApp)
       .get('/total')
@@ -62,5 +64,5 @@ describe('controllers/total.test.js', function () {
         done();
       });
     });
-  });  
+  });
 });

test/controllers/web/dist.test.js

@@ -0,0 +1,37 @@
+/**!
+ * cnpmjs.org - test/controllers/web/dist.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../../servers/web');
+
+describe('controllers/web/dist.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('GET /dist', function (done) {
+    it('should 302 to config.disturl', function (done) {
+      request(app)
+      .get('/dist')
+      .expect('Location', 'http://dist.u.qiniudn.com/')
+      .expect(302, done);
+    });
+  });
+});

test/controllers/web/package.test.js

@@ -17,14 +17,30 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var path = require('path');
 var mysql = require('../../../common/mysql');
 var app = require('../../../servers/web');
+var registry = require('../../../servers/registry');
 var pkg = require('../../../controllers/web/package');
 
+var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
+
 describe('controllers/web/package.test.js', function () {
+  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
+
   before(function (done) {
-    app.listen(0, done);
+    registry.listen(0, function () {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      request(registry)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(201, function () {
+        app.listen(0, done);
+      });
+    });
   });
+
   after(function (done) {
     app.close(done);
   });
@@ -32,32 +48,33 @@ describe('controllers/web/package.test.js', function () {
   afterEach(mm.restore);
 
   describe('GET /_list/search/search', function () {
-    it('should search with "c"', function (done) {
+    it('should search with "m"', function (done) {
       request(app)
-      .get('/_list/search/search?startkey="c"&limit=2')
+      .get('/_list/search/search?startkey="m"&limit=2')
+      .expect('content-type', 'application/json')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('rows');
-        res.body.rows.should.eql([
-          { key: 'c', count: 1, value: { name: 'c', description: 'Give folders or directories comments and view them easy.' } },
-          { key: 'charset', count: 1,
-            value: { name: 'charset', description: 'Get the content charset from header and html content-type.' } }
-        ]);
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
         done();
       });
     });
 
-    it('should search with c', function (done) {
+    it('should search with m', function (done) {
       request(app)
-      .get('/_list/search/search?startkey=c&limit=2')
+      .get('/_list/search/search?startkey=m&limit=2')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('rows');
-        res.body.rows.should.eql([
-          { key: 'c', count: 1, value: { name: 'c', description: 'Give folders or directories comments and view them easy.' } },
-          { key: 'charset', count: 1,
-            value: { name: 'charset', description: 'Get the content charset from header and html content-type.' } }
-        ]);
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
         done();
       });
     });
@@ -76,11 +93,18 @@ describe('controllers/web/package.test.js', function () {
   describe('GET /package/:name', function (done) {
     it('should get 200', function (done) {
       request(app)
-      .get('/package/cutter')
+      .get('/package/mk2testmodule')
       .expect(200)
+      .expect('content-encoding', 'gzip')
+      .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
-      .expect(/<th>Version<\/th>/, done);
+      .expect(/<th>Version<\/th>/, function (err, res) {
+        should.not.exist(err);
+        res.should.have.header('etag');
+        res.text.should.include('<meta charset="utf-8">');
+        done();
+      });
     });
 
     it('should get 404', function (done) {
@@ -93,7 +117,7 @@ describe('controllers/web/package.test.js', function () {
   describe('GET /package/:name/:version', function (done) {
     it('should 200 when get by version', function (done) {
       request(app)
-      .get('/package/cutter/0.0.2')
+      .get('/package/mk2testmodule/0.0.1')
       .expect(200)
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
@@ -102,21 +126,22 @@ describe('controllers/web/package.test.js', function () {
 
     it('should 200 when get by tag', function (done) {
       request(app)
-      .get('/package/cutter/latest')
+      .get('/package/mk2testmodule/latest')
       .expect(200)
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
       .expect(/<th>Version<\/th>/, done);
     });
+
     it('should 404 when get by version not exist', function (done) {
       request(app)
-      .get('/package/cutter/1.1.2')
+      .get('/package/mk2testmodule/1.1.2')
       .expect(404, done);
     });
 
     it('should 404 when get by tag', function (done) {
       request(app)
-      .get('/package/cutter/notexisttag')
+      .get('/package/mk2testmodule/notexisttag')
       .expect(404, done);
     });
   });
@@ -124,11 +149,18 @@ describe('controllers/web/package.test.js', function () {
   describe('GET　/browse/keyword/:word', function () {
     it('should list by keyword ok', function (done) {
       request(app)
-      .get('/browse/keyword/cnpm')
+      .get('/browse/keyword/mk2testmodule')
       .expect(200)
       .expect(/Packages match/, done);
     });
 
+    it('should list by keyword with json ok', function (done) {
+      request(app)
+      .get('/browse/keyword/mk2testmodule?type=json')
+      .expect(200)
+      .expect('content-type', 'application/json; charset=utf-8', done);
+    });
+
     it('should list no match ok', function (done) {
       request(app)
       .get('/browse/keyword/notexistpackage')
@@ -141,7 +173,7 @@ describe('controllers/web/package.test.js', function () {
       request(app)
       .get('/browse/keyword/notexistpackage')
       .expect(500)
-      .expect(/MockError: mm mock error/, done);
+      .expect(/Internal Server Error/, done);
     });
   });
 
@@ -155,7 +187,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://opensource.org/licenses/MIT'
       });
 
-      var p = {license: ['http://foo/MIT']};
+      p = {license: ['http://foo/MIT']};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({
@@ -163,7 +195,7 @@ describe('controllers/web/package.test.js', function () {
         url: 'http://foo/MIT'
       });
 
-      var p = {license: {name: 'mit', url: 'http://foo/mit'}};
+      p = {license: {name: 'mit', url: 'http://foo/mit'}};
       pkg.setLicense(p);
       p.license.should.have.keys('name', 'url');
       p.license.should.eql({

test/controllers/web/user.test.js

@@ -29,8 +29,9 @@ describe('controllers/web/user.test.js', function () {
   describe('GET /~:name', function (done) {
     it('should get 200', function (done) {
       request(app)
-      .get('/~dead_horse')
+      .get('/~cnpmjstest10')
       .expect(200)
+      .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="profile">/)
       .expect(/Packages by /, done);
     });
@@ -43,7 +44,7 @@ describe('controllers/web/user.test.js', function () {
 
     it('should get not eixst user but have modules 200', function (done) {
       request(app)
-      .get('/~tjholowaychuk')
+      .get('/~cnpmjstest101')
       .expect(200)
       .expect(/<div id="profile">/)
       .expect(/Packages by/, done);

test/fixtures/500.txt

@@ -0,0 +1,7 @@
+Internal routing error
+
+Sorry, we cannot connect to the intended server.
+
+We have just been notified of this problem. We will correct it as soon as possible.
+
+Feel free to contact us if you have any questions: support@iriscouch.com

test/fixtures/cnpmtest-package.json

@@ -0,0 +1 @@
+{"_id":"cnpmtest-package","_rev":"382002","name":"cnpmtest-package","description":"cnpmtest-package","dist-tags":{"latest":"0.0.0"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"time":{"modified":"2014-04-09T10:54:45.000Z","created":"2014-04-09T10:54:39.425Z","0.0.0":"2014-04-09T10:54:39.425Z"},"users":{},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"versions":{"0.0.0":{"name":"cnpmtest-package","version":"0.0.0","description":"cnpmtest-package","main":"index.js","scripts":{"test":"make test-all"},"config":{"cov":{"threshold":100}},"dependencies":{},"devDependencies":{"autod":"*","contributors":"*","should":"*","jshint":"*","cov":"*","istanbul-harmony":"*","mocha":"*"},"homepage":"https://github.com/{{group}}/cnpmtest-package","repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"keywords":["cnpmtest-package"],"engines":{"node":">= 0.10.0"},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"license":"MIT","contributors":[],"_id":"cnpmtest-package@0.0.0","dist":{"shasum":"81ea57c24a7f9f6e7263385116fc433f0ad9e179","size":1839,"noattachment":false,"tarball":"http://r.cnpmjs.org/cnpmtest-package/download/cnpmtest-package-0.0.0.tgz"},"_from":".","_npmVersion":"1.4.6","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"directories":{},"publish_time":1397040879425,"_cnpm_publish_time":1397040879425}},"readme":"cnpmtest-package\n=======\n\n[![Build Status](https://secure.travis-ci.org/{{group}}/cnpmtest-package.png)](http://travis-ci.org/{{group}}/cnpmtest-package)\n[![Dependency Status](https://gemnasium.com/{{group}}/cnpmtest-package.png)](https://gemnasium.com/{{group}}/cnpmtest-package)\n\n[![NPM](https://nodei.co/npm/cnpmtest-package.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmtest-package/)\n\n![logo](https://raw.github.com/{{group}}/cnpmtest-package/master/logo.png)\n\ncnpmtest-package desc\n\n## Install\n\n```bash\n$ npm install cnpmtest-package\n```\n\n## Usage\n\n```js\nvar cnpmtest-package = require('cnpmtest-package');\n\ncnpmtest-package.foo(function (err) {\n\n});\n```\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2014 fengmk2 <fengmk2@gmail.com> and other contributors\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","_attachments":{},"homepage":"https://github.com/{{group}}/cnpmtest-package","bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"license":"MIT"}

test/fixtures/fengmk2.json

@@ -0,0 +1 @@
+{"_id":"org.couchdb.user:fengmk2","_rev":"25-ba29aa975022944b4ba547ab11ce3af5","name":"fengmk2","email":"fengmk2@gmail.com","type":"user","roles":[],"date":"2014-03-15T08:06:18.870Z","fullname":"fengmk2","avatar":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=50&d=retro","freenode":"","github":"fengmk2","homepage":"http://fengmk2.github.com","twitter":"fengmk2","avatarMedium":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=100&d=retro","avatarLarge":"https://secure.gravatar.com/avatar/95b9d41231617a05ced5604d242c9670?s=496&d=retro","fields":[{"name":"fullname","value":"fengmk2","title":"Full Name","show":"fengmk2"},{"name":"email","value":"fengmk2@gmail.com","title":"Email","show":"<a href=\"mailto:fengmk2@gmail.com\">fengmk2@gmail.com</a>"},{"name":"github","value":"fengmk2","title":"Github","show":"<a rel=\"me\" href=\"https://github.com/fengmk2\">fengmk2</a>"},{"name":"twitter","value":"fengmk2","title":"Twitter","show":"<a rel=\"me\" href=\"https://twitter.com/fengmk2\">@fengmk2</a>"},{"name":"appdotnet","value":"","title":"App.net","show":""},{"name":"homepage","value":"http://fengmk2.github.com","title":"Homepage","show":"<a rel=\"me\" href=\"http://fengmk2.github.com/\">http://fengmk2.github.com</a>"},{"name":"freenode","value":"","title":"IRC Handle","show":""}],"appdotnet":"fengmk2"}

test/fixtures/package_and_tgz.json

@@ -0,0 +1 @@
+{"_id":"mk2testmodule","name":"mk2testmodule","description":"","dist-tags":{"latest":"0.0.1"},"versions":{"0.0.1":{"name":"mk2testmodule","version":"0.0.1","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"author":"","license":"ISC","readme":"ERROR: No README data found!","_id":"mk2testmodule@0.0.1","dist":{"shasum":"fa475605f88bab9b1127833633ca3ae0a477224c","tarball":"http://127.0.0.1:7001/mk2testmodule/-/mk2testmodule-0.0.1.tgz"},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}]}},"readme":"ERROR: No README data found!","maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_attachments":{"mk2testmodule-0.0.1.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2SsWrDMBCGPfspDg2ZinOyEgeylg6Zu2YR8rVRHEtGkkOg5N0jWaFdujVQAv6W4/7/dHcSGqTq5Ccthxyro7emeDCI2KxWkOKmaaaIdc4TouZQ8FqgwI3AdVMgF8ijho9e5DdGH6SLq/y1T74LfMcn4asEYEb2xLbA+q4O5ENv2/FE7CVZZ3JeW5NcrLDiWW3JK6eHcHey2Es9Zdq0dIkfKau50EcjjYpCmpDKSB0s7Nmbc9ZtwVhIBviBlP7Q1O4ZLBZAFx2As3jyOnWTYzhY9zPzpBUZPy2/e39l5bX87wedmZmZeRJuheTX2wAIAAA=","length":251}}}

test/init.js

@@ -0,0 +1,41 @@
+/**!
+ * cnpmjs.org - test/init.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var utility = require('utility');
+var User = require('../proxy/user');
+
+var usernames = [
+  'cnpmjstest101',
+  'cnpmjstest10'
+];
+
+usernames.forEach(function (name) {
+  var user = {
+    name: name,
+    email: 'fengmk2@gmail.com',
+    // password: 'cnpmjstest10',
+    ip: '127.0.0.1'
+  };
+  user.salt = crypto.randomBytes(30).toString('hex');
+  user.password_sha = utility.sha1(user.name + user.salt);
+
+  User.add(user, function (err, result) {
+    if (err) {
+      throw err;
+    }
+  });
+});

test/middleware/auth.test.js

@@ -34,7 +34,7 @@ describe('middleware/auth.test.js', function () {
   describe('auth()', function () {
     it('should pass if no authorization', function (done) {
       request(app)
-      .get('/-/user/org.couchdb.user:cnpmjstest1')
+      .get('/-/user/org.couchdb.user:cnpmjstest10')
       .expect(200, done);
     });
 

test/middleware/opensearch.test.js

@@ -30,8 +30,8 @@ describe('middleware/opensearch.test.js', function () {
     it('should get 200', function (done) {
       request(app)
       .get('/opensearch.xml')
-      .set('host', 'localhost:7002')
-      .expect(/http:\/\/localhost:7002/, done);
+      .set('host', 'localhost')
+      .expect(/http:\/\/localhost/, done);
     });
   });
 });

test/middleware/web_not_found.test.js

@@ -0,0 +1,63 @@
+/**!
+ * cnpmjs.org - test/middleware/web_not_found.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var app = require('../../servers/web');
+
+describe('middleware/web_not_found.test.js', function () {
+  before(function (done) {
+    app.listen(0, done);
+  });
+
+  after(function (done) {
+    app.close(done);
+  });
+
+  describe('web_not_found()', function () {
+    it('should redirect /mk2testmodule to /package/mk2testmodule', function (done) {
+      request(app)
+      .get('/mk2testmodule')
+      .expect('Location', '/package/mk2testmodule')
+      .expect(302, done);
+    });
+
+    it('should redirect /mk2testmodule/ to /package/mk2testmodule', function (done) {
+      request(app)
+      .get('/mk2testmodule/')
+      .expect('Location', '/package/mk2testmodule')
+      .expect(302, done);
+    });
+
+    it('should 404 /~byte', function (done) {
+      request(app)
+      .get('/~byte')
+      .expect(404, done);
+    });
+
+    it('should 200 /package/mk2testmodule', function (done) {
+      request(app)
+      .get('/package/mk2testmodule')
+      .expect(200, done);
+    });
+
+    it('should 404 /package/byte404', function (done) {
+      request(app)
+      .get('/package/byte404')
+      .expect(404, done);
+    });
+  });
+});

test/proxy/module.test.js

@@ -37,6 +37,84 @@ describe('proxy/module.test.js', function () {
     });
   });
 
+  describe('search()', function () {
+    before(function (done) {
+      Module.addKeywords('aaaa', 'mock aaaaaa', ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        done();
+      });
+    });
+
+    it.skip('should search modules', function (done) {
+      Module.search('mock', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.searchMatchs.length.should.above(0);
+        data.searchMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search match keywords modules', function (done) {
+      Module.search('aa', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.keywordMatchs.length.should.above(0);
+        data.keywordMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      Module.search('emptyemptyemptyempty', function (err, data) {
+        should.not.exist(err);
+        data.should.eql({
+          keywordMatchs: [],
+          searchMatchs: []
+        });
+        done();
+      });
+    });
+  });
+
+  var mockName = 'aa' + Date.now();
+  describe('addKeywords()', function () {
+    it('should add diff keywords to module', function (done) {
+      Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        done();
+      });
+    });
+
+    it('should add same keywords to module', function (done) {
+      Module.addKeywords(mockName, 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        // results.should.length(0);
+        done();
+      });
+    });
+  });
+
+  describe('getKeywords()', function () {
+    it('should get aa module keywords', function (done) {
+      Module.getKeywords(mockName, function (err, keywords) {
+        should.not.exist(err);
+        keywords.should.eql(['aa', 'bb', 'cc']);
+        done();
+      });
+    });
+  });
+
   describe('add()', function () {
     it('should success ad he@0.3.6', function (done) {
       var sourcePackage = require('../fixtures/0.3.6.json');
@@ -85,7 +163,7 @@ describe('proxy/module.test.js', function () {
           should.not.exist(err);
           data.package.readme.should.equal('test');
           done();
-        })
+        });
       });
     });
   });

test/proxy/module_deps.test.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - test/proxy/module_deps.test.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var pedding = require('pedding');
+var ModuleDeps = require('../../proxy/module_deps');
+
+describe('proxy/module_deps.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    ModuleDeps.remove('testmodule', 'foo', done);
+    ModuleDeps.remove('testmodule', 'bar', done);
+  });
+
+  describe('add()', function () {
+    it('should add foo, bar success', function (done) {
+      done = pedding(2, done);
+      ModuleDeps.add('testmodule', 'foo', function (err) {
+        should.not.exist(err);
+        // add again should work
+        ModuleDeps.add('testmodule', 'foo', function (err) {
+          should.not.exist(err);
+          done();
+        });
+      });
+
+      ModuleDeps.add('testmodule', 'bar', done);
+    });
+  });
+
+  describe('list()', function () {
+    it('should list testmodule deps', function (done) {
+      ModuleDeps.list('testmodule', function (err, rows) {
+        should.not.exist(err);
+        should.exist(rows);
+        rows.should.be.an.Array;
+        rows.should.length(2);
+        rows.forEach(function (row) {
+          row.should.have.property('deps');
+        });
+        done();
+      });
+    });
+  });
+});

test/proxy/module_star.test.js

@@ -0,0 +1,51 @@
+/**!
+ * cnpmjs.org - test/proxy/module_star.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var Star = require('../../proxy/module_star');
+
+describe('proxy/module_star.test.js', function () {
+  before(function *() {
+    yield Star.remove('testmodule', 'fengmk2');
+    yield Star.remove('testmodule', 'mk1');
+    yield Star.remove('testmodule', 'mk2');
+  });
+
+  it('should add a star', function *() {
+    yield Star.add('testmodule', 'fengmk2');
+    // again should be ok
+    yield Star.add('testmodule', 'fengmk2');
+    yield Star.add('testmodule', 'fengmk2');
+  });
+
+  it('should get all star users', function *() {
+    yield Star.add('testmodule', 'fengmk2');
+    yield Star.add('testmodule', 'mk1');
+    yield Star.add('testmodule', 'mk2');
+
+    var rows = yield Star.listUsers('testmodule');
+    rows.should.containDeep(['fengmk2', 'mk1', 'mk2']);
+  });
+
+  it('should get user all star modules', function *() {
+    yield Star.add('testmodule', 'fengmk2');
+    yield Star.add('testmodule1', 'fengmk2');
+    yield Star.add('testmodule2', 'fengmk2');
+
+    var rows = yield Star.listUserModules('fengmk2');
+    rows.should.containDeep(['testmodule', 'testmodule1', 'testmodule2']);
+  });
+});

test/proxy/npm.test.js

@@ -16,35 +16,46 @@
 
 var should = require('should');
 var mm = require('mm');
+var fs = require('fs');
+var path = require('path');
+var ChunkStream = require('chunkstream');
 var npm = require('../../proxy/npm');
 
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
+
 describe('proxy/npm.test.js', function () {
   afterEach(mm.restore);
 
-  it('should return a module info from source npm', function (done) {
-    npm.get('pedding', function (err, data) {
-      should.not.exist(err);
-      should.exist(data);
-      data.name.should.equal('pedding');
-      done();
-    });
+  it('should return a module info from source npm', function *() {
+    var data = yield npm.get('pedding');
+    data.name.should.equal('pedding');
   });
 
-  it('should return null when module not exist', function (done) {
-    npm.get('pedding-not-exists', function (err, data) {
-      should.not.exist(err);
-      should.not.exist(data);
-      done();
-    });
+  it('should return null when module not exist', function *() {
+    var data = yield npm.get('pedding-not-exists');
+    should.not.exist(data);
   });
 
-  it('should return error when http error', function (done) {
-    mm.http.request(/\//, '{');
-    npm.get('pedding-not-exists', function (err, data) {
-      should.exist(err);
+  it('should return error when http error', function *() {
+    mm.http.request(/\//, new ChunkStream(['{']));
+    try {
+      var data = yield npm.get('pedding-not-exists');
+      throw new Error('should not run this');
+    } catch (err) {
       err.name.should.equal('JSONResponseFormatError');
-      should.not.exist(data);
-      done();
-    });
+    }
+  });
+
+  it('should return ServerError when http 500 response', function *() {
+    var content = fs.createReadStream(path.join(fixtures, '500.txt'));
+    mm.http.request(/\//, content, { statusCode: 500 });
+    // http://registry.npmjs.org/octopie
+    try {
+      var data = yield npm.get('octopie');
+      throw new Error('should not run this');
+    } catch (err) {
+      err.name.should.equal('NPMServerError');
+      err.message.should.equal('Status 500, ' + fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8'));
+    }
   });
 });

test/proxy/sync_module_worker.test.js

@@ -22,14 +22,14 @@ var Log = require('../../proxy/module_log');
 describe('proxy/sync_module_worker.test.js', function () {
   it('should start a sync worker', function (done) {
     Log.create({
-      name: 'cnpmjs.org',
+      name: 'mk2testmodule',
       username: 'fengmk2',
     }, function (err, result) {
       should.not.exist(err);
       result.id.should.above(0);
       var worker = new SyncModuleWorker({
         logId: result.id,
-        name: 'cnpmjs.org',
+        name: 'mk2testmodule',
         username: 'fengmk2'
       });
 
@@ -40,21 +40,23 @@ describe('proxy/sync_module_worker.test.js', function () {
 
   it('should start a sync worker with names and noDep', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['cnpmjs.org', 'cutter'],
+      name: ['mk2testmodule'],
       noDep: true,
       username: 'fengmk2'
     });
 
     worker.start();
     worker.on('end', function () {
-      worker.successes.concat(worker.fails).should.eql(['cnpmjs.org', 'cutter']);
+      var names = worker.successes.concat(worker.fails);
+      names.sort();
+      names.should.eql(['mk2testmodule']);
       done();
     });
   });
 
   it('should start a sync worker with names', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['cnpmjs.org', 'cutter'],
+      name: ['mk2testmodule'],
       username: 'fengmk2'
     });
 

test/proxy/user.test.js

@@ -6,6 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
  */
 
 'use strict';
@@ -14,10 +15,14 @@
  * Module dependencies.
  */
 
-var mysql = require('../../common/mysql');
 var should = require('should');
-var user = require('../../proxy/user');
 var mm = require('mm');
+var path = require('path');
+var fs = require('fs');
+var mysql = require('../../common/mysql');
+var user = require('../../proxy/user');
+
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 
 var mockUser = {
   name: 'mockuser',
@@ -49,7 +54,9 @@ describe('proxy/user.test.js', function () {
     it('should get user ok', function (done) {
       user.get('mockuser', function (err, data) {
         should.not.exist(err);
-        data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
+        data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
+          'json', 'npm_user',
+          'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
         done();
       });
     });
@@ -68,7 +75,9 @@ describe('proxy/user.test.js', function () {
     it('should auth user ok', function (done) {
       user.auth(mockUser.name, mockUser.password, function (err, data) {
         should.not.exist(err);
-        data.should.have.keys('id', 'rev', 'name', 'email', 'salt', 'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
+        data.should.have.keys('id', 'rev', 'name', 'email', 'salt',
+          'json', 'npm_user',
+          'password_sha', 'ip', 'roles', 'gmt_create', 'gmt_modified');
         done();
       });
     });
@@ -118,11 +127,12 @@ describe('proxy/user.test.js', function () {
 
   describe('update()', function () {
     before(initUser);
+
     it('should update ok', function (done) {
       user.update(mockUser, function (err, data) {
         should.not.exist(err);
         should.exist(data);
-        data.should.have.keys(['rev']);
+        data.should.have.keys('rev', 'result');
         done();
       });
     });
@@ -131,7 +141,8 @@ describe('proxy/user.test.js', function () {
       mockUser.rev = '1-error';
       user.update(mockUser, function (err, data) {
         should.not.exist(err);
-        should.not.exist(data);
+        should.exist(data);
+        data.result.affectedRows.should.equal(0);
         done();
       });
     });
@@ -153,4 +164,32 @@ describe('proxy/user.test.js', function () {
       });
     });
   });
+
+  describe('saveNpmUser()', function () {
+    var existUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
+    var notExistUser = JSON.parse(fs.readFileSync(path.join(fixtures, 'fengmk2.json')));
+    notExistUser.name = 'fengmk2-not-exists';
+
+    before(function *() {
+      yield mysql.query('delete from user where name=?', [notExistUser.name]);
+    });
+
+    it('should save npm user to exists user', function *() {
+      yield user.saveNpmUser(existUser);
+      var r = yield mysql.queryOne('select rev, json, npm_user from user where name=?', existUser.name);
+      should.exist(r);
+      // r.npm_user.should.equal(0);
+      r.rev.should.equal(existUser._rev);
+      JSON.parse(r.json).should.eql(existUser);
+    });
+
+    it('should save npm user to not exists user and create it', function *() {
+      yield user.saveNpmUser(notExistUser);
+      var r = yield mysql.queryOne('select name, json, npm_user from user where name=?', notExistUser.name);
+      r.name.should.equal(notExistUser.name);
+      should.exist(r);
+      r.npm_user.should.equal(1);
+      JSON.parse(r.json).should.eql(notExistUser);
+    });
+  });
 });

test/sync.js

@@ -17,21 +17,27 @@
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var mysql = require('../common/mysql');
 var Log = require('../proxy/module_log');
+var config = require('../config');
 
-var name = process.argv[2] || 'address,pedding';
-var names = name.split(',');
+config.sourceNpmRegistry = 'http://r.cnpmjs.org';
+
+var names = process.argv[2] || 'byte';
+names = names.split(',');
 
 Log.create({
   name: names[0],
   username: 'fengmk2',
 }, function (err, result) {
+  if (err) {
+    throw err;
+  }
   var worker = new SyncModuleWorker({
     logId: result.id,
     name: names,
     username: 'fengmk2',
     concurrency: names.length,
-    noDep: true,
-    publish: true,
+    // noDep: true,
+    // publish: true,
   });
 
   worker.start();

test/sync/sync_all.test.js

@@ -1,5 +1,5 @@
 /*!
- * cnpmjs.org - test/sync/sync_all.js
+ * cnpmjs.org - test/sync/sync_all.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -20,39 +20,32 @@ var Total = require('../../proxy/total');
 var should = require('should');
 var Module = require('../../proxy/module');
 
-describe('sync/sync_all.js', function () {
+describe('sync/sync_all.test.js', function () {
   describe('sync()', function () {
     afterEach(mm.restore);
 
-    it('should sync first time ok', function (done) {
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+    it('should sync first time ok', function *() {
+      mm.data(Npm, 'getShort', ['mk2testmodule']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpmjs.org', 'cutter']);
-        mm.restore();
-        Total.getTotalInfo(function (err, result) {
-          should.not.exist(err);
-          result.last_sync_module.should.equal('cutter');
-          done();
-        });
-      });
+      var data = yield sync;
+      data.successes.should.eql(['mk2testmodule']);
+      mm.restore();
+      var result = yield Total.getTotalInfo();
+      result.last_sync_module.should.equal('mk2testmodule');
     });
 
-    it('should sync common ok', function (done) {
+    it('should sync common ok', function *() {
       mm.data(Npm, 'getAllSince', {
         _updated: Date.now(),
-        'cnpmjs.org': {},
-        cutter: {}
+        'mk2testmodule': {},
+        // cutter: {}
       });
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
+      mm.data(Npm, 'getShort', ['mk2testmodule']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
-      mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpm', 'cnpmjs.org', 'cutter']);
-        done();
-      });
+      mm.data(Module, 'listAllModuleNames', [{name: 'mk2testmodule'}]);
+      var data = yield sync;
+      data.successes.should.eql(['mk2testmodule']);
+      mm.restore();
     });
   });
 });

test/sync/sync_exist.test.js

@@ -1,5 +1,5 @@
 /*!
- * cnpmjs.org - test/sync/sync_exist.js
+ * cnpmjs.org - test/sync/sync_exist.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -13,38 +13,32 @@
 /**
  * Module dependencies.
  */
-var sync = require('../../sync/sync_exist');
+
+var should = require('should');
 var mm = require('mm');
+var sync = require('../../sync/sync_exist');
 var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
-var should = require('should');
 
-describe('sync/sync_exist.js', function () {
+describe('sync/sync_exist.test.js', function () {
   describe('sync()', function () {
     afterEach(mm.restore);
 
-    it('should sync first time ok', function (done) {
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+    it('should sync first time ok', function *() {
+      mm.data(Npm, 'getShort', ['mk2testmodule']);
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpmjs.org', 'cutter']);
-        done();
-      });
+      var data = yield sync();
+      data.successes.should.eql(['mk2testmodule']);
     });
 
-    it('should sync common ok', function (done) {
+    it('should sync common ok', function *() {
       mm.data(Npm, 'getAllSince', {
         _updated: Date.now(),
-        'cnpmjs.org': {},
-        cutter: {}
+        'mk2testmodule': {},
       });
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
-      sync(function (err, data) {
-        should.not.exist(err);
-        data.successes.should.eql(['cnpmjs.org', 'cutter']);
-        done();
-      });
+      var data = yield sync();
+      data.successes.should.eql(['mk2testmodule']);
     });
   });
 });

view/web/layout.html

@@ -1,6 +1,7 @@
 <!DOCTYPE html>
 <html>
   <head>
+    <meta charset="utf-8">
     <title><%- locals.title %></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <link rel="shortcut icon" href="/public/favicon.png">
@@ -29,6 +30,10 @@
       table.downloads td.count {
         text-align: right;
       }
+
+      .nav-tabs{margin:20px 0;}
+      .nav-cont{display:none;}
+      .nav-cont.active{display:block;}
     </style>
     <script>
       $(function () {
@@ -41,6 +46,16 @@
             return location.href = '/browse/keyword/' + val;
           }
         });
+
+        $(".nav-tabs li").each(function (index) {
+          $(this).data("index", index);
+        })
+        .on("click", function (e) {
+          e.preventDefault();
+          $(".nav-tabs li.active,.nav-cont.active").removeClass("active");
+          $(this).addClass("active");
+          $(".nav-cont").eq($(this).data("index")).addClass("active");
+        });
       });
     </script>
   </head>

view/web/package.html

@@ -18,198 +18,243 @@
       <p class="description"><%= package.description %></p>
     <% } %>
 
-    <pre class="sh"><code>$ <%- config.npmClientName %> install <%= package.name %></code></pre>
-
-    <table class="downloads">
-      <tbody>
-        <tr>
-          <td class="count"><%- download.today %></td><td> downloads today</td>
-          <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
-          <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
-        </tr>
-        <tr>
-          <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
-          <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
-          <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
-        </tr>
-      </tbody>
-    </table>
-
-    <table class="metadata">
-      <% if (Array.isArray(package.maintainers) && package.maintainers.length > 0) { %>
-      <tr>
-        <th>Maintainers</th>
-        <td>
-        <% package.maintainers.forEach(function (m) { %>
-          <span class="user">
-            <a class="username" href="/~<%= m.name %>">
-              <% if (m.gravatar) { %>
-              <img src="<%- m.gravatar %>" class="avatar">
-              <% } %>
-              <%= m.name %>
-            </a>
-          </span>
-        <% }) %>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <th>Version</th>
-        <td>
-          <b>
-          <%= package.version %>
-          </b>
-          <% if (package.fromNow) { %>
-          last updated
-          <%= package.fromNow %>
-          <% } %>
-        </td>
-      </tr>
-      <% if (package.license) { %>
-      <tr>
-        <th>License</th>
-        <td>
-          <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
-        </td>
-      </tr>
-      <% } %>
-      <%
-      if (typeof package.keywords === 'string') {
-        package.keywords = package.keywords.split(/\s*,?\s*/)
-      }
-      if (Array.isArray(package.keywords)) {
-      %>
-      <tr>
-        <th>Keywords</th>
-        <td><%-
-          package.keywords.map(function (kw) {
-            kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
-            return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
-          }).join(', ')
-          %></td>
-      </tr>
-      <% } %>
-      <% if (package.repository && package.repository !== 'undefined') { %>
-      <tr>
-        <th>Repository</td>
-        <td>
-          <a href="<%= package.repository.weburl || package.repository.url %>" target="_blank">
-            <%= package.repository.url %>
-          </a>
-          (<%= package.repository.type %>)
-        </td>
-      </tr>
-      <% } %>
-      <% if (typeof package.homepage === 'string') { %>
-      <tr>
-        <th>Homepage</td>
-        <td>
-          <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
-        </td>
-      </tr>
-      <% } %>
-      <% if (package.bugs && package.bugs.url) { %>
-      <tr>
-        <th>Bugs</td>
-        <td>
-          <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <%
-          var deps = Object.keys(package.dependencies || {})
-          var l = deps.length
-        %>
-        <th>Dependencies<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          if (l === 0) {
-            %>None<%
-          } else {
-            var m = 200
-            if (l > m) deps = deps.slice(0, m)
-            deps.forEach(function(pkg, i) {
-              if (i > 0) { %>, <% }
-              %>
-              <a href="/package/<%= pkg %>"><%= pkg %></a><%
-            })
-            if (l > m) {
-              %>, and <%= l-m %> more<%
-            }
-          }
-          %>
-        </td>
-      </tr>
-
-      <% if (package.users) {
-        var starredBy = package.starredBy
-        var l = starredBy.length
-      %>
-      <tr>
-        <th>Starred by<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          var max = 20
-          if (l > max) {
-            starredBy = starredBy.sort(function (a, b) {
-              return Math.random() * 2 - 1
-            }).slice(0, max)
-          }
-          starredBy.forEach(function (user, i) {
-            if (i > 0) { %>, <% }
-            %><a href="/~<%= user %>"><%= user %></a><%
-          })
-          if (l > max) {
-            %><br><a href="/browse/star/<%= package.name %>">and
-            <%= (l-max) %> more</a><%
-          }
-          %>
-        </td>
-      </tr>
-      <% } %>
-
-      <% if (Array.isArray(package.contributors) && package.contributors.length > 0) { %>
-      <tr>
-        <th>Contributors</th>
-        <td>
-        <% package.contributors.forEach(function (m) { %>
-          <span class="user">
-            <a class="username" target="_blank" href="<%= m.url %>">
-              <% if (m.gravatar) { %>
-              <img src="<%- m.gravatar %>" class="avatar">
-              <% } %>
-              <%= m.name %>
-            </a>
-          </span>
-        <% }) %>
-        </td>
-      </tr>
-      <% } %>
-
-      <% if (package.dist && package.dist.tarball) { %>
-      <tr>
-        <th>Download</th>
-        <td>
-          <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
-            <%= package.dist.tarball %>
-          </a>
-        </td>
-      </tr>
-      <% } %>
-    </table>
-
-    <div class="details">
-      <ul class="toc">
-        <% if (package.readme) { %>
-        <li><a href="#readme">Read Me</a></li>
-        <% } %>
-      </ul>
+    <pre class="sh"><code>$ <%- config.npmClientName %> install <%= package.name %> <% if (package.preferGlobal) { %>-g<% } %></code></pre>
 
+    <ul class="nav nav-tabs">
       <% if (package.readme) { %>
-      <section id="readme">
-      <%- package.readme %>
-      </section>
+        <li class="active"><a href="#readme">Readme.md</a></li>
+        <li><a href="#meta">package.json</a></li>
+      <% } else { %>
+        <li class="active"><a href="#meta">package.json</a></li>
       <% } %>
+    </ul>
+
+    <% if (package.readme) { %>
+    <div class="nav-cont active">
+      <div class="details">
+        <section id="readme">
+        <%- package.readme %>
+        </section>
+      </div>
+    </div>
+    <% } %>
+
+    <div class="nav-cont">
+      <table class="downloads">
+        <tbody>
+          <tr>
+            <td class="count"><%- download.today %></td><td> downloads today</td>
+            <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
+            <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
+          </tr>
+          <tr>
+            <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
+            <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
+            <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
+          </tr>
+        </tbody>
+      </table>
+
+      <table class="metadata">
+        <% if (Array.isArray(package.maintainers) && package.maintainers.length > 0) { %>
+        <tr>
+          <th>Maintainers</th>
+          <td>
+          <% package.maintainers.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" href="/~<%= m.name %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <th>Version</th>
+          <td>
+            <b>
+            <%= package.version %>
+            </b>
+            <% if (package.fromNow) { %>
+            last updated
+            <%= package.fromNow %>
+            <% } %>
+          </td>
+        </tr>
+        <% if (package.engines) {%>
+        <tr>
+          <th>Engines</th>
+          <td>
+            <ul>
+            <% for (var k in package.engines) { %>
+              <li><%= k %>: <%= package.engines[k] %></li>
+            <% } %>
+            </ul>
+          </td>
+        </tr>
+        <% } %>
+        <% if (package.license) { %>
+        <tr>
+          <th>License</th>
+          <td>
+            <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
+          </td>
+        </tr>
+        <% } %>
+        <%
+        if (typeof package.keywords === 'string') {
+          package.keywords = package.keywords.split(/\s*,?\s*/)
+        }
+        if (Array.isArray(package.keywords)) {
+        %>
+        <tr>
+          <th>Keywords</th>
+          <td><%-
+            package.keywords.map(function (kw) {
+              kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
+              return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
+            }).join(', ')
+            %></td>
+        </tr>
+        <% } %>
+        <% if (package.repository && package.repository !== 'undefined') { %>
+        <tr>
+          <th>Repository</td>
+          <td>
+            <a href="<%= package.repository.weburl || package.repository.url %>" target="_blank">
+              <%= package.repository.url %>
+            </a>
+            (<%= package.repository.type %>)
+          </td>
+        </tr>
+        <% } %>
+        <% if (typeof package.homepage === 'string') { %>
+        <tr>
+          <th>Homepage</td>
+          <td>
+            <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
+          </td>
+        </tr>
+        <% } %>
+        <% if (package.bugs && package.bugs.url) { %>
+        <tr>
+          <th>Bugs</td>
+          <td>
+            <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <%
+            var deps = Object.keys(package.dependencies || {})
+            var l = deps.length
+          %>
+          <th>Dependencies<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <tr>
+          <%
+            var deps = package.dependents || [];
+            var l = deps.length
+          %>
+          <th>Dependents<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <% if (package.users.length > 0) {
+          var users = package.users
+          var l = users.length
+        %>
+        <tr>
+          <th>Starred by (<%= l %>)</th>
+          <td>
+            <%
+            var max = 20
+            if (l > max) {
+              users = users.sort(function (a, b) {
+                return Math.random() * 2 - 1
+              }).slice(0, max)
+            }
+            users.forEach(function (user, i) {
+              if (i > 0) { %>, <% }
+              %><a href="/~<%= user %>"><%= user %></a><%
+            })
+            if (l > max) {
+              %><br><a href="/browse/star/<%= package.name %>">and
+              <%= (l - max) %> more</a><%
+            }
+            %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (Array.isArray(package.contributors) && package.contributors.length > 0) { %>
+        <tr>
+          <th>Contributors</th>
+          <td>
+          <% package.contributors.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" target="_blank" href="<%= m.url %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (package.dist && package.dist.tarball) { %>
+        <tr>
+          <th>Download</th>
+          <td>
+            <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
+              <%= package.dist.tarball %>
+            </a>
+          </td>
+        </tr>
+        <% } %>
+      </table>
     </div>
   </div>

view/web/search.html

@@ -16,26 +16,52 @@
 <div id="search">
   <% if (!packages.length) { %>
     <div class="alert alert-warning">
-      Can not found package match <%= keyword %>. You can 
+      Can not found package match <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
     </div>
-  <% } else if (packages[0].name !== keyword) { %>
+  <% } else if (!match) { %>
     <div class="alert alert-info">
-      Can not found package <%= keyword %>. You can 
+      Can not found package <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
-    </div>    
+    </div>
   <% } %>
   <% if (packages.length) { %>
     <h1>
-      Packages match <span style="color: #09f;"><%= keyword %></span>
+      Packages match "<span style="color: #09f;"><%= keyword %></span>"
     </h1>
     <hr />
-    <% for (var i = 0; i < packages.length; i++) { %>
-      <div class="package <%= packages[i].name === keyword ? 'match' : '' %>">
-        <a href="/package/<%= packages[i].name %>" class="package-name"><%= packages[i].name %></a>
-        <span class="package-description"><%= packages[i].description %></span>
+    <% if (match) { %>
+      <div class="package match">
+        <a href="/package/<%= match.name %>" class="package-name"><%= match.name %></a>
+        <span class="package-description"><%= match.description %></span>
+      </div>
+    <% } %>
+    <% for (var i = 0; i < packages.length; i++) {
+      var item = packages[i];
+      if (item.name === keyword) {
+        continue;
+      }
+    %>
+      <div class="package">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
+      </div>
+    <% } %>
+  <% } %>
+
+  <% if (keywords.length) { %>
+    <h1>
+      Keywords match "<span style="color: #09f;"><%= keyword %></span>"
+    </h1>
+    <hr />
+    <% for (var i = 0; i < keywords.length; i++) {
+      var item = keywords[i];
+    %>
+      <div class="package <%= item.name === keyword ? 'match' : '' %>">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
       </div>
     <% } %>
   <% } %>

worker.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,7 +16,7 @@
  */
 
 var graceful = require('graceful');
-
+var logger = require('./common/logger');
 var config = require('./config');
 var registry = require('./servers/registry');
 var web = require('./servers/web');
@@ -34,6 +34,7 @@ graceful({
     if (err.message) {
       err.message += ' (uncaughtException throw ' + throwErrorCount + ' times on pid:' + process.pid + ')';
     }
-    console.error(err);
+    console.error(err.stack);
+    logger.error(err);
   }
 });