Release 0.3.2

update koa-sess and koa-redis

.gitignore

@@ -15,4 +15,10 @@ results
 node_modules
 npm-debug.log
 public/dist/
-config/config.js
+config/config.js
+backup/*.json
+backup/*.gz
+docs/web/history.md
+view/web/_layout.html
+bin/mysql.js
+bin/test.sql

.npmignore

@@ -5,4 +5,8 @@ Makefile
 .travis.yml
 logo.png
 .tmp/
-public/dist/
+public/dist/
+backup/*.json
+backup/*.gz
+view/web/_layout.html
+bin/mysql.js

.travis.yml

@@ -1,4 +1,4 @@
 language: node_js
 node_js:
-  - '0.10'
+  - '0.11'
 script: make test-coveralls

AUTHORS

@@ -0,0 +1,8 @@
+# Ordered by date of first contribution.
+# Auto-generated by 'contributors' on Fri, 24 Jan 2014 08:35:59 GMT.
+# https://github.com/xingrz/node-contributors
+
+fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)
+dead-horse <dead_horse@qq.com> (https://github.com/dead-horse)
+alsotang <alsotang@gmail.com> (https://github.com/alsotang)
+4simple <wondger@qq.com> (https://github.com/4simple)

History.md

@@ -1,4 +1,252 @@
 
+0.3.2 / 2014-02-28 
+==================
+
+  * update koa-sess and koa-redis
+  * fix sync all test
+  * remove nfs.downloadStream first, fix tmppath error
+  * fix fengmk2/giturl#1 bug
+
+0.3.1 / 2014-02-27 
+==================
+
+  * add etag fixed #224
+  * travis ci install on source npm
+
+0.3.0 / 2014-02-27 
+==================
+
+  * fix typo and dont sync not exists pkgs
+  * use koa-middlewares
+  * fix signed cookie not work on npm@1.3.25; node --harmony-generators
+  * fix opensearch test case
+  * update koa bodyparser
+  * logger.error(err) should send err stack email notice
+  * json body parse limit and bug fix.
+  * fix sync 404 reason not clear
+  * all controllers to koa
+  * controller/web/user.js to koa
+  * change web connect to koa
+  * use outputError
+  * use yield exports.addPackageAndDist.call(this, next);
+  * add end() when ws write end
+  * fix yield coWrite
+  * fix all the test of registry module.test.js
+  * convert registry/module.js to koa type
+  * fix auth middleware
+  * finish registry user controller koa and update mm to support thunkify. fixed #196
+  * change controllers/user.js to koa
+  * thunkify all proxy
+  * convert all middlewares to koa type
+  * change regsitry sync to koa
+  * addd koa-jsonp, koa-bodyparser, fix / controller
+  * first koa run registry home page /
+  * Merge pull request #212 from cnpm/fix-sync-404
+  * return friendly 404 reason
+  * Merge pull request #211 from cnpm/bug-fix
+  * override json limit to default 10mb. fixed #209
+  * fix #210 addPackageAndDist package version detect bug
+
+0.2.27 / 2014-02-19 
+==================
+
+  * support json result in search, fixed #189
+
+0.2.26 / 2014-02-19 
+==================
+
+  * npm publish also need to add deps
+
+0.2.25 / 2014-02-19 
+==================
+
+  * max handle number of package.json `dependencies` property
+  * Dependents support. fixed #190
+
+0.2.24 / 2014-02-13 
+==================
+
+  * fix if delete all the versions
+  * refactor remove module, fixed #186
+
+0.2.23 / 2014-01-26 
+==================
+
+  * system admin can add, publish, remove the packages. fixed #176
+
+0.2.22 / 2014-01-26 
+==================
+
+  * add keyword and search support keyword. #181
+
+0.2.21 / 2014-01-24 
+==================
+
+  * refactor code styles on package.html
+  * nav-tabs e.preventDefault
+  * Show registry server error response. fixed #178
+  * nav-tabs for package.html (@4simple)
+
+0.2.20 / 2014-01-23 
+==================
+
+  * hotfix sync missing dependencies and readmes
+  * fix sync readme error, fixed #174
+  * add updateReadme in module
+
+0.2.19 / 2014-01-22 
+==================
+
+  * npm install no need to check authorization header. fixed #171
+
+0.2.18 / 2014-01-20 
+==================
+
+  * Support gitlab git url to display and click. fixed #160
+  * fix redis crash
+
+0.2.17 / 2014-01-17 
+==================
+
+  * custom logo url
+  * hotfix layout bug
+
+0.2.16 / 2014-01-16 
+==================
+
+  * fix publish-time bug
+
+0.2.15 / 2014-01-16 
+==================
+
+  * add publish_time to debug
+
+0.2.14 / 2014-01-16 
+==================
+
+  * add make autod
+  * update publish_time, fixed #163
+
+0.2.13 / 2014-01-15 
+==================
+
+  * markdown tmpl not support footer, need to wrap on app start
+
+0.2.12 / 2014-01-15 
+==================
+
+  * add footer and npm client name customable
+
+0.2.11 / 2014-01-15 
+==================
+
+  * package page contributor link to search, default is true
+
+0.2.10 / 2014-01-14 
+==================
+
+  * fix #155 Content-Disposition wrong.
+
+0.2.9 / 2014-01-14 
+==================
+
+  * support startkey=c and startkey="c"
+  * support couch db search api. fixed #153
+  * fix fork me image link
+  * support sync by query.name
+
+0.2.8 / 2014-01-14 
+==================
+
+  * dont show err stack on test env
+  * add download link for package page
+
+0.2.7 / 2014-01-13 
+==================
+
+  * add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
+
+0.2.6 / 2014-01-13 
+==================
+
+  * support custom session store, fixed #146
+
+0.2.5 / 2014-01-13 
+==================
+
+  * add download timeout and unit test
+  * use downloadStream() first
+  * nfs download to a writeable stream.
+
+0.2.4 / 2014-01-10 
+==================
+
+  * set main script to  index.js, fixed #142
+
+0.2.3 / 2014-01-10 
+==================
+
+  * Dont show sync button on private package
+  * Sync package as publish with no deps. fixed #138
+
+0.2.2 / 2014-01-10 
+==================
+
+  * keep compatibility
+  * qnfs upload only callback a url
+  * compat remove package
+  * set tarball url
+  * new npm publish in one request, add _publish_in_cnpm
+  * support unsure name ufs
+  * contributors maybe a object
+  * Object #<Object> has no method 'forEach' fixed #134
+  * support custom config as a module, fixed issue #132
+  * support npm new publish flow. fixed #129
+  * add toString and constructor to test admin
+  * fix #119 hasOwnProperty check admin bug.
+
+0.2.0 / 2013-12-27 
+==================
+
+  * remove to lower case
+  * fix #127 execSync and execsync.
+  * add contributors list on package page
+  * mv blanket to config
+  * sync typeerror fix #statusCode
+  * add disturl
+  * fix #122 admin security bug
+  * fixed #121, let pkg 404 as success
+  * fix sql insert error
+  * fix typos
+
+0.1.3 / 2013-12-20
+==================
+
+  * add favicon
+  * Fix auth middleware bug (@alsotang)
+  * make sure all packages name are lower case
+  * select ids from tag
+  * fix nodejsctl
+  * fix #112 missing versions and time no sync
+  * remove restart command
+  * fix sync missing packages error
+  * fix web/readme.md, add install
+  * fix #109 pkg no times and no versions bug.
+
+0.1.2 / 2013-12-19
+==================
+
+  * fix times not exists canot sync bug. fixed #101
+  * support npm run command
+  * remove before_install and install in travis, fixed #102
+  * split all sub queries, fixed #104
+  * fix doc, fixed #103
+  * fix search too slow.
+  * dont email sync log level info
+  * only sync missing packages at first time
+  * update dependencies
+  * sync all will sync all the missing packages, fixed #97
+
 0.1.0 / 2013-12-12
 ==================
 

Makefile

@@ -1,15 +1,17 @@
 TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
 REPORTER = tap
-TIMEOUT = 20000
+TIMEOUT = 30000
 MOCHA_OPTS =
 
 install:
-	@npm install
+	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm --disturl=http://dist.u.qiniudn.com
 
 test: install
 	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
+		--harmony-generators \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
+		--require should \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
@@ -27,4 +29,11 @@ test-coveralls: test
 
 test-all: test test-cov
 
+contributors: install
+	@./node_modules/contributors/bin/contributors -f plain -o AUTHORS
+
+autod: install
+	@./node_modules/.bin/autod -w -e public,view,docs,backup
+	@$(MAKE) install
+
 .PHONY: test

README.md

@@ -1,30 +1,53 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
-![logo](https://raw.github.com/fengmk2/cnpmjs.org/master/logo.png)
+![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
+
 ## Install
 
 ```bash
-$ npm install
+$ npm install --registry=http://r.cnpmjs.org --disturl=http://dist.u.qiniudn.com
 ```
 
 ## Usage
 
 ```js
-$ node dispatch.js
+$ node --harmony-generators dispatch.js
+```
+
+## Guide
+
+* [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
+* [NFS guide](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide)
+
+## Authors
+
+```bash
+$ git summary
+
+ project  : cnpmjs.org
+ repo age : 3 months
+ active   : 145 days
+ commits  : 366
+ files    : 94
+ authors  :
+   217  fengmk2                 59.3%
+   146  dead_horse              39.9%
+     2  4simple                 0.5%
+     1  Alsotang                0.3%
 ```
 
 ## License

backup/dump.js

@@ -0,0 +1,142 @@
+/**!
+ * cnpmjs.org - backup/dump.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/**
+ * 1. dump module
+ * 2. dump tag
+ * 3. dump user
+ * 4. total
+ * 5. download_total
+ */
+
+var path = require('path');
+var fs = require('fs');
+var moment = require('moment');
+var eventproxy = require('eventproxy');
+var util = require('util');
+var zlib = require('zlib');
+var mysql = require('../common/mysql');
+var nfs = require('../common/nfs');
+var config = require('../config');
+
+function dumpTable(name, lastRow, callback) {
+  var sql = 'SELECT * from ' + name + ' WHERE gmt_modified >=? ORDER BY gmt_modified ASC LIMIT 10000;';
+  mysql.query(sql, [lastRow.gmt_modified], function (err, rows) {
+    if (err || rows.length === 0) {
+      return callback(err, rows);
+    }
+    if (rows[0].id === lastRow.id) {
+      rows = rows.slice(1);
+    }
+    callback(null, rows);
+  });
+}
+
+function log() {
+  var str = '[' + moment().format('YYYY-MM-DD HH:mm:ss') + '] ' + util.format.apply(util, arguments);
+  console.log(str);
+}
+
+function syncTable(name, callback) {
+  var datadir = __dirname;
+  var dataFile = path.join(datadir, moment().format('YYYY-MM-DD-HH') + '_' + name + '.json');
+  var lastRowFile = path.join(datadir, name + '_lastdate.json');
+  var lastRow = null;
+  if (fs.existsSync(lastRowFile)) {
+    lastRow = require(lastRowFile);
+    lastRow.gmt_modified = new Date(Date.parse(lastRow.gmt_modified));
+  } else {
+    lastRow = {
+      gmt_modified: new Date('2011-11-11'),
+    };
+  }
+
+  log('getting "%s" since %j', name, lastRow);
+  dumpTable(name, lastRow, function (err, rows) {
+    console.log('[%s] got %d rows', Date(), rows && rows.length || 0);
+    if (err) {
+      return callback(err);
+    }
+
+    if (!rows || rows.length === 0) {
+      return callback();
+    }
+
+    var writeStream = fs.createWriteStream(dataFile, {flags: 'a'});
+    writeStream.once('error', callback);
+    for (var i = 0; i < rows.length; i++) {
+      writeStream.write(JSON.stringify(rows[i]) + '\n');
+    }
+    writeStream.end();
+    writeStream.on('finish', function () {
+      log('append %d rows to %s', rows.length, dataFile);
+      var gzfile = dataFile + '.gz';
+      var gzip = zlib.createGzip();
+      var inp = fs.createReadStream(dataFile);
+      var out = fs.createWriteStream(gzfile);
+      inp.pipe(gzip).pipe(out);
+      out.once('error', callback);
+      out.on('finish', function () {
+        var key = path.join(config.backupFilePrefix, path.basename(gzfile));
+        log('saving %s to %s ...', gzfile, key);
+        nfs.upload(gzfile, {key: key}, function (err, result) {
+          if (err) {
+            return callback(err);
+          }
+
+          lastRow = rows[rows.length - 1];
+          lastRow = {gmt_modified: lastRow.gmt_modified, id: lastRow.id};
+          fs.writeFileSync(lastRowFile, JSON.stringify(lastRow));
+          log('save %s data file to %j, lastrow: %j', name, result, lastRow);
+          callback();
+        });
+      });
+    });
+  });
+}
+
+var ep = eventproxy.create();
+ep.fail(function (err) {
+  log('error: %s', err.stack);
+  process.exit(1);
+});
+
+syncTable('module', ep.done('module'));
+
+ep.on('module', function () {
+  syncTable('tag', ep.done('tag'));
+});
+
+ep.on('tag', function () {
+  syncTable('user', ep.done('user'));
+});
+
+ep.on('user', function () {
+  syncTable('total', ep.done('total'));
+});
+
+ep.on('total', function () {
+  syncTable('download_total', ep.done('download_total'));
+});
+
+ep.on('download_total', function () {
+  ep.emit('finish');
+});
+
+ep.on('finish', function () {
+  log('finish, %d process exit', process.pid);
+  process.exit(0);
+});

bin/nodejsctl

@@ -7,10 +7,11 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS=node
+NODEJS='node --harmony-generators'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
-STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'))"`
+STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\
+process.exit(0);"`
 NODE_DISPATH_PATH=${BASE_HOME}/dispatch.js
 PROG_NAME=$0
 ACTION=$1
@@ -47,9 +48,9 @@ stop()
 {
   get_pid
   if [ ! -z "$PID" ]; then
-    echo "Waiting $PROJECT_NAME stop for 5s ..."
+    echo "Waiting $PROJECT_NAME stop for 2s ..."
     kill -15 $PID
-    sleep 5
+    sleep 2
 
     node_num=`ps -ef |grep node|grep ${PROJECT_NAME}|grep -v grep|wc -l`
     if [ $node_num != 0 ]; then
@@ -77,6 +78,7 @@ status()
   else
     echo "$PROJECT_NAME is not running"
   fi
+  exit 0;
 }
 
 case "$ACTION" in

bin/restore_module_deps.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - bin/restore_module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var mysql = require('../common/mysql');
+var Module = require('../proxy/module');
+var ModuleDeps = require('../proxy/module_deps');
+
+var addCount = 0;
+
+function restore(id, callback) {
+  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
+  mysql.query(sql, [id], function (err, rows) {
+    if (err) {
+      return callback(err);
+    }
+    if (rows.length === 0) {
+      return callback(null, []);
+    }
+
+    console.log('[%s] got %d rows', id, rows.length);
+
+    rows.forEach(function (r) {
+      Module.parseRow(r);
+      if (!r.package) {
+        return;
+      }
+      var deps = Object.keys(r.package.dependencies || {});
+      if (!Array.isArray(deps) || !deps.length) {
+        return;
+      }
+      deps.forEach(function (dep) {
+        ModuleDeps.add(dep, r.name, function (err) {
+          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
+        });
+      });
+      addCount += deps.length;
+    });
+    setTimeout(function () {
+      console.log('[%s] add %d relations', id, addCount);
+      callback(null, rows);
+    }, 1000);
+  });
+}
+
+var id = 0;
+function run() {
+  restore(id, function (err, rows) {
+    if (err) {
+      throw err;
+    }
+    if (rows.length === 0) {
+      console.log('finished, last id: %s, exit.', id);
+      process.exit(0);
+    }
+
+    id = rows[rows.length - 1].id;
+    run();
+  });
+}
+
+run();

common/logger.js

@@ -1,7 +1,12 @@
-/*!
+/**!
  * cnpmjs.org - common/logger.js
- * Copyright(c) 2013 
- * Author: dead_horse <undefined>
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
  */
 
 'use strict';
@@ -10,11 +15,12 @@
  * Module dependencies.
  */
 
-var config = require('../config');
 var util = require('util');
 var moment = require('moment');
 var logstream = require('logfilestream');
 var ms = require('ms');
+var config = require('../config');
+var mail = require('./mail');
 
 var isTEST = process.env.NODE_ENV === 'test';
 var ONE_DAY = ms('1d');
@@ -29,7 +35,9 @@ levels.forEach(function (catetory) {
   var stream = logstream(options);
   function write(msg) {
     var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
+    var subject = null;
     if (msg instanceof Error) {
+      subject = msg.name;
       var err = {
         name: msg.name,
         code: msg.code,
@@ -57,12 +65,21 @@ levels.forEach(function (catetory) {
     } else {
       msg = time + ' ' + util.format.apply(util, arguments) + '\n';
     }
+
     if (!isTEST) {
+      stream.write(msg);
       if (config.debug) {
         var level = catetory;
         console.log('[' + level + '] ' + msg);
       } else {
-        stream.write(msg);
+        if (catetory === 'error' && subject) {
+          // send error email
+          var to = [];
+          for (var name in config.admins) {
+            to.push(config.admins[name]);
+          }
+          mail.error(to, subject, msg);
+        }
       }
     }
   }

common/mail.js

@@ -0,0 +1,76 @@
+/**!
+ * cnpmjs.org - common/mail.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var utility = require('utility');
+var nodemailer = require('nodemailer');
+var os = require('os');
+var mailConfig = require('../config').mail;
+
+/**
+ * Send notice email with mail level and appname.
+ *
+ * @param {String|Array} to, email or email list.
+ * @param {String} level, e.g.: 'log, warn, error'.
+ * @param {String} subject
+ * @param {String} html
+ * @param {Function(err, result)} callback
+ */
+exports.notice = function sendLogMail(to, level, subject, html, callback) {
+  subject = '[' + mailConfig.appname + '] [' + level + '] [' + os.hostname() + '] ' + subject;
+  html = String(html);
+  exports.send(to, subject, html.replace(/\n/g, '<br/>'), callback);
+};
+
+var LEVELS = [ 'log', 'warn', 'error' ];
+LEVELS.forEach(function (level) {
+  exports[level] = function (to, subject, html, callback) {
+    exports.notice(to, level, subject, html, callback);
+  };
+});
+
+/**
+ * Send email.
+ * @param {String|Array} to, email or email list.
+ * @param {String} subject
+ * @param {String} html
+ * @param {Function(err, result)} callback
+ */
+exports.send = function (to, subject, html, callback) {
+  callback = callback || utility.noop;
+
+  var transport = nodemailer.createTransport("SMTP", {
+    host: mailConfig.host,
+    port: mailConfig.port,
+    secureConnection: mailConfig.ssl,
+    debug: mailConfig.debug,
+    auth: {
+      user: mailConfig.user,
+      pass: mailConfig.pass,
+    }
+  });
+
+  var message = {
+    sender: mailConfig.sender,
+    to: to,
+    subject: subject,
+    html: html,
+  };
+
+  transport.sendMail(message, function (err, result) {
+    transport.close();
+    callback(err, result);
+  });
+};

common/mysql.js

@@ -35,7 +35,9 @@ var pool = mysql.createPool({
 exports.pool = pool;
 
 exports.query = function (sql, values, cb) {
-  pool.query(sql, values, cb);
+  pool.query(sql, values, function (err, rows) {
+    cb(err, rows);
+  });
 };
 
 exports.queryOne = function (sql, values, cb) {
@@ -51,6 +53,10 @@ exports.queryOne = function (sql, values, cb) {
   });
 };
 
+exports.escape = function (val) {
+  return pool.escape(val);
+};
+
 ready(exports);
 
 function init() {

common/qnfs.js

@@ -14,11 +14,13 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var qn = require('qn');
 var config = require('../config');
-
 var client = qn.create(config.qn);
 
+exports._client = client;
+
 /**
  * Upload file
  *
@@ -32,7 +34,23 @@ var client = qn.create(config.qn);
  */
 exports.upload = function (filepath, options, callback) {
   client.delete(options.key, function (err, data) {
-    client.uploadFile(filepath, {key: options.key, size: options.size}, callback);
+    client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
+  });
+};
+
+exports.uploadBuffer = function (buf, options, callback) {
+  client.delete(options.key, function (err, data) {
+    client.upload(buf, {key: options.key}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
   });
 };
 
@@ -43,3 +61,5 @@ exports.url = function (key) {
 exports.remove = function (key, callback) {
   client.delete(key, callback);
 };
+
+thunkify(exports);

common/session.js

@@ -15,27 +15,18 @@
  * Module dependencies.
  */
 
-var connect = require('connect');
-var RedisStore = require('connect-redis')(connect);
+var middlewares = require('koa-middlewares');
 var config = require('../config');
 
-var session;
 var key = 'AuthSession';
-var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 30 };
+var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 365, signed: false };
+var options = {
+  key: key,
+  cookie: cookie,
+};
 
-if (config.debug) {
-  session = connect.cookieSession({
-    secret: config.sessionSecret,
-    key: key,
-    cookie: cookie
-  });
-} else {
-  session = connect.session({
-    key: key,
-    secret: config.sessionSecret,
-    store: new RedisStore(config.redis),
-    cookie: cookie,
-  });
+if (!config.debug) {
+  options.store = config.sessionStore || middlewares.RedisStore(config.redis);
 }
 
-module.exports = session;
+module.exports = middlewares.session(options);

config/index.js

@@ -35,7 +35,7 @@ var config = {
   // mysql config
   mysqlServers: [
     {
-      host: 'db4free.net',
+      host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
       port: 3306,
       user: 'cnpmjs',
       password: 'cnpmjs123'
@@ -51,6 +51,7 @@ var config = {
     port: 19533,
     pass: 'cnpmjs_dev'
   },
+  jsonLimit: '10mb', // max request json body size
   uploadDir: path.join(root, 'public', 'dist'),
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
   qn: {
@@ -60,15 +61,35 @@ var config = {
     domain: "http://qtestbucket.qiniudn.com"
   },
 
+  mail: {
+    appname: 'cnpmjs.org',
+    sender: 'cnpmjs.org mail sender <adderss@gmail.com>',
+    host: 'smtp.gmail.com',
+    port: 465,
+    user: 'address@gmail.com',
+    pass: 'your password',
+    ssl: true,
+    debug: false
+  },
+
+  logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
+  registryHost: 'r.cnpmjs.org',
+  customFooter: '', // you can add copyright and site total script html here
+  npmClientName: 'cnpm', // use `${name} install package`
+  packagePageContributorSearch: true, // package page contributor link to search, default is true
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
-    admin: true,
-    fengmk2: true,
-    dead_horse: true,
-    cnpmjstest10: true,
+    fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
+    dead_horse: 'dead_horse@qq.com',
+    cnpmjstest10: 'cnpmjstest10@cnpmjs.org',
   },
-  syncByInstall: true
+  syncByInstall: true,
+  backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
+  syncModel: 'none', // 'none', 'all', 'exist'
+  syncConcurrency: 1,
+  maxDependencies: 200, // max handle number of package.json `dependencies` property
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js
@@ -84,3 +105,12 @@ mkdirp.sync(config.logdir);
 mkdirp.sync(config.uploadDir);
 
 module.exports = config;
+
+config.loadConfig = function (customConfig) {
+  if (!customConfig) {
+    return;
+  }
+  for (var key in customConfig) {
+    config[key] = customConfig[key];
+  }
+};

controllers/common.js

@@ -1,35 +0,0 @@
-/**!
- * cnpmjs.org - controllers/registry/common.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var crypto = require('crypto');
-var path = require('path');
-var config = require('../config');
-
-exports.getTarballFilepath = function (filename) {
-  // ensure download file path unique
-  var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex') + '.tgz');
-  return path.join(config.uploadDir, name);
-};
-
-exports.getCDNKey = function (name, filename) {
-  return '/' + name + '/-/' + filename;
-};
-
-exports.downloadURL = function (pkg, req) {
-  if (pkg.dist && pkg.dist.tarball) {
-    pkg.dist.tarball = 'http://' + req.headers.host + '/' + pkg.name + '/download/' + path.basename(pkg.dist.tarball);
-  }
-};

controllers/download.js

@@ -14,10 +14,15 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var moment = require('moment');
 var DownloadTotal = require('../proxy/download');
 
 exports.total = function (name, callback) {
+  if (typeof name === 'function') {
+    callback = name;
+    name = null;
+  }
   var end = moment();
   var start = end.clone().subtract('months', 1).startOf('month');
   var lastday = end.clone().subtract('days', 1).format('YYYY-MM-DD');
@@ -74,3 +79,5 @@ exports.total = function (name, callback) {
 
   DownloadTotal[method].apply(DownloadTotal, args);
 };
+
+thunkify(exports);

controllers/registry/home.js

@@ -1,46 +0,0 @@
-/**!
- * cnpmjs.org - controllers/registry/home.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var microtime = require('microtime');
-var eventproxy = require('eventproxy');
-var Total = require('../../proxy/total');
-var down = require('../download');
-var version = require('../../package.json').version;
-
-var startTime = '' + microtime.now();
-
-exports.show = function (req, res, next) {
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Total.get(ep.done('total'));
-  down.total(null, ep.done('download'));
-  ep.all('total', 'download', function (total, download) {
-    total.download = download;
-    total.db_name = 'registry';
-    total.instance_start_time = startTime;
-    total.node_version = process.version;
-    total.app_version = version;
-    total.donate = 'https://me.alipay.com/imk2';
-    var callback = req.query.callback;
-    if (callback) {
-      // support jsonp
-      res.jsonp(total, callback);
-    } else {
-      res.json(total);
-    }
-  });
-};

controllers/registry/user.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - controllers/registry/user.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -18,30 +18,24 @@
 var debug = require('debug')('cnpmjs.org:controllers:registry');
 var logger = require('../../common/logger');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  User.get(name, function (err, row) {
-    if (err) {
-      return next(err);
-    }
-    if (!row) {
-      return next();
-    }
-
-    res.setHeader('etag', '"' + row.rev + '"');
-    var data = {
-      _id: 'org.couchdb.user:' + row.name,
-      _rev: row.rev,
-      name: row.name,
-      email: row.email,
-      type: 'user',
-      roles: [],
-      date: row.gmt_modified,
-    };
-    res.json(data);
-  });
+exports.show = function *(next) {
+  var name = this.params.name;
+  var user = yield User.get(name);
+  if (!user) {
+    return yield next;
+  }
+  this.etag = '"' + user.rev + '"';
+  var data = {
+    _id: 'org.couchdb.user:' + user.name,
+    _rev: user.rev,
+    name: user.name,
+    email: user.email,
+    type: 'user',
+    roles: [],
+    date: user.gmt_modified,
+  };
+  this.body = data;
 };
 
 // json:
@@ -53,110 +47,109 @@ exports.show = function (req, res, next) {
 //    type: 'user',
 //    roles: [],
 //    date: '2013-12-04T12:56:13.714Z' } }
-exports.add = function (req, res, next) {
-  var name = req.params.name;
-  var body = req.body || {};
+exports.add = function *() {
+  var name = this.params.name;
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     // roles: body.roles || [],
   };
+
   if (!user.name || !user.salt || !user.password_sha || !user.email) {
-    return res.json(422, {
+    this.status = 422;
+    this.body = {
       error: 'paramError',
       reason: 'params missing'
-    });
+    };
+    return;
   }
   debug('add user: %j', user);
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  User.get(name, ep.doneLater(function (row) {
-    if (row) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    User.add(user, ep.done('add'));
-  }));
+  var existUser = yield User.get(name);
+  if (existUser) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
 
-  ep.once('add', function (result) {
-    res.setHeader('etag', '"' + result.rev + '"');
-    // location: 'http://registry.npmjs.org/_users/org.couchdb.user:cnpmjstest1',
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.add(user);
+  this.etag = '"' + result.rev + '"';
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
 };
 
-exports.authSession = function (req, res, next) {
+exports.authSession = function *() {
   // body: {"name":"foo","password":"****"}
-  var body = req.body || {};
+  var body = this.request.body || {};
   var name = body.name;
   var password = body.password;
-  User.auth(name, password, function (err, user) {
-    debug('authSession %s: %j', name, user);
-    if (err) {
-      return next(err);
-    }
-    if (!user) {
-      return res.json(401, {ok: false, name: null, roles: []});
-    }
+  var user = yield User.auth(name, password);
+  debug('authSession %s: %j', name, user);
 
-    req.session.name = user.name;
-    res.json(200, {ok: true, name: user.name, roles: []});
-  });
+  if (!user) {
+    this.status = 401;
+    this.body = {ok: false, name: null, roles: []};
+    return;
+  }
+
+  this.session.name = user.name;
+  this.body = {ok: true, name: user.name, roles: []};
 };
 
-exports.update = function (req, res, next) {
-  var name = req.params.name;
-  var rev = req.params.rev;
+exports.update = function *(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
   if (!name || !rev) {
-    return next();
+    return yield next;
   }
 
-  debug('update: %s, rev: %s, session.name: %s', name, rev, req.session.name);
+  debug('update: %s, rev: %s, session.name: %s', name, rev, this.session.name);
 
-  if (name !== req.session.name) {
+  if (name !== this.session.name) {
     // must authSession first
-    res.statusCode = 401;
-    return res.json({
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Name is incorrect.'
-    });
+    };
+    return;
   }
 
-  var body = req.body || {};
+  var body = this.request.body || {};
   var user = {
     name: body.name,
     salt: body.salt,
     password_sha: body.password_sha,
     email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
+    ip: this.ip || '0.0.0.0',
     rev: body.rev || body._rev,
     // roles: body.roles || [],
   };
-  User.update(user, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    //check rev error
-    if (!result) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + user.name,
-      rev: result.rev
-    });
-  });
+  var result = yield User.update(user);
+  if (!result) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
 };

controllers/sync.js

@@ -13,38 +13,64 @@
 /**
  * Module dependencies.
  */
+
 var Log = require('../proxy/module_log');
-var SyncModuleWorker = require('./sync_module_worker');
+var SyncModuleWorker = require('../proxy/sync_module_worker');
 
-exports.sync = function (req, res, next) {
-  var username = req.session.name || 'anonymous';
-  var name = req.params.name;
-  SyncModuleWorker.sync(name, username, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    if (!result.ok) {
-      return res.json(result.statusCode, result.pkg);
-    }
-    res.json(201, {
-      ok: true,
-      logId: result.logId
-    });
-  });
+exports.sync = function *() {
+  var username = this.session.name || 'anonymous';
+  var name = this.params.name;
+  var publish = this.query.publish === 'true';
+  var noDep = this.query.nodeps === 'true';
+  if (publish && !this.session.isAdmin) {
+    this.status = 403;
+    this.body = {
+      error: 'no_perms',
+      reason: 'Only admin can publish'
+    };
+    return;
+  }
+
+  var options = {
+    publish: publish,
+    noDep: noDep,
+  };
+
+  var result = yield SyncModuleWorker.sync(name, username, options);
+
+  // friendly 404 reason info
+  if (result.staticCache === 404) {
+    this.status = 404;
+    this.body = {
+      ok: false,
+      reason: 'can not found ' + name + ' in the source registry'
+    };
+    return;
+  }
+  if (!result.ok) {
+    this.status = result.statusCode;
+    this.body = result.pkg;
+    return;
+  }
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logId: result.logId
+  };
 };
 
-exports.getSyncLog = function (req, res, next) {
-  var logId = req.params.id;
-  var name = req.params.name;
-  var offset = Number(req.query.offset) || 0;
-  Log.get(logId, function (err, row) {
-    if (err || !row) {
-      return next(err);
-    }
-    var log = row.log.trim();
-    if (offset > 0) {
-      log = log.split('\n').slice(offset).join('\n');
-    }
-    res.json(200, {ok: true, log: log});
-  });
+exports.getSyncLog = function *(next) {
+  var logId = this.params.id;
+  var name = this.params.name;
+  var offset = Number(this.query.offset) || 0;
+  var row = yield Log.get(logId);
+  if (!row) {
+    return yield next;
+  }
+
+  var log = row.log.trim();
+  if (offset > 0) {
+    log = log.split('\n').slice(offset).join('\n');
+  }
+  this.body = {ok: true, log: log};
 };

controllers/total.js

@@ -0,0 +1,40 @@
+/**!
+ * cnpmjs.org - controllers/total.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var microtime = require('microtime');
+var Total = require('../proxy/total');
+var Download = require('./download');
+var version = require('../package.json').version;
+var config = require('../config');
+
+var startTime = '' + microtime.now();
+
+exports.show = function *() {
+  var r = yield [Total.get(), Download.total()];
+  var total = r[0];
+  var download = r[1];
+
+  total.download = download;
+  total.db_name = 'registry';
+  total.instance_start_time = startTime;
+  total.node_version = process.version;
+  total.app_version = version;
+  total.donate = 'https://me.alipay.com/imk2';
+  total.sync_model = config.syncModel;
+
+  this.body = total;
+};

controllers/web/package.js

@@ -14,87 +14,158 @@
  * Module dependencies.
  */
 
+var giturl = require('giturl');
 var moment = require('moment');
 var eventproxy = require('eventproxy');
 var semver = require('semver');
 var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
+var config = require('../../config');
 var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var ModuleDeps = require('../../proxy/module_deps');
+var setDownloadURL = require('../../lib/common').setDownloadURL;
 
-exports.display = function (req, res, next) {
-  var params = req.params;
+exports.display = function *(next) {
+  var params = this.params;
   var name = params.name;
   var tag = params.version;
-  var ep = eventproxy.create();
-  ep.fail(next);
 
-  if (tag) {
-    var version = semver.valid(tag);
-    if (version) {
-      Module.get(name, version, ep.done('pkg'));
-    } else {
-      Module.getByTag(name, tag, ep.done('pkg'));
-    }
+  var getPackageMethod;
+  var getPackageArgs;
+  var version = semver.valid(tag || '');
+  if (version) {
+    getPackageMethod = 'get';
+    getPackageArgs = [name, version];
   } else {
-    Module.getByTag(name, 'latest', ep.done('pkg'));
+    getPackageMethod = 'getByTag';
+    getPackageArgs = [name, tag || 'latest'];
+  }
+  var r = yield [
+    Module[getPackageMethod].apply(Module, getPackageArgs),
+    down.total(name),
+    ModuleDeps.list(name)
+  ];
+  var pkg = r[0];
+  var download = r[1];
+  var dependents = r[2];
+
+  if (!pkg || !pkg.package) {
+    return yield next;
   }
 
-  down.total(name, ep.done('download'));
+  pkg.package.fromNow = moment(pkg.publish_time).fromNow();
+  pkg = pkg.package;
+  pkg.readme = marked(pkg.readme || '');
+  if (!pkg.readme) {
+    pkg.readme = pkg.description || '';
+  }
 
-  ep.all('pkg', 'download', function (pkg, download) {
-    if (!pkg || !pkg.package) {
-      return next();
-    }
-
-    pkg.package.fromNow = moment(pkg.publish_time).fromNow();
-    pkg = pkg.package;
-    pkg.readme = marked(pkg.readme || '');
-    if (pkg.maintainers) {
-      for (var i = 0; i < pkg.maintainers.length; i++) {
-        var maintainer = pkg.maintainers[i];
-        if (maintainer.email) {
-          maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
-        }
+  if (pkg.maintainers) {
+    for (var i = 0; i < pkg.maintainers.length; i++) {
+      var maintainer = pkg.maintainers[i];
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
       }
     }
+  }
 
-    setLicense(pkg);
-
-    for (var k in download) {
-      download[k] = humanize(download[k]);
+  if (pkg.contributors) {
+    // registry.cnpmjs.org/compressible
+    if (!Array.isArray(pkg.contributors)) {
+      pkg.contributors = [pkg.contributors];
     }
+    for (var i = 0; i < pkg.contributors.length; i++) {
+      var contributor = pkg.contributors[i];
+      if (contributor.email) {
+        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
+      }
+      if (config.packagePageContributorSearch || !contributor.url) {
+        contributor.url = '/~' + encodeURIComponent(contributor.name);
+      }
+    }
+  }
 
-    res.render('package', {
-      title: 'Package - ' + pkg.name,
-      package: pkg,
-      download: download
-    });
+  if (pkg.repository && pkg.repository.url) {
+    pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
+  }
+
+  setLicense(pkg);
+
+  for (var k in download) {
+    download[k] = humanize(download[k]);
+  }
+  setDownloadURL(pkg, this, config.registryHost);
+
+  pkg.dependents = dependents;
+
+  yield this.render('package', {
+    title: 'Package - ' + pkg.name,
+    package: pkg,
+    download: download
   });
 };
 
-exports.search = function (req, res, next) {
-  var params = req.params;
-  var word = req.params.word;
-  Module.search(word, function (err, packages) {
-    if (err) {
-      return next(err);
-    }
-
-    res.render('search', {
-      title: 'Keyword - ' + word,
+exports.search = function *(next) {
+  var params = this.params;
+  var word = params.word;
+  var result = yield Module.search(word);
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.body = {
       keyword: word,
-      packages: packages || []
-    });
+      packages: result.searchMatchs,
+      keywords: result.keywordMatchs
+    };
+    this.charset = 'utf-8';
+    return;
+  }
+
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
   });
 };
 
-exports.displaySync = function (req, res, next) {
-  var name = req.params.name;
-  res.render('sync', {
+exports.rangeSearch = function *(next) {
+  var startKey = this.query.startkey || '';
+  if (startKey[0] === '"') {
+    startKey = startKey.substring(1);
+  }
+  if (startKey[startKey.length - 1] === '"') {
+    startKey = startKey.substring(0, startKey.length - 1);
+  }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield Module.search(startKey, {limit: limit});
+
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
+
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
+};
+
+exports.displaySync = function *(next) {
+  var name = this.params.name || this.query.name;
+  yield this.render('sync', {
     name: name,
     title: 'Sync - ' + name
   });
@@ -132,6 +203,7 @@ function setLicense(pkg) {
     }
   }
 }
+exports.setLicense = setLicense;
 
 function getOssLicenseUrlFromName(name) {
   var base = 'http://opensource.org/licenses/';

controllers/web/user.js

@@ -15,31 +15,24 @@
  */
 var Module = require('../../proxy/module');
 var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
 
-exports.display = function (req, res, next) {
-  var name = req.params.name;
+exports.display = function *(next) {
+  var name = this.params.name;
 
-  var ep = eventproxy.create();
-  ep.fail(next);
-  Module.listByAuthor(name, ep.done('packages'));
-  User.get(name, ep.done('user'));
+  var r = yield [Module.listByAuthor(name), User.get(name)];
+  var packages = r[0];
+  var user = r[1];
+  if (!user && !packages.length) {
+    return yield next;
+  }
+  user = {
+    name: name,
+    email: user && user.email
+  };
 
-  ep.all('packages', 'user', function (packages, user) {
-    //because of sync, maybe no this user in database,
-    //but his packages in this registry
-    if (!user && !packages.length) {
-      return next();
-    }
-    user = {
-      name: name,
-      email: user && user.email
-    };
-
-    return res.render('profile', {
-      title: 'User - ' + name,
-      packages: packages || [],
-      user: user
-    });
+  yield this.render('profile', {
+    title: 'User - ' + name,
+    packages: packages || [],
+    user: user
   });
 };

dispatch.js

@@ -21,6 +21,8 @@ var fs = require('fs');
 var cluster = require('cluster');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
+var childProcess = require('child_process');
+var syncPath = path.join(__dirname, 'sync');
 
 if (config.enableCluster) {
   cluster.setupMaster({
@@ -50,6 +52,8 @@ if (config.enableCluster) {
     cluster.fork();
   }
 
+  childProcess.fork(syncPath);
 } else {
   require(workerPath);
+  require(syncPath);
 }

docs/db.sql

@@ -14,15 +14,26 @@ CREATE TABLE `user` (
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
 
+CREATE TABLE `module_keyword` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `description` longtext,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `keyword_module_name` (`keyword`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
+
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `author` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `description` longtext,
- `package` longtext COMMENT 'package.json',
+ `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
  `dist_shasum` varchar(100) DEFAULT NULL,
  `dist_tarball` varchar(2048) DEFAULT NULL,
  `dist_size` int(10) unsigned NOT NULL DEFAULT '0',
@@ -35,23 +46,28 @@ CREATE TABLE `module` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module info';
 -- ALTER TABLE `module` ADD `description` longtext;
 -- ALTER TABLE `module` ADD `publish_time` bigint(20) unsigned, ADD KEY `publish_time` (`publish_time`);
+-- ALTER TABLE `module` CHANGE `package` `package` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
+-- ALTER TABLE `module` CHANGE `description` `description` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
+-- show create table module\G
+-- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `module_log` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `username` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `log` longtext,
  PRIMARY KEY (`id`),
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+-- ALTER TABLE  `module_log` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `tag` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `tag` varchar(30) NOT NULL COMMENT 'tag name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
@@ -59,22 +75,50 @@ CREATE TABLE `tag` (
  UNIQUE KEY `name` (`name`, `tag`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
 -- ALTER TABLE  `tag` ADD  `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
+-- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `total` (
  `name` varchar(100) NOT NULL COMMENT 'total name',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `module_delete` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'module delete count',
+ `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry',
+ `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry',
+ `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status',
+ `need_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages need to be sync',
+ `success_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync success at this time',
+ `fail_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync fail at this time',
+ `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync',
+ `last_sync_module` varchar(100) COMMENT 'last sync success module name',
  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
-INSERT INTO total(name, gmt_modified, module_delete) VALUES('total', now(), 0);
+INSERT INTO total(name, gmt_modified) VALUES('total', now());
+-- ALTER TABLE `total` ADD `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry'
+-- ALTER TABLE `total` ADD `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry'
+-- ALTER TABLE `total` ADD `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status'
+-- ALTER TABLE `total` ADD `need_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages need to be sync'
+-- ALTER TABLE `total` ADD `success_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync success at this time'
+-- ALTER TABLE `total` ADD `fail_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync fail at this time'
+-- ALTER TABLE `total` ADD `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync'
+-- ALTER TABLE `total` ADD `last_sync_module` varchar(100) NOT NULL COMMENT 'last sync success module name';
 
 CREATE TABLE `download_total` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
  PRIMARY KEY (`id`),
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+
+CREATE TABLE `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name_deps` (`name`,`deps`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';

docs/web/install.md

@@ -10,14 +10,13 @@
 ## Clone
 
 ```bash
-$ git clone get://github.com/fengmk2/cnpmjs.org.git $HOME/cnpmjs.org
+$ git clone git://github.com/fengmk2/cnpmjs.org.git $HOME/cnpmjs.org
 $ cd $HOME/cnpmjs.org
 ```
 
 ## Create your `config.js`
 
 ```bash
-$ touch config/config.js
 $ vim config/config.js
 ```
 
@@ -47,10 +46,12 @@ module.exports = {
     bucket: "foobucket",
     domain: "http://foobucket.u.qiniudn.com"
   },
+  nfs: null, // you can set a nfs to replace qiniu cdn
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
-    admin: true,
+    admin: 'admin@cnpmjs.org',
   },
+  syncModel: 'exist', //`all` sync all packages, `exist` only update  exist packages, `none` do nothing
 };
 ```
 
@@ -63,6 +64,9 @@ mysql> use cnpmjs;
 mysql> source docs/db.sql
 ```
 
+## Use your own CDN
+If you wan to use your own CDN instead of qiniu. Just look at `common/qnfs.js` and implement the interface like it, then pass it by set `config.nfs`.
+
 ## npm install
 
 ```bash
@@ -86,3 +90,26 @@ $ open http://localhost:7001
 # web
 $ open http://localhost:7002
 ```
+
+## use cnpm cli with your own registry
+You do not need to write another command line tool with your own registry,
+just alias [cnpm](http://github.com/fengmk2/cnpm), then you can get a npm client for you own registry.
+
+```
+# install cnpm first
+npm install -g cnpm
+
+# then alias lnpm to cnpm, but change config to your own registry
+alias lnpm='cnpm --registry=http://localhost:7001\
+ --registryweb=http://localhost:7002\
+ --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://dist.u.qiniudn.com\
+ --userconfig=$HOME/.lnpmrc'
+
+ #or put this in .zshrc or .bashrc
+ echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
+ --registryweb=http://localhost:7002\
+ --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://dist.u.qiniudn.com\
+ --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
+```

docs/web/readme.md

@@ -1,12 +1,12 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
 ## What is this?
 
-> Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+> Private npm registry and web for Enterprise, base on [koa](http://koajs.com/), MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
@@ -16,13 +16,16 @@
 
 ## Registry
 
-* Our public registry: [registry.cnpmjs.org](http://registry.cnpmjs.org)
+* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
 * Current [cnpmjs.org](/) version: <span id="app-version"></span>
 
-Total Packages: <span id="total-packages"></span>
-
 <table class="downloads">
   <tbody>
+    <tr>
+      <td class="count" id="total-packages"></td><td>total packages</td>
+      <td class="count" id="total-versions"></td><td>total package versions</td>
+      <td class="count" id="total-deletes"></td><td>total delete packages</td>
+    </tr>
     <tr>
       <td class="count"></td><td> downloads today</td>
       <td class="count"></td><td> downloads in this week</td>
@@ -36,6 +39,27 @@ Total Packages: <span id="total-packages"></span>
   </tbody>
 </table>
 
+<div class="sync" style="display:none;">
+  <h3>Sync Status</h3>
+  <p id="sync-model"></p>
+  <p>Last sync time is <span id="last-sync-time"></span>. </p>
+  <p class="syncing alert alert-info">The sync worker is working in the backend now. </p>
+  <table class="sync-status">
+    <tbody>
+      <tr>
+        <td><span id="need-sync"></span> packages need to be sync</td>
+        <td class="syncing"><span id="left-sync"></span> packages and dependencies waiting for sync</td>
+        <td><span id="percent-sync"></span>% progress</td>
+      </tr>
+      <tr>
+        <td><span id="success-sync"></span> packages and dependencies sync successed</td>
+        <td><span id="fail-sync"></span> packages and dependencies sync failed</td>
+        <td>last success: <span id="last-success-name"></span></td>
+      </tr>
+    </tbody>
+  </table>
+</div>
+
 Running on [Node.js](http://nodejs.org), version <span id="node-version"></span>.
 
 <script>
@@ -49,18 +73,43 @@ $(function () {
     return n.join(s);
   }
 
-  $.getJSON('http://registry.cnpmjs.org/?callback=?', function (data) {
+  $.getJSON('/total', function (data) {
     $('#total-packages').html(humanize(data.doc_count));
+    $('#total-versions').html(humanize(data.doc_version_count));
+    $('#total-deletes').html(humanize(data.doc_del_count));
+
     var downloads = $('table.downloads');
-    downloads.find('td.count:eq(0)').html(humanize(data.download.today));
-    downloads.find('td.count:eq(1)').html(humanize(data.download.thisweek));
-    downloads.find('td.count:eq(2)').html(humanize(data.download.thismonth));
-    downloads.find('td.count:eq(3)').html(humanize(data.download.lastday));
-    downloads.find('td.count:eq(4)').html(humanize(data.download.lastweek));
-    downloads.find('td.count:eq(5)').html(humanize(data.download.lastmonth));
+    downloads.find('td.count:eq(3)').html(humanize(data.download.today));
+    downloads.find('td.count:eq(4)').html(humanize(data.download.thisweek));
+    downloads.find('td.count:eq(5)').html(humanize(data.download.thismonth));
+    downloads.find('td.count:eq(6)').html(humanize(data.download.lastday));
+    downloads.find('td.count:eq(7)').html(humanize(data.download.lastweek));
+    downloads.find('td.count:eq(8)').html(humanize(data.download.lastmonth));
 
     $('#node-version').html(data.node_version || 'v0.10.22');
     $('#app-version').html(data.app_version || '0.0.0');
+
+    if (data.sync_model === 'all') {
+      $('#sync-model').html('This registry will sync all packages from official registry.');
+      $('#last-sync-time').html(new Date(data.last_sync_time));
+    } else if (data.sync_model === 'exist') {
+      $('#sync-model').html('This registry will only update exist packages from official registry.');
+      $('#last-sync-time').html(new Date(data.last_exist_sync_time));
+    }
+
+    $('#need-sync').html(data.need_sync_num);
+    $('#success-sync').html(data.success_sync_num);
+    $('#fail-sync').html(data.fail_sync_num);
+    $('#left-sync').html(data.left_sync_num);
+    $('#percent-sync').html(Math.floor(data.success_sync_num / data.need_sync_num * 100));
+    $('#last-success-name').html('<a target="_blank" href="/package/' + data.last_sync_module + '">' +
+      data.last_sync_module + '</a>');
+
+    if (!data.sync_status) {
+      $('.syncing').html('');
+    }
+
+    $('.sync').show();
   });
 });
 </script>
@@ -70,11 +119,16 @@ $(function () {
 alias it:
 
 ```bash
-alias cnpm="npm --registry=http://registry.cnpmjs.org --cache=$HOME/.npm/.cache/cnpm"
+alias cnpm="npm --registry=http://r.cnpmjs.org \
+--cache=$HOME/.npm/.cache/cnpm \
+--disturl=http://dist.u.qiniudn.com \
+--userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.cnpmjs.org \
-  --cache=$HOME/.npm/.cache/cnpm"' >> ~/.zshrc && source ~/.zshrc
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
+  --cache=$HOME/.npm/.cache/cnpm \
+  --disturl=http://dist.u.qiniudn.com \
+  --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
 Or you can just use our `cnpm` cli:
@@ -83,58 +137,71 @@ Or you can just use our `cnpm` cli:
 $ npm install cnpm -g
 ```
 
-### adduser
+### install
 
-```bash
-$ cnpm adduser
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+
+```
+$ cnpm install [name]
 ```
 
 ### sync
 
-Only `cnpm` cli has this command:
+Only `cnpm` cli has this command. Meaning sync package from source npm.
 
 ```bash
-$ cnpm sync mocha
+$ cnpm sync connect
 ```
 
-(TODO) sync package web page
+sync package on web: [cnpmjs.org/sync/connect](http://cnpmjs.org/sync/connect)
 
 ```bash
-$ open http://cnpmjs.org/sync
+$ open http://cnpmjs.org/sync/connect
 ```
 
-### publish
+### publish / unpublish
 
-Meaning sync package from source npm.
-
-Only `admin` user can publish package to private registry.
+Only `admin` user can publish / unpublish package to private registry.
 
 ```bash
 $ cnpm publish [name]
+$ cnpm unpublish [name]
+```
+
+### Other commands
+
+Support all the other npm commands. e.g.:
+
+```bash
+$ cnpm info cnpm
 ```
 
 ## TODO list
 
-@see Github [Issues](https://github.com/fengmk2/cnpmjs.org/issues)
+@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
 
 ## Authors
 
+Release [History](/history).
+
 ```bash
 $ git summary
 
  project  : cnpmjs.org
- repo age : 9 days
- active   : 49 days
- commits  : 126
- files    : 70
+ repo age : 7 weeks
+ active   : 132 days
+ commits  : 315
+ files    : 88
  authors  :
-    73  fengmk2                 57.9%
-    53  dead_horse              42.1%
+   190  fengmk2                 60.3%
+   122  dead_horse              38.7%
+     2  4simple                 0.6%
+     1  Alsotang                0.3%
 ```
 
 ## npm and cnpm relation
 
-![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=960&h=720)
+![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
 
 ## 捐赠 Donate
 如果您觉得 [cnpmjs.org] 对您有帮助，欢迎请作者一杯咖啡.

index.js

@@ -0,0 +1,30 @@
+/*!
+ * cnpmjs.org - index.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('./config');
+
+exports.loadConfig = config.loadConfig;
+exports.config = config;
+
+exports.startWorker = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./worker');
+};
+
+exports.startSync = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./sync');
+};

lib/common.js

@@ -0,0 +1,57 @@
+/**!
+ * cnpmjs.org - lib/common.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var path = require('path');
+var config = require('../config');
+var util = require('util');
+
+exports.getTarballFilepath = function (filename) {
+  // ensure download file path unique
+  var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex') + '.tgz');
+  return path.join(config.uploadDir, name);
+};
+
+exports.getCDNKey = function (name, filename) {
+  return '/' + name + '/-/' + filename;
+};
+
+exports.setDownloadURL = function (pkg, ctx, host) {
+  if (pkg.dist) {
+    host = host || ctx.host;
+    pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
+      ctx.protocol,
+      host, pkg.name, pkg.name, pkg.version);
+  }
+};
+
+exports.isAdmin = function (username) {
+  return typeof config.admins[username] === 'string';
+};
+
+exports.isMaintainer = function (ctx, maintainers) {
+  if (ctx.session.isAdmin) {
+    return true;
+  }
+
+  var username = ctx.session.name;
+  maintainers = maintainers || [];
+  var match = maintainers.filter(function (item) {
+    return item.name === username;
+  });
+
+  return match.length > 0;
+};

middleware/auth.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - middleware/auth.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -17,47 +17,49 @@
 var debug = require('debug')('cnpmjs.org:middleware:auth');
 var User = require('../proxy/user');
 var config = require('../config');
+var common = require('../lib/common');
 
 module.exports = function (options) {
-  return function auth(req, res, next) {
-    req.session.onlySync = config.enablePrivate ? true : false;
-    if (req.session.name) {
-      if (config.admins[req.session.name] === true) {
-        req.session.isAdmin = true;
-      }
-      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
-        req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
-      return next();
+  return function *auth(next) {
+    debug('%s, %s, %j', this.url, this.sessionId, this.session);
+    if (!this.session) {
+      // redis crash
+      this.session = {};
+      return yield next;
     }
-    var authorization = (req.headers.authorization || '').split(' ')[1] || '';
+    this.session.onlySync = config.enablePrivate ? true : false;
+    if (this.session.name) {
+      this.session.isAdmin = common.isAdmin(this.session.name);
+      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
+        this.session.name, this.session.onlySync, this.session.isAdmin, this.header);
+      return yield next;
+    }
+    var authorization = (this.get('authorization') || '').split(' ')[1] || '';
     authorization = authorization.trim();
     if (!authorization) {
-      return next();
+      return yield next;
     }
+
     authorization = new Buffer(authorization, 'base64').toString().split(':');
+    if (authorization.length !== 2) {
+      return yield next;
+    }
+
     var username = authorization[0];
     var password = authorization[1];
 
-    User.auth(username, password, function (err, row) {
-      if (err) {
-        return next(err);
-      }
+    var row = yield User.auth(username, password);
+    if (!row) {
+      debug('auth fail user: %j, headers: %j', row, this.header);
+      this.session.name = null;
+      this.session.isAdmin = false;
+      return yield next;
+    }
 
-      if (!row) {
-        debug('auth fail user: %j, headers: %j', row, req.headers);
-        return res.json(401, {
-          error: 'unauthorized',
-          reason: 'Name or password is incorrect.'
-        });
-      }
-
-      req.session.name = row.name;
-      if (config.admins[req.session.name] === true) {
-        req.session.isAdmin = true;
-      }
-      debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
-        row, req.session.onlySync, req.session.isAdmin, req.headers);
-      next();
-    });
+    this.session.name = row.name;
+    this.session.isAdmin = common.isAdmin(this.session.name);
+    debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
+      row, this.session.onlySync, this.session.isAdmin, this.header);
+    yield next;
   };
 };

middleware/login.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
-module.exports = function login(req, res, next) {
-  if (!req.session.name) {
-    return res.json(401, {
+module.exports = function *login(next) {
+  if (!this.session.name) {
+    this.status = 401;
+    this.body = {
       error: 'unauthorized',
       reason: 'Login first.'
-    });
+    };
+    return;
   }
-  next();
+  yield next;
 };

middleware/opensearch.js

@@ -24,8 +24,11 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
 
 var lastModifyDate = new Date();
 
-module.exports = function publishable(req, res, next) {
-  res.charset = res.charset || 'utf-8';
-  res.setHeader('Content-Type', 'text/xml');
-  res.send(template.replace('${host}', req.headers.host));
+module.exports = function *publishable(next) {
+  if (this.path === '/opensearch.xml') {
+    this.type = 'text/xml';
+    this.charset = 'utf-8';
+    this.body = template.replace('${host}', this.host);
+  }
+  yield next;
 };

middleware/publishable.js

@@ -14,13 +14,15 @@
  * Module dependencies.
  */
 
-module.exports = function publishable(req, res, next) {
-  if (req.session.onlySync && !req.session.isAdmin) {
+module.exports = function *publishable(next) {
+  if (this.session.onlySync && !this.session.isAdmin) {
     // private mode, only admin user can publish
-    return res.json(403, {
+    this.status = 403;
+    this.body = {
       error: 'no_perms',
       reason: 'Private mode enable, only admin can publish this module'
-    });
+    };
+    return;
   }
-  next();
+  yield next;
 };

middleware/registry_not_found.js

@@ -0,0 +1,28 @@
+/**!
+ * cnpmjs.org - middleware/registry_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield next;
+
+  if (this.status) {
+    return;
+  }
+  this.status = 404;
+  this.body = {
+    error: 'not_found',
+    reason: 'document not found'
+  };
+};

middleware/sync_by_install.js

@@ -14,19 +14,32 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
 var config = require('../config');
 
 /**
  * req.session.allowSync  -  allow sync triggle by cnpm install
  */
-module.exports = function (req, res, next) {
-  if (!config.syncByInstall) {
-    return next();
+
+module.exports = function *(next) {
+  if (!config.syncByInstall || !config.enablePrivate) {
+    // only config.enablePrivate should enable sync on install
+    return yield next;
   }
   // request not by node, consider it request from web
-  if (req.headers['user-agent'] && req.headers['user-agent'].indexOf('node') !== 0) {
-    return next();
+  if (this.get('user-agent') && this.get('user-agent').indexOf('node') !== 0) {
+    return yield next;
   }
-  req.session.allowSync = true;
-  next();
+
+  this.session.allowSync = true;
+  if (this.session.isAdmin) {
+    // if current user is admin, should not enable auto sync on install, because it would be unpublish
+    this.session.allowSync = false;
+  }
+
+  // TODO: allow sync will let publish sync package...
+  this.session.allowSync = false;
+
+  debug('%s allowSync: %s', this.session.name, this.session.allowSync);
+  yield next;
 };

middleware/web_not_found.js

@@ -0,0 +1,27 @@
+/**!
+ * cnpmjs.org - middleware/web_not_found.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function *notFound(next) {
+  yield next;
+
+  if (this.status) {
+    return;
+  }
+  this.status = 404;
+  this.type = 'text/html';
+  this.charset = 'utf-8';
+  this.body = 'Cannot GET ' + this.path;
+};

package.json

@@ -1,74 +1,79 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.1.0",
-  "description": "Private npm registry and web",
-  "main": "dispatch.js",
+  "version": "0.3.2",
+  "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
+  "main": "index.js",
   "scripts": {
     "test": "make test-all",
+    "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
+    "status": "./bin/nodejsctl status",
+    "stop": "./bin/nodejsctl stop"
+  },
+  "config": {
     "blanket": {
-      "pattern": "//^((?!(node_modules|test|common)).)*$/",
-      "data-cover-flags": {
-        "debug": false
-      }
+      "pattern": "//^((?!(node_modules|test|common)).)*$/"
     },
     "travis-cov": {
       "threshold": 90
     }
   },
   "dependencies": {
-    "humanize-number": "0.0.2",
-    "gravatar": "1.0.6",
-    "urllib": "0.5.5",
-    "connect-markdown": "0.0.3",
-    "qn": "0.2.0",
-    "microtime": "0.5.0",
+    "co-read": "0.0.1",
+    "co-write": "0.3.0",
     "debug": "0.7.4",
-    "utility": "0.1.9",
-    "ready": "0.1.1",
-    "connect": "2.11.2",
-    "connect-rt": "0.0.2",
-    "connect-redis": "1.4.6",
-    "connect-render": "0.3.2",
-    "urlrouter": "0.5.3",
-    "graceful": "0.0.5",
-    "moment": "2.4.0",
+    "eventproxy": "0.2.7",
+    "forward": "0.0.4",
+    "giturl": "0.0.2",
+    "graceful": "0.0.6",
+    "gravatar": "1.0.6",
+    "humanize-number": "0.0.2",
+    "koa": "0.5.0",
+    "koa-markdown": "0.0.2",
+    "koa-middlewares": "0.0.6",
     "logfilestream": "0.1.0",
-    "ms": "0.6.1",
+    "marked": "0.3.1",
+    "microtime": "0.5.1",
+    "mime": "1.2.11",
     "mkdirp": "0.3.5",
-    "mysql": "2.0.0-rc1",
-    "response-patch": "0.1.1",
-    "response-cookie": "0.0.2",
-    "bagpipe": "0.3.5",
+    "moment": "2.5.1",
+    "ms": "0.6.2",
+    "mysql": "2.1.0",
+    "nodemailer": "0.6.1",
+    "qn": "0.2.0",
+    "ready": "0.1.1",
     "semver": "2.2.1",
-    "marked": "0.2.10",
-    "eventproxy": "0.2.6"
+    "thunkify-wrap": "0.0.1",
+    "urllib": "0.5.5",
+    "utility": "0.1.10"
   },
   "devDependencies": {
-    "supertest": "*",
-    "should": "*",
+    "autod": ">=0.0.13",
     "blanket": "*",
-    "travis-cov": "*",
+    "contributors": "*",
     "coveralls": "*",
-    "mocha-lcov-reporter": "*",
+    "mm": "0.2.0",
     "mocha": "*",
-    "mm": "*",
-    "pedding": "*"
+    "mocha-lcov-reporter": "*",
+    "pedding": "0.0.3",
+    "should": "3.1.3",
+    "supertest": "0.9.0",
+    "travis-cov": "*"
   },
-  "homepage": "https://github.com/fengmk2/cnpmjs.org",
+  "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
     "type": "git",
-    "url": "git://github.com/fengmk2/cnpmjs.org.git",
-    "web": "https://github.com/fengmk2/cnpmjs.org"
+    "url": "git://github.com/cnpm/cnpmjs.org.git",
+    "web": "https://github.com/cnpm/cnpmjs.org"
   },
   "bugs": {
-    "url": "https://github.com/fengmk2/cnpmjs.org/issues",
+    "url": "https://github.com/cnpm/cnpmjs.org/issues",
     "email": "fengmk2@gmail.com"
   },
   "keywords": [
     "cnpmjs.org", "npm", "npmjs", "npmjs.org", "registry"
   ],
   "engines": {
-    "node": ">= 0.8.0"
+    "node": ">= 0.11.9"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

proxy/download.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var mysql = require('../common/mysql');
 
@@ -41,3 +42,5 @@ var SELECT_ALL_TOTAL_SQL = 'SELECT date, sum(count) AS count \
 exports.getTotal = function (start, end, callback) {
   mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
 };
+
+thunkify(exports);

proxy/module.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var eventproxy = require('eventproxy');
 var config = require('../config');
@@ -21,9 +22,6 @@ var mysql = require('../common/mysql');
 
 var MODULE_COLUMNS = 'id, publish_time, gmt_create, gmt_modified, author, name, version, description, package, dist_tarball, dist_shasum, dist_size';
 
-// var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, author, name, version, package, dist_tarball, dist_shasum, dist_size) \
-//   VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?);';
-
 var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
   publish_time, author, name, version, package, dist_tarball, dist_shasum, dist_size, description) \
   VALUES(now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?) \
@@ -32,9 +30,10 @@ var INSERT_MODULE_SQL = 'INSERT INTO module(gmt_create, gmt_modified, \
     dist_tarball=VALUES(dist_tarball), dist_shasum=VALUES(dist_shasum), dist_size=VALUES(dist_size);';
 
 exports.add = function (mod, callback) {
+  var keywords = mod.package.keywords;
   var pkg;
   try {
-    pkg = JSON.stringify(mod.package);
+    pkg = stringifyPackage(mod.package);
   } catch (e) {
     return callback(e);
   }
@@ -54,6 +53,74 @@ exports.add = function (mod, callback) {
       return callback(err);
     }
     callback(null, {id: result.insertId, gmt_modified: new Date()});
+
+    if (typeof keywords === 'string') {
+      keywords = [keywords];
+    }
+
+    if (!Array.isArray(keywords)) {
+      return;
+    }
+
+    var words = [];
+    for (var i = 0; i < keywords.length; i++) {
+      var w = keywords[i];
+      if (typeof w === 'string') {
+        w = w.trim();
+        if (w) {
+          words.push(w);
+        }
+      }
+    }
+
+    if (words.length === 0) {
+      return;
+    }
+
+    // add keywords
+    exports.addKeywords(mod, description, words, utility.noop);
+  });
+};
+
+var GET_KEYWORD_SQL = 'SELECT keyword FROM module_keyword WHERE name=? ORDER BY keyword;';
+
+exports.getKeywords = function (name, callback) {
+  mysql.query(GET_KEYWORD_SQL, [name], function (err, rows) {
+    var keywords = [];
+    if (rows && rows.length) {
+      keywords = rows.map(function (r) {
+        return r.keyword;
+      });
+    }
+    callback(err, keywords);
+  });
+};
+
+var ADD_KEYWORD_SQL = 'INSERT INTO module_keyword(gmt_create, keyword, name, description) \
+  VALUES(now(), ?, ?, ?) \
+  ON DUPLICATE KEY UPDATE description=VALUES(description);';
+
+exports.addKeywords = function (name, description, keywords, callback) {
+  var sql = '';
+  var values = [];
+  for (var i = 0; i < keywords.length; i++) {
+    sql += ADD_KEYWORD_SQL;
+    values.push(keywords[i]);
+    values.push(name);
+    values.push(description);
+  }
+  mysql.query(sql, values, function (err, results) {
+    if (err) {
+      return callback(err);
+    }
+    var ids = [];
+    for (var i = 0; i < results.length; i++) {
+      var r = results[i];
+      if (r.insertId) {
+        ids.push(r.insertId);
+      }
+    }
+    callback(null, ids);
   });
 };
 
@@ -62,17 +129,32 @@ exports.updateDescription = function (id, description, callback) {
   mysql.query(UPDATE_DESC_SQL, [description, id], callback);
 };
 
-var UPDATE_DIST_SQL = 'UPDATE module SET version=?, package=?, dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
+var UPDATE_PACKAGE_SQL = 'UPDATE module SET package=? WHERE id=?;';
+exports.updateReadme = function (id, readme, callback) {
+  exports.getById(id, function (err, data) {
+    if (err) {
+      return callback(err);
+    }
+    data.package = data.package || {};
+    data.package.readme = readme;
+    var pkg = stringifyPackage(data.package);
+    mysql.query(UPDATE_PACKAGE_SQL, [pkg, id], callback);
+  });
+};
+
+var UPDATE_DIST_SQL = 'UPDATE module SET publish_time=?, version=?, package=?, \
+  dist_tarball=?, dist_shasum=?, dist_size=? WHERE id=?;';
 
 exports.update = function (mod, callback) {
   var pkg;
   try {
-    pkg = JSON.stringify(mod.package);
+    pkg = stringifyPackage(mod.package);
   } catch (e) {
     return callback(e);
   }
   var dist = mod.package.dist;
-  mysql.query(UPDATE_DIST_SQL, [mod.version, pkg, dist.tarball, dist.shasum, dist.size, mod.id],
+  mysql.query(UPDATE_DIST_SQL,
+    [mod.publish_time, mod.version, pkg, dist.tarball, dist.shasum, dist.size, mod.id],
   function (err, result) {
     if (err) {
       return callback(err);
@@ -82,7 +164,22 @@ exports.update = function (mod, callback) {
 };
 
 function parseRow(row) {
-  row.package = JSON.parse(row.package);
+  if (row && row.package) {
+    try {
+      if (row.package.indexOf('%7B%22') === 0) {
+        // now store package will encodeURIComponent() after JSON.stringify
+        row.package = decodeURIComponent(row.package);
+      }
+      row.package = JSON.parse(row.package);
+    } catch (e) {
+      console.warn('parse package error: %s, id: %s version: %s, error: %s', row.name, row.id, row.version, e);
+    }
+  }
+}
+exports.parseRow = parseRow;
+
+function stringifyPackage(pkg) {
+  return encodeURIComponent(JSON.stringify(pkg));
 }
 
 var SELECT_MODULE_BY_ID_SQL = 'SELECT ' + MODULE_COLUMNS + ' FROM module WHERE id=?;';
@@ -160,7 +257,12 @@ exports.removeTags = function (name, callback) {
   mysql.query(DELETE_TAGS_SQL, [name], callback);
 };
 
-var SELECT_ALL_TAGS_SQL = 'SELECT tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
+var DELETE_TAGS_BY_IDS_SQL = 'DELETE FROM tag WHERE id in (?)';
+exports.removeTagsByIds = function (ids, callback) {
+  mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
+};
+
+var SELECT_ALL_TAGS_SQL = 'SELECT id, tag, version, gmt_modified, module_id FROM tag WHERE name=?;';
 
 exports.listTags = function (name, callback) {
   mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
@@ -212,19 +314,37 @@ exports.listByName = function (name, callback) {
   });
 };
 
-var LIST_SINCE_SQL = 'SELECT name, package FROM module WHERE id IN\
-                      (SELECT module_id FROM tag WHERE tag="latest" AND name IN\
-                      (SELECT distinct(name) FROM module WHERE gmt_modified > ?))\
-                      ORDER BY name';
+var LIST_SINCE_SQLS = [
+  'SELECT module_id FROM tag WHERE tag="latest" AND gmt_modified>?',
+  'SELECT name, package FROM module WHERE id IN (?);'
+];
 exports.listSince = function (start, callback) {
-  mysql.query(LIST_SINCE_SQL, [start], callback);
+  var ep = eventproxy.create();
+  ep.fail(callback);
+  mysql.query(LIST_SINCE_SQLS[0], [new Date(start)], ep.done(function (rows) {
+    if (!rows || rows.length === 0) {
+      return callback(null, []);
+    }
+    ep.emit('ids', rows.map(function (r) {
+      return r.module_id;
+    }));
+  }));
+
+  ep.once('ids', function (ids) {
+    mysql.query(LIST_SINCE_SQLS[1], [ids], callback);
+  });
 };
 
-var LIST_SHORT_SQL = 'SELECT distinct(name) FROM module ORDER BY name';
+var LIST_SHORT_SQL = 'SELECT distinct(name) FROM tag ORDER BY name';
 exports.listShort = function (callback) {
   mysql.query(LIST_SHORT_SQL, callback);
 };
 
+var LIST_ALL_MODULE_NAMES_SQL = 'SELECT distinct(name) FROM module ORDER BY name';
+exports.listAllModuleNames = function (callback) {
+  mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
+};
+
 var DELETE_MODULE_BY_NAME_SQL = 'DELETE FROM module WHERE name=?;';
 exports.removeByName = function (name, callback) {
   mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
@@ -235,15 +355,15 @@ exports.removeByNameAndVersions = function (name, versions, callback) {
   mysql.query(DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL, [name, versions], callback);
 };
 
-var LIST_RECENTLY_NAMES_SQL = 'SELECT distinct(name) AS name FROM module WHERE author = ? ORDER BY publish_time DESC LIMIT 100;';
-var LIST_BY_NAMES_SQL = 'SELECT name, description FROM module WHERE id IN \
-  ( \
-    SELECT module_id FROM tag WHERE tag="latest" AND name IN (?) \
-  ) ORDER BY publish_time DESC;';
+var LIST_BY_AUTH_SQLS = [
+  'SELECT distinct(name) AS name FROM module WHERE author = ? ORDER BY publish_time DESC LIMIT 100;',
+  'SELECT module_id FROM tag WHERE tag="latest" AND name IN (?)',
+  'SELECT name, description FROM module WHERE id IN (?) ORDER BY publish_time DESC'
+];
 exports.listByAuthor = function (author, callback) {
   var ep = eventproxy.create();
   ep.fail(callback);
-  mysql.query(LIST_RECENTLY_NAMES_SQL, [author], ep.done(function (rows) {
+  mysql.query(LIST_BY_AUTH_SQLS[0], [author], ep.done(function (rows) {
     if (!rows || rows.length === 0) {
       return callback(null, []);
     }
@@ -252,16 +372,83 @@ exports.listByAuthor = function (author, callback) {
     }));
   }));
   ep.on('names', function (names) {
-    mysql.query(LIST_BY_NAMES_SQL, [names], ep.done('modules'));
+    mysql.query(LIST_BY_AUTH_SQLS[1], [names], ep.done(function (rows) {
+      if (!rows || rows.length === 0) {
+        return callback(null, []);
+      }
+      ep.emit('ids', rows.map(function (r) {
+        return r.module_id;
+      }));
+    }));
   });
-  ep.on('modules', function (modules) {
-    callback(null, modules);
+  ep.on('ids', function (ids) {
+    mysql.query(LIST_BY_AUTH_SQLS[2], [ids], callback);
   });
 };
 
-var SEARCH_SQL = 'SELECT name, description FROM module WHERE id IN\
-  (SELECT module_id FROM tag WHERE name LIKE ? AND  tag="latest")\
-  ORDER BY name LIMIT 20';
-exports.search = function (word, callback) {
-  mysql.query(SEARCH_SQL, [word + '%'], callback);
+var SEARCH_MODULES_SQL = 'SELECT module_id FROM tag WHERE name LIKE ? AND tag="latest" ORDER BY name LIMIT ?;';
+var SEARCH_MODULES_BY_KEYWORD_SQL = 'SELECT name, description FROM module_keyword WHERE keyword = ? ORDER BY id DESC LIMIT ?;';
+var QUERY_MODULES_BY_ID_SQL = 'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;';
+
+exports.search = function (word, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
+  var limit = options.limit || 100;
+  word = word.replace(/^%/, ''); //ignore prefix %
+  var ep = eventproxy.create();
+  ep.fail(callback);
+
+  // search flows:
+  // 1. prefix search by name
+  // 2. like search by name
+  // 3. keyword equal search
+  var ids = {};
+
+  mysql.query(SEARCH_MODULES_SQL, [word + '%', limit ], ep.done(function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
+    }
+    if (rows.length >= 20) {
+      return ep.emit('ids', Object.keys(ids));
+    }
+
+    mysql.query(SEARCH_MODULES_SQL, [ '%' + word + '%', limit ], ep.done('likeSearch'));
+  }));
+
+  mysql.query(SEARCH_MODULES_BY_KEYWORD_SQL, [ word, limit ], ep.done('keywordRows'));
+
+  ep.on('likeSearch', function (rows) {
+    rows = rows || [];
+    if (rows.length > 0) {
+      for (var i = 0; i < rows.length; i++) {
+        ids[rows[i].module_id] = 1;
+      }
+    }
+
+    ep.emit('ids', Object.keys(ids));
+  });
+
+  ep.all('ids', 'keywordRows', function (ids, keywordRows) {
+    keywordRows = keywordRows || [];
+    var data = {
+      keywordMatchs: keywordRows,
+      searchMatchs: []
+    };
+    if (ids.length === 0) {
+      return callback(null, data);
+    }
+
+    mysql.query(QUERY_MODULES_BY_ID_SQL, [ids], ep.done(function (modules) {
+      data.searchMatchs = modules;
+      callback(null, data);
+    }));
+  });
 };
+
+thunkify(exports);

proxy/module_deps.js

@@ -0,0 +1,44 @@
+/**!
+ * cnpmjs.org - proxy/module_deps.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var thunkify = require('thunkify-wrap');
+var mysql = require('../common/mysql');
+
+var LIST_DEPS_SQL = 'SELECT deps FROM module_deps WHERE name=?;';
+
+exports.list = function (name, callback) {
+  mysql.query(LIST_DEPS_SQL, [name], callback);
+};
+
+var INSERT_DEPS_SQL = 'INSERT INTO module_deps(gmt_create, name, deps) \
+  VALUES(now(), ?, ?);';
+
+exports.add = function (name, deps, callback) {
+  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
+    if (err && err.code === 'ER_DUP_ENTRY') {
+      err = null;
+    }
+    callback(err);
+  });
+};
+
+var DELETE_DEPS_SQL = 'DELETE FROM module_deps WHERE name=? AND deps=?;';
+
+exports.remove = function (name, deps, callback) {
+  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
+};
+
+thunkify(exports);

proxy/module_log.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var mysql = require('../common/mysql');
 
 var INSERT_LOG_SQL = 'INSERT INTO module_log(gmt_create, gmt_modified, name, username, log) \
@@ -40,3 +41,5 @@ var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?;';
 exports.get = function (id, callback) {
   mysql.queryOne(SELECT_SQL, [id], callback);
 };
+
+thunkify(exports);

proxy/npm.js

@@ -14,16 +14,29 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var urllib = require('urllib');
 var config = require('../config');
 
-function request(url, callback) {
+function request(url, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
+  options.dataType = options.dataType || 'json';
+  options.timeout = options.timeout || 120000;
   url = config.sourceNpmRegistry + url;
-  var options = {
-    dataType: 'json',
-    timeout: 10000
-  };
-  urllib.request(url, options, callback);
+  urllib.request(url, options, function (err, data, res) {
+    if (err) {
+      var statusCode = res && res.statusCode || -1;
+      if (err.name === 'JSONResponseFormatError' && statusCode >= 500) {
+        err.name = 'NPMServerError';
+        err.message = 'Status ' + statusCode + ', ' + (data && data.toString() || 'empty body');
+      }
+    }
+    callback(err, data, res);
+  });
 }
 
 exports.get = function (name, callback) {
@@ -38,3 +51,19 @@ exports.get = function (name, callback) {
     callback(null, data, res);
   });
 };
+
+exports.getAllSince = function (startkey, callback) {
+  request('/-/all/since?stale=update_after&startkey=' + startkey, {
+    dataType: 'json',
+    timeout: 300000
+  }, callback);
+};
+
+exports.getShort = function (callback) {
+  request('/-/short', {
+    dataType: 'json',
+    timeout: 300000
+  }, callback);
+};
+
+thunkify(exports);

proxy/sync_module_worker.js

@@ -1,11 +1,12 @@
 /**!
- * cnpmjs.org - controllers/sync_module_worker.js
+ * cnpmjs.org - proxy/sync_module_worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
  *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
  */
 
 'use strict';
@@ -14,7 +15,8 @@
  * Module dependencies.
  */
 
-var debug = require('debug')('cnpmjs.org:controllers:registry:sync_module_worker');
+var thunkify = require('thunkify-wrap');
+var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
 var util = require('util');
 var fs = require('fs');
@@ -23,20 +25,40 @@ var crypto = require('crypto');
 var eventproxy = require('eventproxy');
 var urllib = require('urllib');
 var utility = require('utility');
-var nfs = require('../common/nfs');
-var npm = require('../proxy/npm');
-var common = require('./common');
-var Module = require('../proxy/module');
-var Log = require('../proxy/module_log');
 var ms = require('ms');
+var nfs = require('../common/nfs');
+var npm = require('./npm');
+var common = require('../lib/common');
+var Module = require('./module');
+var ModuleDeps = require('./module_deps');
+var Log = require('./module_log');
+var config = require('../config');
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
   this._logId = options.logId;
   this.startName = options.name;
-  this.names = [options.name];
+  if (!Array.isArray(options.name)) {
+    options.name = [options.name];
+  }
+
+  this.names = options.name;
+  // for (var i = 0; i < this.names.length; i++) {
+  //   // ensure package name is lower case
+  //   this.names[i] = this.names[i].toLowerCase();
+  // }
+
   this.username = options.username;
+  this.concurrency = options.concurrency || 1;
+  this._publish = options.publish; // _publish_on_cnpm
+
+  this.syncingNames = {};
   this.nameMap = {};
+  this.names.forEach(function (name) {
+    this.nameMap[name] = true;
+  }.bind(this));
+  this.noDep = options.noDep; //do not sync dependences
+
   this.successes = [];
   this.fails = [];
 }
@@ -46,22 +68,30 @@ util.inherits(SyncModuleWorker, EventEmitter);
 module.exports = SyncModuleWorker;
 
 SyncModuleWorker.prototype.finish = function () {
+  debug('syncingNames: %j', this.syncingNames);
+  if (this._finished || Object.keys(this.syncingNames).length > 0) {
+    return;
+  }
   this.log('[done] Sync %s module finished, %d success, %d fail\nSuccess: [ %s ]\nFail: [ %s ]',
     this.startName,
     this.successes.length, this.fails.length,
     this.successes.join(', '), this.fails.join(', '));
   this.emit('end');
+  this._finished = true;
 };
 
 SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
   var str = '[' + utility.YYYYMMDDHHmmss() + '] ' + util.format.apply(util, arguments);
   debug(str);
-  Log.append(this._logId, str, utility.noop);
+  this._logId && Log.append(this._logId, str, utility.noop);
 };
 
 SyncModuleWorker.prototype.start = function () {
-  this.log('user: %s, sync %s worker start.', this.username, this.names[0]);
-  this.next();
+  this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
+    this.username, this.names[0], this.concurrency, this.noDep, this._publish);
+  for (var i = 0; i < this.concurrency; i++) {
+    this.next(i);
+  }
 };
 
 SyncModuleWorker.prototype.add = function (name) {
@@ -73,50 +103,73 @@ SyncModuleWorker.prototype.add = function (name) {
   this.log('    add dependencies: %s', name);
 };
 
-SyncModuleWorker.prototype.next = function () {
+SyncModuleWorker.prototype.next = function (concurrencyId) {
   var name = this.names.shift();
   if (!name) {
-    return this.finish();
+    return process.nextTick(this.finish.bind(this));
   }
 
   var that = this;
+  that.syncingNames[name] = true;
   npm.get(name, function (err, pkg, response) {
+    var statusCode = response && response.statusCode || -1;
     if (!err && !pkg) {
-      err = new Error('Module ' + name + ' not exists, http status ' + response.statusCode);
+      err = new Error('Module ' + name + ' not exists, http status ' + statusCode);
       err.name = 'NpmModuleNotExsitsError';
     }
     if (err) {
-      that.fails.push(name);
+      if (statusCode === 404) {
+        that.successes.push(name);
+      } else {
+        that.fails.push(name);
+      }
       that.log('[error] [%s] get package error: %s', name, err.stack);
-      return that.next();
+      delete that.syncingNames[name];
+      return that.next(concurrencyId);
     }
 
-    that.log('[%s] start...', pkg.name);
-    that._sync(pkg, function (err, versions) {
+    that.log('[c#%d] [%s] start...', concurrencyId, name);
+    that._sync(name, pkg, function (err, versions) {
+      delete that.syncingNames[name];
       if (err) {
-        that.fails.push(pkg.name);
+        that.fails.push(name);
         that.log('[error] [%s] sync error: %s', name, err.stack);
-        return that.next();
+        return that.next(concurrencyId);
       }
       that.log('[%s] synced success, %d versions: %s',
         name, versions.length, versions.join(', '));
       that.successes.push(name);
-      that.next();
+      that.emit('success', name);
+      that.next(concurrencyId);
     });
   });
 };
 
-SyncModuleWorker.prototype._sync = function (pkg, callback) {
-  var username = this.name;
-  var name = pkg.name;
+SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
+  var username = this.username;
   var that = this;
   var ep = eventproxy.create();
   ep.fail(callback);
 
+  var hasModules = false;
   Module.listByName(name, ep.done(function (rows) {
+    hasModules = rows.length > 0;
     var map = {};
     for (var i = 0; i < rows.length; i++) {
       var r = rows[i];
+      if (!r.package || !r.package.dist) {
+        // package json parse error
+        continue;
+      }
+
+      if (r.package && r.package._publish_on_cnpm) {
+        // publish on cnpm, dont sync this version package
+        that.log('  [%s] publish on local cnpm, don\'t sync', name);
+        ep.unbind();
+        callback(null, []);
+        return;
+      }
+
       if (r.version === 'next') {
         continue;
       }
@@ -144,25 +197,70 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
   var missingVersions = [];
   var missingTags = [];
   var missingDescriptions = [];
+  var missingReadmes = [];
+
   ep.all('existsMap', 'existsTags', function (map, tags) {
     var times = pkg.time || {};
+    pkg.versions = pkg.versions || {};
+    var versionNames = Object.keys(times);
+    if (versionNames.length === 0) {
+      versionNames = Object.keys(pkg.versions);
+    }
+    if (versionNames.length === 0) {
+      that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
+      if (!hasModules) {
+        // save a next module
+        var maintainer = pkg.maintainers && pkg.maintainers[0];
+        if (maintainer && maintainer.name) {
+          maintainer = maintainer.name;
+        }
+        if (!maintainer) {
+          maintainer = '-';
+        }
+        var nextMod = {
+          name: name,
+          version: 'next',
+          author: maintainer,
+          package: {
+            name: name,
+            version: 'next',
+            description: pkg.description || '',
+            readme: pkg.readme || '',
+            maintainers: pkg.maintainers || {
+              name: maintainer
+            },
+          },
+        };
+        Module.add(nextMod, function (err, result) {
+          that.log('  [%s] save next module, %j, error: %s', name, result, err);
+        });
+      }
+    }
     var versions = [];
-    for (var v in times) {
+    for (var i = 0; i < versionNames.length; i++) {
+      var v = versionNames[i];
+      var exists = map[v] || {};
       var version = pkg.versions[v];
       if (!version || !version.dist) {
         continue;
       }
-      var exists = map[v] || {};
-
-      version.publish_time = Date.parse(times[v]);
+      //patch for readme
+      if (!version.readme) {
+        version.readme = pkg.readme;
+      }
+      var publish_time = times[v];
+      version.publish_time = publish_time ? Date.parse(publish_time) : null;
       if (!version.maintainers || !version.maintainers[0]) {
         version.maintainers = pkg.maintainers;
       }
       var sourceAuthor = version.maintainers && version.maintainers[0] && version.maintainers[0].name || exists.author;
+
       if (exists.package && exists.package.dist.shasum === version.dist.shasum && exists.author === sourceAuthor) {
         // * author make sure equal
         // * shasum make sure equal
-        if (version.publish_time === exists.publish_time) {
+        if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
+          debug('  [%s] %s publish_time equal: %s, %s',
+            name, version.version, version.publish_time, exists.publish_time);
           // * publish_time make sure equal
           if (exists.description === null && version.description) {
             // * make sure description exists
@@ -171,6 +269,14 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
               description: version.description
             });
           }
+
+          if (!exists.package.readme && version.readme) {
+            // * make sure readme exists
+            missingReadmes.push({
+              id: exists.id,
+              readme: version.readme
+            });
+          }
           continue;
         }
       }
@@ -186,7 +292,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
     }
 
     if (versions.length === 0) {
-      that.log('  [%s] all versions are exists', pkg.name);
+      that.log('  [%s] all versions are exists', name);
       return ep.emit('syncDone');
     }
 
@@ -194,7 +300,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return a.publish_time - b.publish_time;
     });
     missingVersions = versions;
-    that.log('  [%s] %d versions', pkg.name, versions.length);
+    that.log('  [%s] %d versions', name, versions.length);
     ep.emit('syncModule', missingVersions.shift());
   });
 
@@ -223,14 +329,15 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return ep.emit('descriptionDone');
     }
 
-    that.log('  [%s] saving %d descriptions', pkg.name, missingDescriptions.length);
+    that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
     missingDescriptions.forEach(function (item) {
       Module.updateDescription(item.id, item.description, function (err, result) {
         if (err) {
-          return that.log('    save error, %s: description %j, error: %s', item.id, item.description, err);
+          that.log('    save error, id： %s, description: %s, error: %s', item.id, item.description, err);
+        } else {
+          that.log('    saved, id: %s, description length: %d', item.id, item.description.length);
         }
-        that.log('    saved, id: %s, description length: %d', item.id, item.description.length);
-        ep.emit('saveDescription');
+        ep.emitLater('saveDescription');
       });
     });
 
@@ -244,12 +351,12 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return ep.emit('tagDone');
     }
 
-    that.log('  [%s] adding %d tags', pkg.name, missingTags.length);
+    that.log('  [%s] adding %d tags', name, missingTags.length);
     // sync tags
     missingTags.forEach(function (item) {
-      Module.addTag(pkg.name, item[0], item[1], ep.done(function (result) {
+      Module.addTag(name, item[0], item[1], ep.done(function (result) {
         that.log('    added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
-        ep.emit('addTag');
+        ep.emitLater('addTag');
       }));
     });
 
@@ -258,7 +365,29 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
     });
   });
 
-  ep.all('tagDone', 'descriptionDone', function () {
+  ep.once('syncDone', function () {
+    if (missingReadmes.length === 0) {
+      return ep.emit('readmeDone');
+    }
+
+    that.log('  [%s] saving %d readmes', name, missingReadmes.length);
+    missingReadmes.forEach(function (item) {
+      Module.updateReadme(item.id, item.readme, function (err, result) {
+        if (err) {
+          that.log('    save error, id: %s, error: %s', item.id, err);
+        } else {
+          that.log('    saved, id: %s', item.id);
+        }
+        ep.emitLater('saveReadme');
+      });
+    });
+
+    ep.after('saveReadme', missingReadmes.length, function () {
+      ep.emit('readmeDone');
+    });
+  });
+
+  ep.all('tagDone', 'descriptionDone', 'readmeDone', function () {
     // TODO: set latest version
     callback(null, versionNames);
   });
@@ -283,17 +412,37 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     callback(err);
   });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j',
-    sourcePackage.name, versionIndex, sourcePackage.version, sourcePackage.dist);
+  var dependencies = Object.keys(sourcePackage.dependencies || {});
+  var devDependencies = Object.keys(sourcePackage.devDependencies || {});
 
-  for (var k in sourcePackage.dependencies) {
-    that.add(k);
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s, dependencies: %d, devDependencies: %d',
+    sourcePackage.name, versionIndex, sourcePackage.version,
+    sourcePackage.dist, that.noDep, that._publish,
+    dependencies.length, devDependencies.length);
+
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
   }
 
-  for (var k in sourcePackage.devDependencies) {
-    that.add(k);
+  if (devDependencies.length > config.maxDependencies) {
+    devDependencies = devDependencies.slice(0, config.maxDependencies);
   }
 
+  if (!that.noDep) {
+    for (var i = 0; i < dependencies.length; i++) {
+      that.add(dependencies[i]);
+    }
+
+    for (var i = 0; i < devDependencies.length; i++) {
+      that.add(devDependencies[i]);
+    }
+  }
+
+  // add deps relations
+  dependencies.forEach(function (depName) {
+    ModuleDeps.add(depName, sourcePackage.name, utility.noop);
+  });
+
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
   urllib.request(downurl, options, ep.done(function (_, response) {
@@ -330,8 +479,12 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
         return ep.emit('error', err);
       }
 
-      var key = common.getCDNKey(sourcePackage.name, filename);
-      nfs.upload(filepath, {key: key, size: dataSize}, ep.done('uploadResult'));
+      var options = {
+        key: common.getCDNKey(sourcePackage.name, filename),
+        size: dataSize,
+        shasum: shasum
+      };
+      nfs.upload(filepath, options, ep.done('uploadResult'));
     });
   }));
 
@@ -353,24 +506,44 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
       author: author,
       publish_time: sourcePackage.publish_time,
     };
+
+    if (that._publish) {
+      // sync as publish
+      mod.package._publish_on_cnpm = true;
+    }
+
     var dist = {
-      tarball: result.url,
       shasum: shasum,
       size: dataSize,
       noattachment: dataSize === 0,
     };
+
+    if (result.url) {
+      dist.tarball = result.url;
+    } else if (result.key) {
+      dist.key = result.key;
+      dist.tarball = result.key;
+    }
+
     mod.package.dist = dist;
     Module.add(mod, ep.done(function (result) {
-      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j',
+      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j, publish on cnpm: %s',
         sourcePackage.name, versionIndex,
         result.id,
-        author, mod.version, dataSize, new Date(mod.publish_time));
+        author, mod.version, dataSize,
+        new Date(mod.publish_time),
+        that._publish);
       callback(null, result);
     }));
   });
 };
 
-SyncModuleWorker.sync = function (name, username, callback) {
+SyncModuleWorker.sync = function (name, username, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
   npm.get(name, function (err, pkg, response) {
     if (err) {
       return callback(err);
@@ -390,6 +563,8 @@ SyncModuleWorker.sync = function (name, username, callback) {
         logId: result.id,
         name: name,
         username: username,
+        noDep: options.noDep,
+        publish: options.publish,
       });
       worker.start();
       callback(null, {
@@ -400,3 +575,5 @@ SyncModuleWorker.sync = function (name, username, callback) {
     });
   });
 };
+
+SyncModuleWorker.sync = thunkify(SyncModuleWorker.sync);

proxy/total.js

@@ -14,9 +14,10 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
+var eventproxy = require('eventproxy');
 var config = require('../config');
 var mysql = require('../common/mysql');
-var eventproxy = require('eventproxy');
 
 var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
   FROM information_schema.tables \
@@ -58,6 +59,14 @@ exports.get = function (callback) {
       doc_version_count: vc.count,
       user_count: uc.count,
       store_engine: 'mysql',
+      sync_status: info.sync_status,
+      need_sync_num: info.need_sync_num || 0,
+      success_sync_num: info.success_sync_num || 0,
+      fail_sync_num: info.fail_sync_num || 0,
+      left_sync_num: info.left_sync_num || 0,
+      last_sync_time: info.last_sync_time || 0,
+      last_exist_sync_time: info.last_exist_sync_time || 0,
+      last_sync_module: info.last_sync_module || '',
     };
 
     for (var i = 0; i < sizes.length; i++) {
@@ -79,3 +88,36 @@ var PLUS_DELETE_MODULE_SQL = 'UPDATE total SET module_delete=module_delete+1 WHE
 exports.plusDeleteModule = function (callback) {
   mysql.query(PLUS_DELETE_MODULE_SQL, callback);
 };
+
+exports.getTotalInfo = function (callback) {
+  mysql.queryOne(TOTAL_INFO_SQL, callback);
+};
+
+var SET_LAST_SYNC_TIME_SQL = 'UPDATE total SET last_sync_time=? WHERE name="total";';
+exports.setLastSyncTime = function (time, callback) {
+  mysql.query(SET_LAST_SYNC_TIME_SQL, Number(time), callback);
+};
+
+var SET_LAST_EXIST_SYNC_TIME_SQL = 'UPDATE total SET last_exist_sync_time=? WHERE name="total";';
+exports.setLastExistSyncTime = function (time, callback) {
+  mysql.query(SET_LAST_EXIST_SYNC_TIME_SQL, Number(time), callback);
+};
+
+var UPDATE_SYNC_STATUS_SQL = 'UPDATE total SET sync_status = ? WHERE name="total";';
+exports.updateSyncStatus = function (status, callback) {
+  mysql.query(UPDATE_SYNC_STATUS_SQL, [status], callback);
+};
+
+var UPDATE_SYNC_NUM_SQL = 'UPDATE total SET sync_status = ?, need_sync_num = ?, \
+  success_sync_num = ?, fail_sync_num = ?, left_sync_num = ?, last_sync_module=? \
+  WHERE name="total";';
+exports.updateSyncNum = function (params, callback) {
+  var query = [
+    params.syncStatus, params.need || 0,
+    params.success || 0, params.fail || 0, params.left || 0,
+    params.lastSyncModule,
+  ];
+  mysql.query(UPDATE_SYNC_NUM_SQL, query, callback);
+};
+
+thunkify(exports);

proxy/user.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var thunkify = require('thunkify-wrap');
 var utility = require('utility');
 var config = require('../config');
 var mysql = require('../common/mysql');
@@ -29,7 +30,6 @@ function sha1(s) {
 function passwordSha(password, salt) {
   return sha1(password + salt);
 }
-exports.passwordSha = passwordSha;
 
 exports.get = function (name, callback) {
   mysql.queryOne(SELECT_USER_SQL, [name], function (err, row) {
@@ -105,3 +105,5 @@ exports.update = function (user, callback) {
   });
 };
 
+thunkify(exports);
+exports.passwordSha = passwordSha;

routes/registry.js

@@ -15,16 +15,17 @@
  * Module dependencies.
  */
 
+var middlewares = require('koa-middlewares');
 var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
-var home = require('../controllers/registry/home');
+var total = require('../controllers/total');
 var mod = require('../controllers/registry/module');
 var user = require('../controllers/registry/user');
 var sync = require('../controllers/sync');
 
 function routes(app) {
-  app.get('/', home.show);
+  app.get('/', middlewares.jsonp(), total.show);
 
   //before /:name/:version
   //get all modules, for npm search
@@ -34,10 +35,10 @@ function routes(app) {
   app.get('/-/short', mod.listAllModuleNames);
 
   // module
-  app.get('/:name', [syncByInstall], mod.show);
-  app.get('/:name/:version', [syncByInstall], mod.get);
+  app.get('/:name', syncByInstall, mod.show);
+  app.get('/:name/:version', syncByInstall, mod.get);
   // try to add module
-  app.put('/:name', [login, publishable], mod.add);
+  app.put('/:name', login, publishable, mod.add);
 
   // sync from source npm
   app.put('/:name/sync', sync.sync);
@@ -47,23 +48,22 @@ function routes(app) {
 
   // put tarball
   // https://registry.npmjs.org/cnpmjs.org/-/cnpmjs.org-0.0.0.tgz/-rev/1-c85bc65e8d2470cc4d82b8f40da65b8e
-  app.put('/:name/-/:filename/-rev/:rev', [login, publishable], mod.upload);
+  app.put('/:name/-/:filename/-rev/:rev', login, publishable, mod.upload);
   // delete tarball
-  app.delete('/:name/-/:filename/-rev/:rev', [login, publishable], mod.removeTar);
+  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
 
   // put package.json to module
-  app.put('/:name/:version/-tag/latest', [login, publishable], mod.updateLatest);
+  app.put('/:name/:version/-tag/latest', login, publishable, mod.updateLatest);
 
   // update module, unpublish will PUT this
-  app.put('/:name/-rev/:rev', [login, publishable], mod.removeWithVersions);
-  app.delete('/:name/-rev/:rev', [login, publishable], mod.removeAll);
+  app.put('/:name/-rev/:rev', login, publishable, mod.removeWithVersions);
+  app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
 
   // try to create a new user
   // https://registry.npmjs.org/-/user/org.couchdb.user:fengmk2
   app.put('/-/user/org.couchdb.user::name', user.add);
   app.get('/-/user/org.couchdb.user::name', user.show);
   app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, user.update);
-
   // _session
   app.post('/_session', user.authSession);
 }

routes/web.js

@@ -18,8 +18,10 @@
 var pkg = require('../controllers/web/package');
 var user = require('../controllers/web/user');
 var sync = require('../controllers/sync');
+var total = require('../controllers/total');
 
 function routes(app) {
+  app.get('/total', total.show);
   app.get('/package/:name', pkg.display);
   app.get('/package/:name/:version', pkg.display);
   app.get('/browse/keyword/:word', pkg.search);
@@ -29,6 +31,9 @@ function routes(app) {
   app.get('/sync/:name', pkg.displaySync);
   app.put('/sync/:name', sync.sync);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
+  app.get('/sync', pkg.displaySync);
+
+  app.get('/_list/search/search', pkg.rangeSearch);
 }
 
 module.exports = routes;

servers/registry.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - servers/registry.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -15,64 +15,54 @@
  * Module dependencies.
  */
 
-require('response-patch');
+var koa = require('koa');
+var app = module.exports = koa();
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var responseCookie = require('response-cookie');
-var urlrouter = require('urlrouter');
+var forward = require('forward');
+var path = require('path');
+var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
+var notFound = require('../middleware/registry_not_found');
+var rootdir = path.dirname(__dirname);
 
-var app = connect();
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
+app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
 
-app.use('/dist', connect.static(config.uploadDir));
-
-app.use(responseCookie());
-app.use(connect.cookieParser());
-app.use(connect.query());
-app.use(connect.json());
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
 app.use(session);
+app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
+app.use(notFound);
+
+app.use(middlewares.fresh());
+app.use(middlewares.etag());
 
 /**
  * Routes
  */
 
-app.use(urlrouter(routes));
-
-app.use(function (req, res, next) {
-  res.json(404, {error: 'not_found', reason: 'document not found'});
-});
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
-  if (process.env.NODE_ENV !== 'test') {
-    console.error(err.stack);
-    logger.error(err);
-  }
-  if (config.debug) {
-    return next(err);
-  }
-  res.json(500, {
-    error: err.name,
-    reason: err.message
-  });
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
+  logger.error(err);
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.registryPort);
+}
 
 module.exports = app;

servers/web.js

@@ -15,81 +15,87 @@
  * Module dependencies.
  */
 
-require('response-patch');
 var path = require('path');
 var http = require('http');
-var connect = require('connect');
-var rt = require('connect-rt');
-var urlrouter = require('urlrouter');
-var connectMarkdown = require('connect-markdown');
+var fs = require('fs');
+var koa = require('koa');
+var middlewares = require('koa-middlewares');
+var markdown = require('koa-markdown');
+var session = require('../common/session');
+var opensearch = require('../middleware/opensearch');
+var notFound = require('../middleware/web_not_found');
 var routes = require('../routes/web');
 var logger = require('../common/logger');
 var config = require('../config');
-var session = require('../common/session');
-var render = require('connect-render');
-var opensearch = require('../middleware/opensearch');
-var app = connect();
+
+var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(rt({headerName: 'X-ReadTime'}));
-app.use(function (req, res, next) {
-  res.req = req;
-  next();
-});
-
-app.use('/public', connect.static(path.join(rootdir, 'public')));
-app.use('/opensearch.xml', opensearch);
-
-app.use(connect.cookieParser());
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
+app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
+  buffer: !config.debug,
+  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+  dir: path.join(rootdir, 'public')
+}));
+app.use(opensearch);
+app.keys = ['todokey', config.sessionSecret];
+app.outputErrors = true;
 app.use(session);
-app.use(connect.query());
-app.use(connect.json());
+app.use(middlewares.bodyParser());
+app.use(notFound);
 
 var viewDir = path.join(rootdir, 'view', 'web');
 var docDir = path.join(rootdir, 'docs', 'web');
 
-app.use('/', connectMarkdown({
+var layoutFile = path.join(viewDir, '_layout.html');
+var footer = config.customFooter || fs.readFileSync(path.join(viewDir, 'footer.html'), 'utf8');
+var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
+  .replace('{{footer}}', footer)
+  .replace('{{logoURL}}', config.logoURL);
+fs.writeFileSync(layoutFile, layout);
+
+app.use(markdown({
+  baseUrl: '/',
   root: docDir,
-  layout: path.join(viewDir, 'layout.html'),
+  layout: layoutFile,
   titleHolder: '<%- locals.title %>',
   bodyHolder: '<%- locals.body %>',
   indexName: 'readme'
 }));
 
-var helpers = {
+var locals = {
   config: config
 };
 
-app.use(render({
+middlewares.render(app, {
   root: viewDir,
-  viewExt: '.html',
-  layout: 'layout',
+  viewExt: 'html',
+  layout: '_layout',
   cache: config.viewCache,
-  helpers: helpers
-}));
+  debug: config.debug,
+  locals: locals
+});
 
 /**
  * Routes
  */
-
-app.use(urlrouter(routes));
+app.use(middlewares.router(app));
+routes(app);
 
 /**
  * Error handler
  */
 
-app.use(function (err, req, res, next) {
-  err.url = err.url || req.url;
-  console.error(err.stack);
+app.on('error', function (err, ctx) {
+  err.url = err.url || ctx.request.url;
   logger.error(err);
-  if (config.debug) {
-    return next(err);
-  }
-  res.statusCode = 500;
-  res.send('Server has some problems. :(');
 });
 
-app = http.createServer(app);
+app = http.createServer(app.callback());
+
+if (!module.parent) {
+  app.listen(config.webPort);
+}
 
 module.exports = app;

sync/index.js

@@ -0,0 +1,104 @@
+/*!
+ * cnpmjs.org - sync/index.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+var ms = require('ms');
+var mail = require('../common/mail');
+var util = require('util');
+var utility = require('utility');
+var debug = require('debug')('cnpmjs.org:sync:index');
+var Total = require('../proxy/total');
+var logger = require('../common/logger');
+
+var sync;
+
+switch (config.syncModel) {
+case 'all':
+  sync = require('./sync_all');
+  break;
+case 'exist':
+  sync = require('./sync_exist');
+  break;
+}
+
+//set sync_status = 0 at first
+Total.updateSyncStatus(0, utility.noop);
+
+// the same time only sync once
+var syncing = false;
+
+function handleSync() {
+  debug('mode: %s, syncing: %s', config.syncModel, syncing);
+  // check sync every one 30 minutes
+  if (!syncing) {
+    syncing = true;
+    debug('start syncing');
+    sync(function (err, data) {
+      if (config.debug) {
+        console.log(err, data);
+      } else {
+        sendMailToAdmin(err, data, new Date());
+      }
+      syncing = false;
+    });
+  }
+}
+
+if (sync) {
+  handleSync();
+  setInterval(handleSync, ms('30m'));
+}
+
+function sendMailToAdmin(err, result, syncTime) {
+  result = result || {};
+  var to = [];
+  for (var name in config.admins) {
+    to.push(config.admins[name]);
+  }
+  debug('Send email to all admins: %j, with err message: %s, result: %j, start sync time: %s.',
+    to, err ? err.message : '', result, syncTime);
+  var subject;
+  var type;
+  var html;
+  if (err) {
+    subject = 'Sync Error';
+    type = 'error';
+    html = util.format('Sync packages from official registry failed.\n' +
+      'Start sync time is %s.\nError message is %s.', syncTime, err.stack);
+  } else if (result.fails && result.fails.length) {
+    subject = 'Sync Finished But Some Packages Failed';
+    type = 'warn';
+    html = util.format('Sync packages from official registry finished, but some packages sync failed.\n' +
+      'Start sync time is %s.\n %d packges sync failed: %j ...\n %d packages sync successes :%j ...',
+      syncTime, result.fails.length, result.fails.slice(0, 10),
+      result.successes.length, result.successes.slice(0, 10));
+  } else {
+    subject = 'Sync Finished';
+    type = 'log';
+    html = util.format('Sync packages from official registry finished.\n' +
+      'Start sync time is %s.\n %d packages sync successes :%j ...',
+      syncTime, result.successes.length, result.successes.slice(0, 10));
+  }
+  debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
+  if (type !== 'log') {
+    mail[type](to, subject, html, function (err) {
+      if (err) {
+        logger.info('send email with type: %s, subject: %s, html: %s', type, subject, html);
+        logger.error(err);
+      }
+    });
+  }
+}

sync/status.js

@@ -0,0 +1,61 @@
+/*!
+ * cnpmjs.org - sync/status.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:sync:status');
+var utility = require('utility');
+var Total = require('../proxy/total');
+
+function Status(options) {
+  this.worker = options.worker;
+  this.need = options.need;
+  this.lastSyncModule = '';
+}
+
+Status.prototype.log = function (syncDone) {
+  var params = {
+    syncStatus: syncDone ? 0 : 1,
+    need: this.need,
+    success: this.worker.successes.length,
+    fail: this.worker.fails.length,
+    left: this.worker.names.length,
+    lastSyncModule: this.lastSyncModule,
+  };
+  Total.updateSyncNum(params, utility.noop);
+};
+
+Status.prototype.start = function () {
+  if (this.started) {
+    return;
+  }
+  this.started = true;
+  //every 30s log it into mysql
+  this.timer = setInterval(this.log.bind(this), 30000);
+  this.worker.on('success', function (moduleName) {
+    debug('sync [%s] success', moduleName);
+    this.lastSyncModule = moduleName;
+  }.bind(this));
+  this.worker.on('end', function () {
+    this.started = false;
+    this.log(true);
+    clearInterval(this.timer);
+  }.bind(this));
+};
+
+Status.init = function (options) {
+  return new Status(options);
+};
+
+module.exports = Status;

sync/sync_all.js

@@ -0,0 +1,163 @@
+/*!
+ * cnpmjs.org - sync/sync_all.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:sync:sync_all');
+var eventproxy = require('eventproxy');
+var ms = require('ms');
+var utility = require('utility');
+var config = require('../config');
+var Status = require('./status');
+var Npm = require('../proxy/npm');
+var Total = require('../proxy/total');
+var SyncModuleWorker = require('../proxy/sync_module_worker');
+var Module = require('../proxy/module');
+
+function subtract(subtracter, minuend) {
+  subtracter = subtracter || [];
+  minuend = minuend || [];
+  var map = {};
+  var results = [];
+  minuend.forEach(function (name) {
+    map[name] = true;
+  });
+  subtracter.forEach(function (name) {
+    if (!map[name]) {
+      results.push(name);
+    }
+  });
+  return results;
+}
+
+function union(arrOne, arrTwo) {
+  arrOne = arrOne || [];
+  arrTwo = arrTwo || [];
+  var map = {};
+  arrOne.concat(arrTwo).forEach(function (name) {
+    map[name] = true;
+  });
+  return Object.keys(map);
+}
+
+/**
+ * when sync from official at the first time
+ * get all packages by short and restart from last synced module
+ * @param {String} lastSyncModule
+ * @param {Function} callback
+ */
+function getFirstSyncPackages(lastSyncModule, callback) {
+  Npm.getShort(function (err, pkgs) {
+    if (err || !lastSyncModule) {
+      return callback(err, pkgs);
+    }
+    // start from last success
+    var lastIndex = pkgs.indexOf(lastSyncModule);
+    if (lastIndex > 0) {
+      pkgs = pkgs.slice(lastIndex);
+    }
+    return callback(null, pkgs);
+  });
+}
+
+/**
+ * get all the packages that update time > lastSyncTime
+ * @param {Number} lastSyncTime
+ * @param {Function} callback
+ */
+function getCommonSyncPackages(lastSyncTime, callback) {
+  Npm.getAllSince(lastSyncTime, function (err, data) {
+    if (err || !data) {
+      return callback(err, []);
+    }
+    delete data._updated;
+    return callback(null, Object.keys(data));
+  });
+}
+
+/**
+ * get all the missing packages
+ * @param {Function} callback
+ */
+function getMissPackages(callback) {
+  var ep = eventproxy.create();
+  ep.fail(callback);
+  Npm.getShort(ep.doneLater('allPackages'));
+  Module.listAllModuleNames(ep.doneLater(function (rows) {
+    var existPackages = rows.map(function (row) {
+      return row.name;
+    });
+    ep.emit('existPackages', existPackages);
+  }));
+  ep.all('allPackages', 'existPackages', function (allPackages, existPackages) {
+    callback(null, subtract(allPackages, existPackages));
+  });
+}
+
+//only sync not exist once
+var syncNotExist = false;
+module.exports = function sync(callback) {
+  var ep = eventproxy.create();
+  ep.fail(callback);
+  var syncTime = Date.now();
+  Total.getTotalInfo(ep.doneLater('totalInfo'));
+
+  ep.once('totalInfo', function (info) {
+    if (!info) {
+      return callback(new Error('can not found total info'));
+    }
+    debug('Last sync time %s', new Date(info.last_sync_time));
+    // TODO: 记录上次同步的最后一个模块名称
+    if (!info.last_sync_time) {
+      debug('First time sync all packages from official registry');
+      return getFirstSyncPackages(info.last_sync_module, ep.done('syncPackages'));
+    }
+    if (syncNotExist) {
+      getMissPackages(ep.done('missPackages'));
+      syncNotExist = false;
+    } else {
+      ep.emitLater('missPackages', []);
+    }
+    getCommonSyncPackages(info.last_sync_time - ms('10m'), ep.doneLater('newestPackages'));
+    ep.all('missPackages', 'newestPackages', function (missPackages, newestPackages) {
+      ep.emit('syncPackages', union(missPackages, newestPackages));
+    });
+  });
+
+  ep.once('syncPackages', function (packages) {
+    packages = packages || [];
+    debug('Total %d packages to sync', packages.length);
+    var worker = new SyncModuleWorker({
+      username: 'admin',
+      name: packages,
+      noDep: true,
+      concurrency: config.syncConcurrency,
+    });
+    Status.init({
+      worker: worker,
+      need: packages.length
+    }).start();
+    worker.start();
+    worker.once('end', function () {
+      debug('All packages sync done, successes %d, fails %d',
+        worker.successes.length, worker.fails.length);
+      //only when all succss, set last sync time
+      !worker.fails.length && Total.setLastSyncTime(syncTime, utility.noop);
+      callback(null, {
+        successes: worker.successes,
+        fails: worker.fails
+      });
+    });
+  });
+};

sync/sync_exist.js

@@ -0,0 +1,109 @@
+/*!
+ * cnpmjs.org - sync/sync_exist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+var Npm = require('../proxy/npm');
+var Module = require('../proxy/module');
+var Total = require('../proxy/total');
+var eventproxy = require('eventproxy');
+var SyncModuleWorker = require('../proxy/sync_module_worker');
+var debug = require('debug')('cnpmjs.org:sync:sync_hot');
+var utility = require('utility');
+var Status = require('./status');
+var ms = require('ms');
+
+function intersection(arrOne, arrTwo) {
+  arrOne = arrOne || [];
+  arrTwo = arrTwo || [];
+  var map = {};
+  var results = [];
+  arrOne.forEach(function (name) {
+    map[name] = true;
+  });
+  arrTwo.forEach(function (name) {
+    map[name] && results.push(name);
+  });
+  return results;
+}
+
+module.exports = function sync(callback) {
+  var ep = eventproxy.create();
+  ep.fail(callback);
+  var syncTime = Date.now();
+
+  Module.listShort(ep.doneLater(function (packages) {
+    packages = packages.map(function (p) {
+      return p.name;
+    });
+    ep.emit('existPackages', packages);
+  }));
+  Total.getTotalInfo(ep.doneLater('totalInfo'));
+
+  ep.once('totalInfo', function (info) {
+    if (!info) {
+      return callback(new Error('can not found total info'));
+    }
+    if (!info.last_exist_sync_time) {
+      debug('First time sync all packages from official registry');
+      return Npm.getShort(ep.done(function (pkgs) {
+        if (!info.last_sync_module) {
+          return ep.emit('allPackages', pkgs);
+        }
+        // start from last success
+        var lastIndex = pkgs.indexOf(info.last_sync_module);
+        if (lastIndex > 0) {
+          pkgs = pkgs.slice(lastIndex);
+        }
+        ep.emit('allPackages', pkgs);
+      }));
+    }
+    Npm.getAllSince(info.last_exist_sync_time - ms('10m'), ep.done(function (data) {
+      if (!data) {
+        return ep.emit('allPackages', []);
+      }
+      if (data._updated) {
+        syncTime = data._updated;
+        delete data._updated;
+      }
+
+      return ep.emit('allPackages', Object.keys(data));
+    }));
+  });
+
+  ep.all('existPackages', 'allPackages', function (existPackages, allPackages) {
+    var packages = intersection(existPackages, allPackages);
+    debug('Total %d packages to sync', packages.length);
+    var worker = new SyncModuleWorker({
+      username: 'admin',
+      name: packages,
+      concurrency: config.syncConcurrency,
+    });
+    Status.init({
+      worker: worker,
+      need: packages.length
+    }).start();
+    worker.start();
+    worker.once('end', function () {
+      debug('All packages sync done, successes %d, fails %d',
+        worker.successes.length, worker.fails.length);
+      Total.setLastExistSyncTime(syncTime, utility.noop);
+      callback(null, {
+        successes: worker.successes,
+        fails: worker.fails
+      });
+    });
+  });
+};

test/controllers/registry/home.test.js

@@ -1,42 +0,0 @@
-/*!
- * cnpmjs.org - test/controllers/registry/home.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var should = require('should');
-var request = require('supertest');
-var app = require('../../../servers/registry');
-
-describe('controllers/registry/home.test.js', function () {
-  before(function (done) {
-    app.listen(0, done);
-  });
-  after(function (done) {
-    app.close(done);
-  });
-
-  describe('GET /', function () {
-    it('should return total info', function (done) {
-      request(app)
-      .get('/')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        res.body.db_name.should.equal('registry');
-        res.body.node_version.should.equal(process.version);
-        // console.log(res.body)
-        done();
-      });
-    });
-  });
-});

test/controllers/registry/module.test.js

@@ -17,6 +17,7 @@
 
 var fs = require('fs');
 var path = require('path');
+var thunkify = require('thunkify-wrap');
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
@@ -25,6 +26,7 @@ var app = require('../../../servers/registry');
 var Module = require('../../../proxy/module');
 var Npm = require('../../../proxy/npm');
 var controller = require('../../../controllers/registry/module');
+var ModuleDeps = require('../../../proxy/module_deps');
 
 var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
 
@@ -64,38 +66,48 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('GET /:name', function () {
-    it('should return module info', function (done) {
+  describe('GET /:name get module package info', function () {
+    var etag;
+
+    it('should return module info and etag', function (done) {
       request(app)
       .get('/cnpmjs.org')
       .expect(200, function (err, res) {
         should.not.exist(err);
+        // should have etag
+        res.headers.should.have.property('etag');
+        etag = res.headers.etag;
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
           'time', 'author', 'repository', '_attachments');
-        // res.body.author.should.eql({
-        //   "name": "fengmk2",
-        //   "email": "fengmk2@gmail.com",
-        //   "url": "http://fengmk2.github.com"
-        // });
         res.body.name.should.equal('cnpmjs.org');
         res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
         done();
       });
     });
+
+    it('should 304 when etag match', function (done) {
+      request(app)
+      .get('/cnpmjs.org')
+      .set('If-None-Match', etag)
+      .expect(304, done)
+    });
   });
 
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.0.0')
+      .get('/cnpmjs.org/0.2.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.0.0');
-        body._id.should.equal('cnpmjs.org@0.0.0');
-        body.dist.tarball.should.include('cnpmjs.org-0.0.0.tgz');
+        body.version.should.equal('0.2.1');
+        body._id.should.equal('cnpmjs.org@0.2.1');
+        body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
+        body.should.have.property('_cnpm_publish_time');
+        body._cnpm_publish_time.should.be.a.Number;
+        body.should.have.property('_publish_on_cnpm', true);
         done();
       });
     });
@@ -113,6 +125,19 @@ describe('controllers/registry/module.test.js', function () {
         done();
       });
     });
+
+    it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
+      request(app)
+      .get('/cnpmjs.org/0.2.1')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        var body = res.body;
+        body.name.should.equal('cnpmjs.org');
+        body.version.should.equal('0.2.1');
+        body._publish_on_cnpm.should.equal(true);
+        done();
+      });
+    });
   });
 
   describe('PUT /:name', function () {
@@ -124,6 +149,13 @@ describe('controllers/registry/module.test.js', function () {
         name: 'cnpmjstest10',
         email: 'cnpmjstest10@cnpmjs.org'
       }],
+      keywords: [
+        'testputmodule', 'test', 'cnpmjstest'
+      ],
+      dependencies: {
+        'foo-testputmodule': "*",
+        'bar-testputmodule': '*'
+      }
     };
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
@@ -159,14 +191,7 @@ describe('controllers/registry/module.test.js', function () {
       .put('/' + pkg.name)
       .set('authorization', baseauth)
       .send(pkg)
-      .expect(409, function (err, res) {
-        should.not.exist(err);
-        res.body.should.eql({
-          error: 'conflict',
-          reason: 'Document update conflict.'
-        });
-        done();
-      });
+      .expect(201, done);
     });
 
     it('should try to add return 403 when not module user and only next module exists',
@@ -278,6 +303,7 @@ describe('controllers/registry/module.test.js', function () {
       var pkg = require(path.join(fixtures, 'testputmodule.json')).versions['0.1.8'];
       pkg.name = 'testputmodule';
       pkg.version = '0.1.9';
+      pkg.dependencies['foo-testputmodule'] = '*';
       request(app)
       .put('/' + pkg.name + '/' + pkg.version + '/-tag/latest')
       .set('authorization', baseauth)
@@ -285,7 +311,16 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('ok', 'rev');
-        done();
+        // should get deps foo-testputmodule contains 'testputmodule'
+        ModuleDeps.list('foo-testputmodule', function (err, rows) {
+          should.not.exist(err);
+          var exists = rows.filter(function (r) {
+            return r.deps === 'testputmodule';
+          });
+          exists.should.length(1);
+          exists[0].deps.should.equal('testputmodule');
+          done();
+        });
       });
     });
 
@@ -333,6 +368,60 @@ describe('controllers/registry/module.test.js', function () {
         done();
       });
     });
+
+    it('should publish with tgz base64, addPackageAndDist()', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      // delete first
+      request(app)
+      .del('/' + pkg.name + '/-rev/1')
+      .set('authorization', baseauth)
+      .expect({ok: true})
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+
+        request(app)
+        .put('/' + pkg.name)
+        .set('authorization', baseauth)
+        .send(pkg)
+        .expect(201, function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'rev');
+          res.body.ok.should.equal(true);
+
+          // upload again should 409
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', baseauth)
+          .send(pkg)
+          .expect(409, function (err, res) {
+            should.not.exist(err);
+            res.body.should.eql({
+              error: 'conflict',
+              reason: 'Document update conflict.'
+            });
+            done();
+          });
+
+        });
+      });
+    });
+
+    it('should version_error when versions missing', function (done) {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      delete pkg.versions;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({
+          error: 'version_error',
+          reason: 'filename or version not found, filename: mk2testmodule-0.0.1.tgz, version: undefined'
+        });
+        done();
+      });
+    });
   });
 
   describe('GET /-/all', function () {
@@ -429,6 +518,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should remove version ok', function (done) {
       //do not really remove it here
       mm.empty(Module, 'removeByNameAndVersions');
+      mm.empty(Module, 'removeTagsByIds');
       request(app)
       .put('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
@@ -440,7 +530,16 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('DELETE /:name/-/:filename/-rev/:rev', function () {
+  describe('GET /:name/download/:filename', function () {
+    it('should download a file with 302 redirect', function (done) {
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
+      .expect(302, done)
+    });
+  });
+
+  describe('DELETE /:name/download/:filename/-rev/:rev', function () {
     var lastRev;
     before(function (done) {
       request(app)
@@ -453,20 +552,20 @@ describe('controllers/registry/module.test.js', function () {
 
     it('should delete 401 when no auth', function (done) {
       request(app)
-      .del('/testputmodule/-/testputmodule-0.1.9.tgz/-rev/' + lastRev)
+      .del('/testputmodule/download/testputmodule-0.1.9.tgz/-rev/' + lastRev)
       .expect(401, done);
     });
 
     it('should delete 403 when auth error', function (done) {
       request(app)
-      .del('/testputmodule/-/testputmodule-0.1.9.tgz/-rev/' + lastRev)
+      .del('/testputmodule/download/testputmodule-0.1.9.tgz/-rev/' + lastRev)
       .set('authorization', baseauthOther)
       .expect(403, done);
     });
 
     it('should delete file ok', function (done) {
       request(app)
-      .del('/testputmodule/-/testputmodule-0.1.9.tgz/-rev/' + lastRev)
+      .del('/testputmodule/download/testputmodule-0.1.9.tgz/-rev/' + lastRev)
       .set('authorization', baseauth)
       .expect(200, done);
     });

test/controllers/registry/sync_module_worker.js

@@ -1,44 +0,0 @@
-/**!
- * cnpmjs.org - test/controllers/registry/sync_module_worker.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var SyncModuleWorker = require('../../../controllers/registry/sync_module_worker');
-var mysql = require('../../../common/mysql');
-var Log = require('../../../proxy/module_log');
-
-var name = process.argv[2] || 'address';
-
-Log.create({
-  name: name,
-  username: 'fengmk2',
-}, function (err, result) {
-  var worker = new SyncModuleWorker({
-    logId: result.id,
-    name: name,
-    username: 'fengmk2'
-  });
-
-  worker.start();
-  worker.on('end', function () {
-    process.exit(0);
-  });
-
-  // mysql.query('delete from module where name=?', [name], function () {
-  //   worker.start();
-  //   worker.on('end', function () {
-  //     process.exit(0);
-  //   });
-  // });
-});

test/controllers/registry/user.test.js

@@ -16,18 +16,22 @@
 
 var should = require('should');
 var request = require('supertest');
+var mm = require('mm');
 var app = require('../../../servers/registry');
 var user = require('../../../proxy/user');
-var mm = require('mm');
+var mysql = require('../../../common/mysql');
 
 describe('controllers/registry/user.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
+
   afterEach(mm.restore);
+
   describe('GET /-/user/org.couchdb.user:name', function () {
     it('should return user info', function (done) {
       request(app)
@@ -47,7 +51,7 @@ describe('controllers/registry/user.test.js', function () {
     });
 
     it('should return 500 when mysql error', function (done) {
-      mm.error(user, 'get', 'mock error');
+      mm.error(mysql, 'query', 'mock mysql error');
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
       .expect(500, done);
@@ -91,7 +95,7 @@ describe('controllers/registry/user.test.js', function () {
         password_sha: 'password_sha',
         email: 'email'
       })
-      .expect(500, done);      
+      .expect(500, done);
     });
 
     it('should 201 when user.add ok', function (done) {

test/controllers/sync.test.js

@@ -40,6 +40,54 @@ describe('controllers/sync.test.js', function () {
   describe('sync source npm package', function () {
     var logIdRegistry;
     var logIdWeb;
+
+    it('should sync as publish success', function (done) {
+      request(registryApp)
+      .del('/utility/-rev/123')
+      .set('authorization', baseauth)
+      // .expect(200)
+      // .expect({ok: true})
+      .end(function (err, res) {
+        should.not.exist(err);
+
+        mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+        request(registryApp)
+        .put('/utility/sync?publish=true&nodeps=true')
+        .set('authorization', baseauth)
+        .end(function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'logId');
+          logIdRegistry = res.body.logId;
+          done();
+          // setTimeout(function () {
+          //   request(registryApp)
+          //   .get('/utility')
+          //   .expect(200)
+          //   .end(function (err, res) {
+          //     should.not.exist(err);
+          //     Object.keys(res.body.versions).length.should.above(0);
+          //     for (var v in res.body.versions) {
+          //       var pkg = res.body.versions[v];
+          //       pkg.should.have.property('_publish_on_cnpm', true);
+          //     }
+          //     done();
+          //   });
+          // }, 5000);
+        });
+      });
+    });
+
+    it('should sync as publish 403 when user not admin', function (done) {
+      mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+      request(registryApp)
+      .put('/utility_unit_test/sync?publish=true&nodeps=true')
+      .expect(403)
+      .expect({
+        error: 'no_perms',
+        reason: 'Only admin can publish'
+      }, done);
+    });
+
     it('should sync success', function (done) {
       mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
       done = pedding(2, done);

test/controllers/total.test.js

@@ -0,0 +1,68 @@
+/**!
+ * cnpmjs.org - test/controllers/total.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var pedding = require('pedding');
+var registryApp = require('../../servers/registry');
+var webApp = require('../../servers/web');
+
+describe('controllers/total.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    registryApp.listen(0, done);
+    webApp.listen(0, done);
+  });
+  after(function (done) {
+    done = pedding(2, done);
+    registryApp.close(done);
+    webApp.close(done);
+  });
+
+  describe('GET / in registry', function () {
+    it('should return total info', function (done) {
+      request(registryApp)
+      .get('/')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.db_name.should.equal('registry');
+        res.body.node_version.should.equal(process.version);
+        done();
+      });
+    });
+
+    it('should return total info by jsonp', function (done) {
+      request(registryApp)
+      .get('?callback=totalCallback')
+      .expect(200)
+      .expect(/totalCallback\({.*}\)/, done);
+    });
+  });
+
+  describe.skip('GET /total in web', function () {
+    it('should return total info', function (done) {
+      request(webApp)
+      .get('/total')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.db_name.should.equal('registry');
+        res.body.node_version.should.equal(process.version);
+        done();
+      });
+    });
+  });
+});

test/controllers/web/package.test.js

@@ -16,7 +16,10 @@
 
 var should = require('should');
 var request = require('supertest');
+var mm = require('mm');
+var mysql = require('../../../common/mysql');
 var app = require('../../../servers/web');
+var pkg = require('../../../controllers/web/package');
 
 describe('controllers/web/package.test.js', function () {
   before(function (done) {
@@ -26,6 +29,50 @@ describe('controllers/web/package.test.js', function () {
     app.close(done);
   });
 
+  afterEach(mm.restore);
+
+  describe('GET /_list/search/search', function () {
+    it('should search with "c"', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="c"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search with c', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey=c&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.length.should.above(0);
+        res.body.rows.forEach(function (row) {
+          row.should.have.keys('key', 'count', 'value');
+          row.value.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="cddddsdasdaasds"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({rows: []});
+        done();
+      });
+    });
+  });
+
   describe('GET /package/:name', function (done) {
     it('should get 200', function (done) {
       request(app)
@@ -82,12 +129,55 @@ describe('controllers/web/package.test.js', function () {
       .expect(/Packages match/, done);
     });
 
+    it('should list by keyword with json ok', function (done) {
+      request(app)
+      .get('/browse/keyword/cnpm?type=json')
+      .expect(200)
+      .expect('content-type', 'application/json; charset=utf-8', done);
+    });
+
     it('should list no match ok', function (done) {
       request(app)
       .get('/browse/keyword/notexistpackage')
       .expect(200)
       .expect(/Can not found package match notexistpackage/, done);
     });
+
+    it('should 500 when mysql error', function (done) {
+      mm.error(mysql, 'query');
+      request(app)
+      .get('/browse/keyword/notexistpackage')
+      .expect(500)
+      .expect(/Internal Server Error/, done);
+    });
+  });
+
+  describe('setLicense()', function () {
+    it('should only use the first license', function () {
+      var p = {license: ['MIT']};
+      pkg.setLicense(p);
+      p.license.should.have.keys('name', 'url');
+      p.license.should.eql({
+        name: 'MIT',
+        url: 'http://opensource.org/licenses/MIT'
+      });
+
+      var p = {license: ['http://foo/MIT']};
+      pkg.setLicense(p);
+      p.license.should.have.keys('name', 'url');
+      p.license.should.eql({
+        name: 'http://foo/MIT',
+        url: 'http://foo/MIT'
+      });
+
+      var p = {license: {name: 'mit', url: 'http://foo/mit'}};
+      pkg.setLicense(p);
+      p.license.should.have.keys('name', 'url');
+      p.license.should.eql({
+        name: 'mit',
+        url: 'http://foo/mit'
+      });
+    });
   });
 
   describe('GET /sync/:name', function (done) {

test/controllers/web/user.test.js

@@ -43,7 +43,7 @@ describe('controllers/web/user.test.js', function () {
 
     it('should get not eixst user but have modules 200', function (done) {
       request(app)
-      .get('/~jdalton')
+      .get('/~tjholowaychuk')
       .expect(200)
       .expect(/<div id="profile">/)
       .expect(/Packages by/, done);

test/fixtures/500.txt

@@ -0,0 +1,7 @@
+Internal routing error
+
+Sorry, we cannot connect to the intended server.
+
+We have just been notified of this problem. We will correct it as soon as possible.
+
+Feel free to contact us if you have any questions: support@iriscouch.com

test/fixtures/package_and_tgz.json

@@ -0,0 +1 @@
+{"_id":"mk2testmodule","name":"mk2testmodule","description":"","dist-tags":{"latest":"0.0.1"},"versions":{"0.0.1":{"name":"mk2testmodule","version":"0.0.1","description":"","main":"index.js","scripts":{"test":"echo \"Error: no test specified\" && exit 1"},"author":"","license":"ISC","readme":"ERROR: No README data found!","_id":"mk2testmodule@0.0.1","dist":{"shasum":"fa475605f88bab9b1127833633ca3ae0a477224c","tarball":"http://127.0.0.1:7001/mk2testmodule/-/mk2testmodule-0.0.1.tgz"},"_from":".","_npmVersion":"1.4.3","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}]}},"readme":"ERROR: No README data found!","maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_attachments":{"mk2testmodule-0.0.1.tgz":{"content_type":"application/octet-stream","data":"H4sIAAAAAAAAA+2SsWrDMBCGPfspDg2ZinOyEgeylg6Zu2YR8rVRHEtGkkOg5N0jWaFdujVQAv6W4/7/dHcSGqTq5Ccthxyro7emeDCI2KxWkOKmaaaIdc4TouZQ8FqgwI3AdVMgF8ijho9e5DdGH6SLq/y1T74LfMcn4asEYEb2xLbA+q4O5ENv2/FE7CVZZ3JeW5NcrLDiWW3JK6eHcHey2Es9Zdq0dIkfKau50EcjjYpCmpDKSB0s7Nmbc9ZtwVhIBviBlP7Q1O4ZLBZAFx2As3jyOnWTYzhY9zPzpBUZPy2/e39l5bX87wedmZmZeRJuheTX2wAIAAA=","length":251}}}

test/fixtures/special.txt

@@ -0,0 +1 @@
+𝌆

test/lib/common.test.js

@@ -0,0 +1,29 @@
+/*!
+ * cnpmjs.org - test/lib/common.test.js.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com>
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var common = require('../../lib/common');
+
+describe('lib/common.test.js', function () {
+  describe('isAdmin()', function () {
+    it('should admin is admin', function () {
+      common.isAdmin('admin').should.equal(true);
+      common.isAdmin('fengmk2').should.equal(true);
+      common.isAdmin('constructor').should.equal(false);
+      common.isAdmin('toString').should.equal(false);
+    });
+  });
+});

test/middleware/auth.test.js

@@ -29,8 +29,9 @@ describe('middleware/auth.test.js', function () {
     app.close(done);
   });
 
+  afterEach(mm.restore);
+
   describe('auth()', function () {
-    afterEach(mm.restore);
     it('should pass if no authorization', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')
@@ -44,11 +45,12 @@ describe('middleware/auth.test.js', function () {
       .expect(200, done);
     });
 
-    it('should 401 with authorization and check fail', function (done) {
+    it('should pass with authorization and check fail', function (done) {
+      // npm install no need to check auth
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest10')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest').toString('base64'))
-      .expect(401, done);
+      .expect(200, done);
     });
 
     it('should 500 with authorization and mysql error', function (done) {

test/middleware/opensearch.test.js

@@ -30,8 +30,8 @@ describe('middleware/opensearch.test.js', function () {
     it('should get 200', function (done) {
       request(app)
       .get('/opensearch.xml')
-      .set('host', 'localhost:7002')
-      .expect(/http:\/\/localhost:7002/, done);
+      .set('host', 'localhost')
+      .expect(/http:\/\/localhost/, done);
     });
   });
 });

test/proxy/module.test.js

@@ -16,9 +16,14 @@
 
 var should = require('should');
 var mm = require('mm');
+var fs = require('fs');
+var path = require('path');
 var mysql = require('../../common/mysql');
 var Module = require('../../proxy/module');
 
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
+
+var id;
 describe('proxy/module.test.js', function () {
   afterEach(mm.restore);
 
@@ -32,6 +37,107 @@ describe('proxy/module.test.js', function () {
     });
   });
 
+  describe('search()', function () {
+    it('should search modules', function (done) {
+      Module.search('as', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.searchMatchs.length.should.above(0);
+        data.searchMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search match keywords modules', function (done) {
+      Module.search('aa', function (err, data) {
+        should.not.exist(err);
+        data.should.have.keys('keywordMatchs', 'searchMatchs');
+        data.keywordMatchs.length.should.above(0);
+        data.keywordMatchs.forEach(function (row) {
+          row.should.have.keys('name', 'description');
+        });
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      Module.search('emptyemptyemptyempty', function (err, data) {
+        should.not.exist(err);
+        data.should.eql({
+          keywordMatchs: [],
+          searchMatchs: []
+        });
+        done();
+      });
+    });
+  });
+
+  describe('addKeywords()', function () {
+    var mockName = 'aa' + Date.now();
+
+    after(function (done) {
+      mysql.query('DELETE FROM module_keyword WHERE name=?', [mockName], done);
+    });
+
+    it('should add diff keywords to module', function (done) {
+      Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        done();
+      });
+    });
+
+    it('should add same keywords to module', function (done) {
+      Module.addKeywords('aa', 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(0);
+        done();
+      });
+    });
+  });
+
+  describe('getKeywords()', function () {
+    it('should get aa module keywords', function (done) {
+      Module.getKeywords('aa', function (err, keywords) {
+        should.not.exist(err);
+        keywords.should.eql(['aa', 'bb', 'cc']);
+        done();
+      });
+    });
+  });
+
+  describe('add()', function () {
+    it('should success ad he@0.3.6', function (done) {
+      var sourcePackage = require('../fixtures/0.3.6.json');
+      var mod = {
+        version: sourcePackage.version,
+        name: sourcePackage.name,
+        package: sourcePackage,
+        author: 'unittest',
+        publish_time: sourcePackage.publish_time || Date.now(),
+      };
+      var dist = {
+        tarball: 'http://registry.npmjs.org/he/-/he-0.3.6.tgz',
+        shasum: '9d7bc446e77963933301dd602d5731cb861135e0',
+        size: 100,
+      };
+      mod.package.dist = dist;
+      Module.add(mod, function (err, result) {
+        id = result.id;
+        should.not.exist(err);
+        Module.getById(result.id, function (err, row) {
+          should.not.exist(err);
+          row.package.readme.should.equal(sourcePackage.readme);
+          done();
+        });
+      });
+    });
+  });
+
   describe('listByAuthor()', function () {
     it('should return author recent modules', function (done) {
       Module.listByAuthor('fengmk2', function (err, rows) {
@@ -43,4 +149,17 @@ describe('proxy/module.test.js', function () {
       });
     });
   });
+
+  describe('updateReadme()', function () {
+    it('should update ok', function (done) {
+      Module.updateReadme(id, 'test', function (err, data) {
+        should.not.exist(err);
+        Module.getById(id, function (err, data) {
+          should.not.exist(err);
+          data.package.readme.should.equal('test');
+          done();
+        })
+      });
+    });
+  });
 });

test/proxy/module_deps.test.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - test/proxy/module_deps.test.js
+ *
+ * Copyright(c) 2014
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+"use strict";
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var pedding = require('pedding');
+var ModuleDeps = require('../../proxy/module_deps');
+
+describe('proxy/module_deps.test.js', function () {
+  before(function (done) {
+    done = pedding(2, done);
+    ModuleDeps.remove('testmodule', 'foo', done);
+    ModuleDeps.remove('testmodule', 'bar', done);
+  });
+
+  describe('add()', function () {
+    it('should add foo, bar success', function (done) {
+      done = pedding(2, done);
+      ModuleDeps.add('testmodule', 'foo', function (err) {
+        should.not.exist(err);
+        // add again should work
+        ModuleDeps.add('testmodule', 'foo', function (err) {
+          should.not.exist(err);
+          done();
+        });
+      });
+
+      ModuleDeps.add('testmodule', 'bar', done);
+    });
+  });
+
+  describe('list()', function () {
+    it('should list testmodule deps', function (done) {
+      ModuleDeps.list('testmodule', function (err, rows) {
+        should.not.exist(err);
+        should.exist(rows);
+        rows.should.be.an.Array;
+        rows.should.length(2);
+        rows.forEach(function (row) {
+          row.should.have.property('deps');
+        });
+        done();
+      });
+    });
+  });
+});

test/proxy/npm.test.js

@@ -0,0 +1,66 @@
+/**!
+ * cnpmjs.org - test/proxy/npm.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var mm = require('mm');
+var fs = require('fs');
+var path = require('path');
+var npm = require('../../proxy/npm');
+
+var fixtures = path.join(path.dirname(__dirname), 'fixtures');
+
+describe('proxy/npm.test.js', function () {
+  afterEach(mm.restore);
+
+  it('should return a module info from source npm', function (done) {
+    npm.get('pedding', function (err, data) {
+      should.not.exist(err);
+      should.exist(data);
+      data.name.should.equal('pedding');
+      done();
+    });
+  });
+
+  it('should return null when module not exist', function (done) {
+    npm.get('pedding-not-exists', function (err, data) {
+      should.not.exist(err);
+      should.not.exist(data);
+      done();
+    });
+  });
+
+  it('should return error when http error', function (done) {
+    mm.http.request(/\//, '{');
+    npm.get('pedding-not-exists', function (err, data) {
+      should.exist(err);
+      err.name.should.equal('JSONResponseFormatError');
+      should.not.exist(data);
+      done();
+    });
+  });
+
+  it('should return ServerError when http 500 response', function (done) {
+    var content = fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8');
+    mm.http.request(/\//, content, { statusCode: 500 });
+    // http://registry.npmjs.org/octopie
+    npm.get('octopie', function (err, data) {
+      should.exist(err);
+      err.name.should.equal('NPMServerError');
+      err.message.should.equal('Status 500, ' + content);
+      done();
+    });
+  });
+});

test/proxy/sync_module_worker.test.js

@@ -0,0 +1,64 @@
+/**!
+ * cnpmjs.org - test/proxy/sync_module_worker.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var SyncModuleWorker = require('../../proxy/sync_module_worker');
+var mysql = require('../../common/mysql');
+var Log = require('../../proxy/module_log');
+
+describe('proxy/sync_module_worker.test.js', function () {
+  it('should start a sync worker', function (done) {
+    Log.create({
+      name: 'cnpmjs.org',
+      username: 'fengmk2',
+    }, function (err, result) {
+      should.not.exist(err);
+      result.id.should.above(0);
+      var worker = new SyncModuleWorker({
+        logId: result.id,
+        name: 'cnpmjs.org',
+        username: 'fengmk2'
+      });
+
+      worker.start();
+      worker.on('end', done);
+    });
+  });
+
+  it('should start a sync worker with names and noDep', function (done) {
+    var worker = new SyncModuleWorker({
+      name: ['cnpmjs.org', 'cutter'],
+      noDep: true,
+      username: 'fengmk2'
+    });
+
+    worker.start();
+    worker.on('end', function () {
+      worker.successes.concat(worker.fails).should.eql(['cnpmjs.org', 'cutter']);
+      done();
+    });
+  });
+
+  it('should start a sync worker with names', function (done) {
+    var worker = new SyncModuleWorker({
+      name: ['cnpmjs.org', 'cutter'],
+      username: 'fengmk2'
+    });
+
+    worker.start();
+    worker.on('end', done);
+  });
+});

test/proxy/user.test.js

@@ -121,6 +121,7 @@ describe('proxy/user.test.js', function () {
     it('should update ok', function (done) {
       user.update(mockUser, function (err, data) {
         should.not.exist(err);
+        should.exist(data);
         data.should.have.keys(['rev']);
         done();
       });

test/sync.js

@@ -0,0 +1,41 @@
+/**!
+ * cnpmjs.org - test/sync.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var SyncModuleWorker = require('../proxy/sync_module_worker');
+var mysql = require('../common/mysql');
+var Log = require('../proxy/module_log');
+
+var name = process.argv[2] || 'address,pedding';
+var names = name.split(',');
+
+Log.create({
+  name: names[0],
+  username: 'fengmk2',
+}, function (err, result) {
+  var worker = new SyncModuleWorker({
+    logId: result.id,
+    name: names,
+    username: 'fengmk2',
+    concurrency: names.length,
+    noDep: true,
+    publish: true,
+  });
+
+  worker.start();
+  worker.on('end', function () {
+    process.exit(0);
+  });
+});

test/sync/sync_all.test.js

@@ -0,0 +1,58 @@
+/*!
+ * cnpmjs.org - test/sync/sync_all.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+var sync = require('../../sync/sync_all');
+var mm = require('mm');
+var Npm = require('../../proxy/npm');
+var Total = require('../../proxy/total');
+var should = require('should');
+var Module = require('../../proxy/module');
+
+describe('sync/sync_all.js', function () {
+  describe('sync()', function () {
+    afterEach(mm.restore);
+
+    it('should sync first time ok', function (done) {
+      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+      mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
+      sync(function (err, data) {
+        should.not.exist(err);
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        mm.restore();
+        Total.getTotalInfo(function (err, result) {
+          should.not.exist(err);
+          result.last_sync_module.should.equal('cutter');
+          done();
+        });
+      });
+    });
+
+    it('should sync common ok', function (done) {
+      mm.data(Npm, 'getAllSince', {
+        _updated: Date.now(),
+        'cnpmjs.org': {},
+        cutter: {}
+      });
+      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
+      mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
+      mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
+      sync(function (err, data) {
+        should.not.exist(err);
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        done();
+      });
+    });
+  });
+});

test/sync/sync_exist.test.js

@@ -0,0 +1,50 @@
+/*!
+ * cnpmjs.org - test/sync/sync_exist.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+var sync = require('../../sync/sync_exist');
+var mm = require('mm');
+var Npm = require('../../proxy/npm');
+var Total = require('../../proxy/total');
+var should = require('should');
+
+describe('sync/sync_exist.js', function () {
+  describe('sync()', function () {
+    afterEach(mm.restore);
+
+    it('should sync first time ok', function (done) {
+      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+      mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
+      sync(function (err, data) {
+        should.not.exist(err);
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        done();
+      });
+    });
+
+    it('should sync common ok', function (done) {
+      mm.data(Npm, 'getAllSince', {
+        _updated: Date.now(),
+        'cnpmjs.org': {},
+        cutter: {}
+      });
+      mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
+      sync(function (err, data) {
+        should.not.exist(err);
+        data.successes.should.eql(['cnpmjs.org', 'cutter']);
+        done();
+      });
+    });
+  });
+});

view/web/footer.html

@@ -0,0 +1,5 @@
+Copyright 2013 - 2014 © cnpmjs.org
+|
+<a href="/">Home</a>
+|
+<script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>

view/web/layout.html

@@ -29,6 +29,10 @@
       table.downloads td.count {
         text-align: right;
       }
+
+      .nav-tabs{margin:20px 0;}
+      .nav-cont{display:none;}
+      .nav-cont.active{display:block;}
     </style>
     <script>
       $(function () {
@@ -41,26 +45,32 @@
             return location.href = '/browse/keyword/' + val;
           }
         });
+
+        $(".nav-tabs li").each(function (index) {
+          $(this).data("index", index);
+        })
+        .on("click", function (e) {
+          e.preventDefault();
+          $(".nav-tabs li.active,.nav-cont.active").removeClass("active");
+          $(this).addClass("active");
+          $(".nav-cont").eq($(this).data("index")).addClass("active");
+        });
       });
     </script>
   </head>
   <body>
     <div class="container">
       <div id="content-header">
-        <a href="/"><img src="http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg"></a>
+        <a href="/"><img src="{{logoURL}}"></a>
         <input type="text" id="search-input" class="form-control" placeholder="Search Packages">
       </div>
       <div id="content-body"><%- locals.body %></div>
     <div class="bottom">
       <hr/>
       <p>
-        Copyright 2013 - 2014 &copy; cnpmjs.org
-        |
-        <a href="/">Home</a>
-        |
-        <script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>
+        {{footer}}
       </p>
-      <a href="https://github.com/fengmk2/cnpmjs.org" id="fork" target="_blank">
+      <a href="https://github.com/cnpm/cnpmjs.org" id="fork" target="_blank">
         <img alt="Fork me on GitHub" src="http://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
     </a>
     </div>

view/web/package.html

@@ -1,7 +1,13 @@
   <div id="package">
     <h1>
       <%= package.name %>
-      <small>(<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)</small>
+      <small>
+      <% if (package._publish_on_cnpm) { %>
+      (Private package)
+      <% } else { %>
+      (<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)
+      <% } %>
+      </small>
     </h1>
 
     <% if (package.deprecated) { %>
@@ -12,175 +18,231 @@
       <p class="description"><%= package.description %></p>
     <% } %>
 
-    <pre class="sh"><code>$ cnpm install <%= package.name %></code></pre>
-
-    <table class="downloads">
-      <tbody>
-        <tr>
-          <td class="count"><%- download.today %></td><td> downloads today</td>
-          <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
-          <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
-        </tr>
-        <tr>
-          <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
-          <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
-          <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
-        </tr>
-      </tbody>
-    </table>
-
-    <table class="metadata">
-      <% if (package.maintainers) { %>
-      <tr>
-        <th>Maintainers</th>
-        <td>
-        <% package.maintainers.forEach(function (m) { %>
-          <div class="user">
-            <a class="username" href="/~<%= m.name %>">
-              <% if (m.gravatar) { %>
-              <img src="<%- m.gravatar %>" class="avatar">
-              <% } %>
-              <%= m.name %>
-            </a>
-          </div>
-        <% }) %>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <th>Version</th>
-        <td>
-          <b>
-          <%= package.version %>
-          </b>
-          <% if (package.fromNow) { %>
-          last updated
-          <%= package.fromNow %>
-          <% } %>
-        </td>
-      </tr>
-      <% if (package.license) { %>
-      <tr>
-        <th>License</th>
-        <td>
-          <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
-        </td>
-      </tr>
-      <% } %>
-      <%
-      if (typeof package.keywords === 'string') {
-        package.keywords = package.keywords.split(/\s*,?\s*/)
-      }
-      if (Array.isArray(package.keywords)) {
-      %>
-      <tr>
-        <th>Keywords</th>
-        <td><%-
-          package.keywords.map(function (kw) {
-            kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
-            return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
-          }).join(', ')
-          %></td>
-      </tr>
-      <% } %>
-      <% if (package.repository && package.repository !== 'undefined') {
-      var gh = package.repository.url &&
-          package.repository.url.match(
-          /^(?:https?:\/\/|git(?::\/\/|@))(gist.github.com|github.com)[:\/](.*?)(?:.git)?$/)
-      if (gh) {
-        gh = 'https://' + gh[1] + '/' + gh[2]
-      }
-      %>
-      <tr>
-        <th>Repository</td>
-        <td>
-          <% if (gh) { %><a href="<%= gh %>" target="_blank"><% } %>
-          <%= package.repository.url %><% if (gh) { %></a><% } %>
-          (<%= package.repository.type %>)
-        </td>
-      </tr>
-      <% } %>
-      <% if (typeof package.homepage === 'string') { %>
-      <tr>
-        <th>Homepage</td>
-        <td>
-          <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
-        </td>
-      </tr>
-      <% } %>
-      <% if (package.bugs && package.bugs.url) { %>
-      <tr>
-        <th>Bugs</td>
-        <td>
-          <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
-        </td>
-      </tr>
-      <% } %>
-      <tr>
-        <%
-          var deps = Object.keys(package.dependencies || {})
-          var l = deps.length
-        %>
-        <th>Dependencies<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          if (l === 0) {
-            %>None<%
-          } else {
-            var m = 200
-            if (l > m) deps = deps.slice(0, m)
-            deps.forEach(function(pkg, i) {
-              if (i > 0) { %>, <% }
-              %>
-              <a href="/package/<%= pkg %>"><%= pkg %></a><%
-            })
-            if (l > m) {
-              %>, and <%= l-m %> more<%
-            }
-          }
-          %>
-        </td>
-      </tr>
-
-      <% if (package.users) {
-        var starredBy = package.starredBy
-        var l = starredBy.length
-      %>
-      <tr>
-        <th>Starred by<%= l > 5 ? ' (' + l + ')' : '' %></th>
-        <td>
-          <%
-          var max = 20
-          if (l > max) {
-            starredBy = starredBy.sort(function (a, b) {
-              return Math.random() * 2 - 1
-            }).slice(0, max)
-          }
-          starredBy.forEach(function (user, i) {
-            if (i > 0) { %>, <% }
-            %><a href="/~<%= user %>"><%= user %></a><%
-          })
-          if (l > max) {
-            %><br><a href="/browse/star/<%= package.name %>">and
-            <%= (l-max) %> more</a><%
-          }
-          %>
-        </td>
-      </tr>
-      <% } %>
-    </table>
-
-    <div class="details">
-      <ul class="toc">
-        <% if (package.readme) { %>
-        <li><a href="#readme">Read Me</a></li>
-        <% } %>
-      </ul>
+    <pre class="sh"><code>$ <%- config.npmClientName %> install <%= package.name %> <% if (package.preferGlobal) { %>-g<% } %></code></pre>
 
+    <ul class="nav nav-tabs">
       <% if (package.readme) { %>
-      <section id="readme">
-      <%- package.readme %>
-      </section>
+        <li class="active"><a href="#readme">Readme.md</a></li>
+        <li><a href="#meta">package.json</a></li>
+      <% } else { %>
+        <li class="active"><a href="#meta">package.json</a></li>
       <% } %>
+    </ul>
+
+    <% if (package.readme) { %>
+    <div class="nav-cont active">
+      <div class="details">
+        <section id="readme">
+        <%- package.readme %>
+        </section>
+      </div>
+    </div>
+    <% } %>
+
+    <div class="nav-cont">
+      <table class="downloads">
+        <tbody>
+          <tr>
+            <td class="count"><%- download.today %></td><td> downloads today</td>
+            <td class="count"><%- download.thisweek %></td><td> downloads in this week</td>
+            <td class="count"><%- download.thismonth %></td><td> downloads in this month</td>
+          </tr>
+          <tr>
+            <td class="count"><%- download.lastday %></td><td> downloads in the last day</td>
+            <td class="count"><%- download.lastweek %></td><td> downloads in the last week</td>
+            <td class="count"><%- download.lastmonth %></td><td> downloads in the last month</td>
+          </tr>
+        </tbody>
+      </table>
+
+      <table class="metadata">
+        <% if (Array.isArray(package.maintainers) && package.maintainers.length > 0) { %>
+        <tr>
+          <th>Maintainers</th>
+          <td>
+          <% package.maintainers.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" href="/~<%= m.name %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <th>Version</th>
+          <td>
+            <b>
+            <%= package.version %>
+            </b>
+            <% if (package.fromNow) { %>
+            last updated
+            <%= package.fromNow %>
+            <% } %>
+          </td>
+        </tr>
+        <% if (package.license) { %>
+        <tr>
+          <th>License</th>
+          <td>
+            <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
+          </td>
+        </tr>
+        <% } %>
+        <%
+        if (typeof package.keywords === 'string') {
+          package.keywords = package.keywords.split(/\s*,?\s*/)
+        }
+        if (Array.isArray(package.keywords)) {
+        %>
+        <tr>
+          <th>Keywords</th>
+          <td><%-
+            package.keywords.map(function (kw) {
+              kw = kw.replace(/</g, '&lt;').replace(/"/g, '&quot;')
+              return '<a href="/browse/keyword/' + kw + '">' + kw + '</a>'
+            }).join(', ')
+            %></td>
+        </tr>
+        <% } %>
+        <% if (package.repository && package.repository !== 'undefined') { %>
+        <tr>
+          <th>Repository</td>
+          <td>
+            <a href="<%= package.repository.weburl || package.repository.url %>" target="_blank">
+              <%= package.repository.url %>
+            </a>
+            (<%= package.repository.type %>)
+          </td>
+        </tr>
+        <% } %>
+        <% if (typeof package.homepage === 'string') { %>
+        <tr>
+          <th>Homepage</td>
+          <td>
+            <a href="<%= encodeURI(package.homepage) %>" target="_blank"><%= package.homepage.replace(/</g, '&lt;') %></a>
+          </td>
+        </tr>
+        <% } %>
+        <% if (package.bugs && package.bugs.url) { %>
+        <tr>
+          <th>Bugs</td>
+          <td>
+            <a href="<%= package.bugs.url %>" target="_blank"><%= package.bugs.url %></a>
+          </td>
+        </tr>
+        <% } %>
+        <tr>
+          <%
+            var deps = Object.keys(package.dependencies || {})
+            var l = deps.length
+          %>
+          <th>Dependencies<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <tr>
+          <%
+            var deps = package.dependents || [];
+            var l = deps.length
+          %>
+          <th>Dependents<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            if (l === 0) {
+              %>None<%
+            } else {
+              var m = 200
+              if (l > m) deps = deps.slice(0, m)
+              deps.forEach(function(pkg, i) {
+                if (i > 0) { %>, <% }
+                %>
+                <a href="/package/<%= pkg %>"><%= pkg %></a><%
+              })
+              if (l > m) {
+                %>, and <%= l-m %> more<%
+              }
+            }
+            %>
+          </td>
+        </tr>
+
+        <% if (package.users) {
+          var starredBy = package.starredBy
+          var l = starredBy.length
+        %>
+        <tr>
+          <th>Starred by<%= l > 0 ? ' (' + l + ')' : '' %></th>
+          <td>
+            <%
+            var max = 20
+            if (l > max) {
+              starredBy = starredBy.sort(function (a, b) {
+                return Math.random() * 2 - 1
+              }).slice(0, max)
+            }
+            starredBy.forEach(function (user, i) {
+              if (i > 0) { %>, <% }
+              %><a href="/~<%= user %>"><%= user %></a><%
+            })
+            if (l > max) {
+              %><br><a href="/browse/star/<%= package.name %>">and
+              <%= (l-max) %> more</a><%
+            }
+            %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (Array.isArray(package.contributors) && package.contributors.length > 0) { %>
+        <tr>
+          <th>Contributors</th>
+          <td>
+          <% package.contributors.forEach(function (m) { %>
+            <span class="user">
+              <a class="username" target="_blank" href="<%= m.url %>">
+                <% if (m.gravatar) { %>
+                <img src="<%- m.gravatar %>" class="avatar">
+                <% } %>
+                <%= m.name %>
+              </a>
+            </span>
+          <% }) %>
+          </td>
+        </tr>
+        <% } %>
+
+        <% if (package.dist && package.dist.tarball) { %>
+        <tr>
+          <th>Download</th>
+          <td>
+            <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
+              <%= package.dist.tarball %>
+            </a>
+          </td>
+        </tr>
+        <% } %>
+      </table>
     </div>
   </div>

view/web/search.html

@@ -16,26 +16,43 @@
 <div id="search">
   <% if (!packages.length) { %>
     <div class="alert alert-warning">
-      Can not found package match <%= keyword %>. You can 
+      Can not found package match <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
     </div>
   <% } else if (packages[0].name !== keyword) { %>
     <div class="alert alert-info">
-      Can not found package <%= keyword %>. You can 
+      Can not found package <%= keyword %>. You can
       <a href="/sync/<%= keyword %>" target="_blank">SYNC</a> from official npm registry or
       <a href="https://npmjs.org/search?q=<%= keyword %>" target="_blank">SEARCH</a> in official npm website.
-    </div>    
+    </div>
   <% } %>
   <% if (packages.length) { %>
     <h1>
-      Packages match <span style="color: #09f;"><%= keyword %></span>
+      Packages match "<span style="color: #09f;"><%= keyword %></span>"
     </h1>
     <hr />
-    <% for (var i = 0; i < packages.length; i++) { %>
-      <div class="package <%= packages[i].name === keyword ? 'match' : '' %>">
-        <a href="/package/<%= packages[i].name %>" class="package-name"><%= packages[i].name %></a>
-        <span class="package-description"><%= packages[i].description %></span>
+    <% for (var i = 0; i < packages.length; i++) {
+      var item = packages[i];
+    %>
+      <div class="package <%= item.name === keyword ? 'match' : '' %>">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
+      </div>
+    <% } %>
+  <% } %>
+
+  <% if (keywords.length) { %>
+    <h1>
+      Keywords match "<span style="color: #09f;"><%= keyword %></span>"
+    </h1>
+    <hr />
+    <% for (var i = 0; i < keywords.length; i++) {
+      var item = keywords[i];
+    %>
+      <div class="package <%= item.name === keyword ? 'match' : '' %>">
+        <a href="/package/<%= item.name %>" class="package-name"><%= item.name %></a>
+        <span class="package-description"><%= item.description %></span>
       </div>
     <% } %>
   <% } %>

view/web/sync.html

@@ -11,6 +11,7 @@
   var $notify = $('#sync-notify');
   var timer;
   var name = '<%= name %>';
+  var resourceURL = '/sync/' + name;
   $(function() {
     var checkLogId = location.hash.match(/logid=(\d+)/);
     var logid = checkLogId ? checkLogId[1] : '';
@@ -18,7 +19,7 @@
       return getSyncLog(logid);
     }
     $.ajax({
-      url: location.pathname,
+      url: resourceURL,
       type: 'PUT',
       dataType: 'json',
       success: handleSyncSucess,
@@ -53,7 +54,7 @@
   var hasFail = false;
   function getSyncLog(id) {
     $.ajax({
-      url: location.pathname + '/log/' + id + '?offset=' + offset,
+      url: resourceURL + '/log/' + id + '?offset=' + offset,
       type: 'GET',
       dataType: 'json',
       success: function (data) {

worker.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,7 +16,7 @@
  */
 
 var graceful = require('graceful');
-
+var logger = require('./common/logger');
 var config = require('./config');
 var registry = require('./servers/registry');
 var web = require('./servers/web');
@@ -34,6 +34,7 @@ graceful({
     if (err.message) {
       err.message += ' (uncaughtException throw ' + throwErrorCount + ' times on pid:' + process.pid + ')';
     }
-    console.error(err);
+    console.error(err.stack);
+    logger.error(err);
   }
 });