Release 2.0.0-beta3

fix(sync): should not sync package when maintainers sort change

.gitignore

@@ -28,3 +28,4 @@ bin/test.sql
 coverage/
 config/web_readme.md
 .tmp/
+*.sqlite

.jshintrc

@@ -24,7 +24,7 @@
                                 //   "single" : require single quotes
                                 //   "double" : require double quotes
     "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
-    "unused"        : false,    // true: Require all defined variables be used
+    "unused"        : true,     // true: Require all defined variables be used
     "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
     "trailing"      : false,    // true: Prohibit trailing whitespaces
     "maxparams"     : false,    // {int} Max number of formal params allowed per function

.npmignore

@@ -20,3 +20,4 @@ coverage/
 config/web_readme.md
 .dist/
 config/config.js
+*.sqlite

.travis.yml

@@ -1,5 +1,6 @@
 language: node_js
 node_js:
-  - '0.11'
-script: "make test-travis"
+  - '0.11.14'
+before_install: "npm install --build-from-source"
+script: "make test-travis-all"
 after_script: "npm install coveralls@2 && cat ./coverage/lcov.info | coveralls"

History.md

@@ -1,8 +1,69 @@
 
+2.0.0-beta3 / 2014-11-12
+==================
+
+ * fix(sync): should not sync package when maintainers sort change
+ * fix(package): detect package is private or not
+ * fix(maintainer): fix missing maintainers
+
+2.0.0-beta2 / 2014-11-09
+==================
+
+ * fix(sync): add missing syncUpstreamFirst argument
+
+2.0.0-beta1 / 2014-11-07
+==================
+
+ * refactor(sync_worker): only sync request need to sync upstream first
+ * fix(sync_worker): make sure end event will emit
+ * fix: mv readme.md script to public/js/readme.js
+ * fix(sync): hotfix co uncaughtException
+ * feat(sync): sync python dist
+ * pin autod@1
+ * remove useless comment
+ * refactor models/_module_maintainer_class_methods.js
+
+2.0.0-beta0 / 2014-11-02
+==================
+
+ * ungrade koa-markdown to use remarkable, close #482
+ * fix(module_log): limit module sync log size to 1MB
+ * refactor(config): remove adaptScope config key
+ * chore(Makefile): $ make install-production
+ * fix(sequelize): show warnning message when using old config.js
+ * docs(readme): Migrating from 1.x to 2.x
+ * feat(sync): add min sync interval time detect
+ * refactor(dispatch): remove unused codes
+ * use sequelize to connect database
+
+1.7.1 / 2014-10-15
+==================
+
+  * fix typo in sync popular, fix [#477](https://github.com/cnpm/cnpmjs.org/issues/477)
+
+1.7.0 / 2014-10-15
+==================
+
+  * Merge pull request [#475](https://github.com/cnpm/cnpmjs.org/issues/475) from KidkArolis/configurable-short-registry-url
+  * If sourceNpmRegistry is cnpm - use it in getShort
+  * Merge pull request [#472](https://github.com/cnpm/cnpmjs.org/issues/472) from cnpm/issue468-upon
+  * fix test label
+  * add sync popular modules, close [#468](https://github.com/cnpm/cnpmjs.org/issues/468)
+  * add sendmail test script
+
+1.6.1 / 2014-10-09
+==================
+
+ * make test on travis faster
+ * ensure not sync user also has his own package names
+ * add [v1.6.x-upgrade.sql](https://github.com/cnpm/cnpmjs.org/blob/master/docs/update_sqls/v1.6.x-upgrade.sql)
+ * save npm original package maintainers to npm_module_maintainer table. fixed [#464](https://github.com/cnpm/cnpmjs.org/issues/464)
+ * use simple 404
+
 1.6.0 / 2014-10-08
 ==================
 
- * list user all packages api. fixed #462
+ * list user all packages api. fixed [#462](https://github.com/cnpm/cnpmjs.org/issues/462)
  * add node-dev: $ make dev
  * always start sync worker
  * update node mailer
@@ -23,14 +84,14 @@
 1.5.3 / 2014-09-24
 ==================
 
- * support sync upstream first. fixed #451
+ * support sync upstream first. fixed [#451](https://github.com/cnpm/cnpmjs.org/issues/451)
 
 1.5.2 / 2014-09-24
 ==================
 
  * support im url on user profile page; update bootstrap to 3.2.0
 
-1.5.1 / 2014-09-23 
+1.5.1 / 2014-09-23
 ==================
 
   * search support case insensitive, close [#450](https://github.com/cnpm/cnpmjs.org/issues/450)

Makefile

@@ -3,23 +3,27 @@ REPORTER = spec
 TIMEOUT = 30000
 MOCHA_OPTS =
 REGISTRY = --registry=https://registry.npm.taobao.org
+DB = sqlite
 
 install:
-	@npm install $(REGISTRY) \
+	@npm install --build-from-source $(REGISTRY) \
 		--disturl=https://npm.taobao.org/dist
 
-jshint: install
-	@-./node_modules/.bin/jshint ./
+install-production production:
+	@NODE_ENV=production $(MAKE) install
 
-pretest:
+jshint: install
+	@-node_modules/.bin/jshint ./
+
+init-database:
+	@node --harmony test/init_db.js
+
+init-mysql:
 	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
 	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
-	@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
-	@mysql -uroot 'cnpmjs_test' -e 'show tables;'
-	@rm -rf .tmp/dist
 
-test: install pretest
-	@NODE_ENV=test ./node_modules/.bin/mocha \
+test: install init-database
+	@NODE_ENV=test DB=${DB} node_modules/.bin/mocha \
 		--harmony \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
@@ -30,10 +34,18 @@ test: install pretest
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
-test-cov cov: install pretest
-	@NODE_ENV=test node --harmony \
+test-sqlite:
+	@$(MAKE) test DB=sqlite
+
+test-mysql: init-mysql
+	@$(MAKE) test DB=mysql
+
+test-all: test-sqlite test-mysql
+
+test-cov cov: install init-database
+	@NODE_ENV=test DB=${DB} node --harmony \
 		node_modules/.bin/istanbul cover --preserve-comments \
-		./node_modules/.bin/_mocha \
+		node_modules/.bin/_mocha \
 		-- -u exports \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
@@ -44,10 +56,17 @@ test-cov cov: install pretest
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
-test-travis: install pretest
-	@NODE_ENV=test node --harmony \
+test-cov-sqlite:
+	@$(MAKE) test-cov DB=sqlite
+
+test-cov-mysql: init-mysql
+	@$(MAKE) test-cov DB=mysql
+
+test-travis: install init-database
+	@NODE_ENV=test DB=${DB} CNPM_SOURCE_NPM=https://registry.npmjs.org CNPM_SOURCE_NPM_ISCNPM=false \
+		node --harmony \
 		node_modules/.bin/istanbul cover --preserve-comments \
-		./node_modules/.bin/_mocha \
+		node_modules/.bin/_mocha \
 		--report lcovonly \
 		-- \
 		--reporter dot \
@@ -59,18 +78,26 @@ test-travis: install pretest
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
+test-travis-sqlite:
+	@$(MAKE) test-travis DB=sqlite
+
+test-travis-mysql: init-mysql
+	@$(MAKE) test-travis DB=mysql
+
+test-travis-all: test-travis-sqlite test-travis-mysql
+
 dev:
 	@node_modules/.bin/node-dev --harmony dispatch.js
 
 contributors: install
-	@./node_modules/.bin/contributors -f plain -o AUTHORS
+	@node_modules/.bin/contributors -f plain -o AUTHORS
 
 autod: install
-	@./node_modules/.bin/autod -w \
-		--prefix "~"\
+	@node_modules/.bin/autod -w \
+		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
-		--dep bluebird \
-		--devdep mocha,should,istanbul-harmony,jshint
+		--dep bluebird,mysql \
+		--keep should,supertest,should-http,chunkstream,mm,pedding
 	@$(MAKE) install
 
 .PHONY: test

README.md

@@ -8,6 +8,7 @@ cnpmjs.org
 [![David deps][david-image]][david-url]
 [![node version][node-image]][node-url]
 [![npm download][download-image]][download-url]
+[![gitter][gitter-image]][gitter-url]
 
 [npm-image]: http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square
 [npm-url]: http://cnpmjs.org/package/cnpmjs.org
@@ -23,6 +24,8 @@ cnpmjs.org
 [node-url]: http://nodejs.org/download/
 [download-image]: https://img.shields.io/npm/dm/cnpmjs.org.svg?style=flat-square
 [download-url]: https://npmjs.org/package/cnpmjs.org
+[gitter-image]: https://badges.gitter.im/Join%20Chat.svg
+[gitter-url]: https://gitter.im/cnpm/cnpmjs.org?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge
 
 ![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
@@ -44,7 +47,8 @@ Our goal is to provide a low cost maintenance and easy to use solution for priva
 * **Support "scoped" packages**: [npm/npm#5239](https://github.com/npm/npm/issues/5239)
 * **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 You can get the source code through `npm` or `git`.
-* **Low cost and easy maintenance**: `package.json` info store in MySQL, tarball(tgz file) store in CDN or other store systems.
+* **Low cost and easy maintenance**: `package.json` info store in MySQL, MariaDB, SQLite or PostgreSQL databases,
+tarball(tgz file) store in CDN or other store systems.
 * **Automatic synchronization**: automatic synchronization from any registry specified, support two sync modes:
   - Sync all modules from a specified registry, like [npm registry](http://registry.npmjs.org).
   - Only sync the modules that exists in your own registry.
@@ -56,6 +60,9 @@ And it easy to wrap for your own registry which build with `cnpmjs.org`.
 only need to change the registry in config. Even include manual synchronization (through `install` command).
 * **Version badge**: base on [shields.io](http://shields.io/) ![cnpm-badge](http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square)
 
+**PROTIP** Be sure to read [Migrating from 1.x to 2.x](https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x)
+as well as [New features in 2.x](https://github.com/cnpm/cnpmjs.org/wiki/New-features-in-2.x).
+
 ## Getting Start
 
 * @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
@@ -70,13 +77,11 @@ only need to change the registry in config. Even include manual synchronization
 ### Dependencies
 
 * [node](http://nodejs.org) >=0.11.12, use `--harmony`
-* [mysql](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
-
-### Start MySQL
-
-```bash
-$ nohup mysqld &
-```
+* Databases: only required one type
+  * [sqlite3](https://npm.taobao.org/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
+  * [MySQL](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
+  * MariaDB
+  * PostgreSQL
 
 ### Clone codes and run test
 

common/logger.js

@@ -17,10 +17,10 @@
 
 var formater = require('error-formater');
 var Logger = require('mini-logger');
-var config = require('../config');
 var utility = require('utility');
-var mail = require('./mail');
 var util = require('util');
+var config = require('../config');
+var mail = require('./mail');
 
 var isTEST = process.env.NODE_ENV === 'test';
 var categories = ['sync_info', 'sync_error'];

common/mysql.js

@@ -1,80 +0,0 @@
-/*!
- * cnpmjs.org - common/mysql.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var ready = require('ready');
-var mysql = require('mysql');
-var config = require('../config');
-
-var server = config.mysqlServers[0];
-
-// TODO: query timeout
-var pool = mysql.createPool({
-  host: server.host,
-  port: server.port,
-  user: server.user,
-  password: server.password,
-  database: config.mysqlDatabase,
-  connectionLimit: config.mysqlMaxConnections,
-  multipleStatements: true,
-});
-
-exports.pool = pool;
-
-exports.query = function (sql, values, cb) {
-  if (typeof values === 'function') {
-    cb = values;
-    values = null;
-  }
-  pool.query(sql, values, function (err, rows) {
-    cb(err, rows);
-  });
-};
-
-exports.queryOne = function (sql, values, cb) {
-  if (typeof values === 'function') {
-    cb = values;
-    values = null;
-  }
-  exports.query(sql, values, function (err, rows) {
-    if (rows) {
-      rows = rows[0];
-    }
-    cb(err, rows);
-  });
-};
-
-exports.escape = function (val) {
-  return pool.escape(val);
-};
-
-ready(exports);
-
-thunkify(exports);
-
-function init() {
-  exports.query('show tables', function (err, rows) {
-    if (err) {
-      console.error('[%s] [worker:%s] mysql init error: %s', Date(), process.pid, err);
-      setTimeout(init, 1000);
-      return;
-    }
-    console.log('[%s] [worker:%s] mysql ready, got %d tables', Date(), process.pid, rows.length);
-    exports.ready(true);
-  });
-}
-init();

common/sequelize.js

@@ -0,0 +1,75 @@
+/**!
+ * cnpmjs.org - common/sequelize.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var Sequelize = require('sequelize');
+var DataTypes = require('sequelize/lib/data-types');
+var config = require('../config');
+
+if (config.mysqlServers && config.database.dialect === 'sqlite') {
+  // https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x
+  // forward compat with old style on 1.x
+  // mysqlServers: [
+  //   {
+  //     host: '127.0.0.1',
+  //     port: 3306,
+  //     user: 'root',
+  //     password: ''
+  //   }
+  // ],
+  // mysqlDatabase: 'cnpmjs_test',
+  // mysqlMaxConnections: 4,
+  // mysqlQueryTimeout: 5000,
+
+  console.warn('[WARNNING] your config.js was too old, please @see https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x');
+  var server = config.mysqlServers[0];
+  config.database = {
+    db: config.mysqlDatabase,
+    username: server.user,
+    password: server.password,
+    dialect: 'mysql',
+    host: server.host,
+    port: server.port,
+    pool: {
+      maxConnections: config.mysqlMaxConnections || 10,
+      minConnections: 0,
+      maxIdleTime: 30000
+    },
+    logging: !!process.env.SQL_DEBUG,
+  };
+}
+
+var database = config.database;
+
+// sync database before app start, defaul is false
+database.syncFirst = false;
+
+// add longtext for mysql
+Sequelize.LONGTEXT = DataTypes.LONGTEXT = DataTypes.TEXT;
+if (config.dialect === 'mysql') {
+  Sequelize.LONGTEXT = DataTypes.LONGTEXT = 'LONGTEXT';
+}
+
+database.define = {
+  timestamps: true,
+  createdAt: 'gmt_create',
+  updatedAt: 'gmt_modified',
+  charset: 'utf8',
+  collate: 'utf8_general_ci',
+};
+
+var sequelize = new Sequelize(database.db, database.username, database.password, database);
+
+module.exports = sequelize;

common/urllib.js

@@ -33,53 +33,3 @@ var client = urllib.create({
 
 module.exports = client;
 module.exports.USER_AGENT = urllib.USER_AGENT;
-
-function startMonitor() {
-  var statInterval = 60000;
-
-  var agents = [
-    ['httpAgent', httpAgent],
-    ['httpsAgent', httpsAgent]
-  ];
-
-  function agentStat() {
-    for (var i = 0; i < agents.length; i++) {
-      var type = agents[i][0];
-      var agent = agents[i][1];
-      var rate = '0';
-      if (agent.createSocketCount > 0) {
-        rate = (agent.requestCount / agent.createSocketCount).toFixed(0);
-      }
-      console.info('[%s] socket: %d created, %d close, %d timeout, request: %d requests, %s req/socket',
-        type,
-        agent.createSocketCount,
-        agent.closeSocketCount,
-        agent.timeoutSocketCount,
-        agent.requestCount,
-        rate
-      );
-
-      var name;
-      for (name in agent.sockets) {
-        console.info('working sockets %s: %d', name, agent.sockets[name].length);
-      }
-      for (name in agent.freeSockets) {
-        console.info('free sockets %s: %d', name, agent.freeSockets[name].length);
-      }
-      for (name in agent.requests) {
-        console.info('pedding requests %s: %d', name, agent.requests[name].length);
-      }
-      if (agent.requestCount >= 100000000) {
-        agent.requestCount = 0;
-        agent.createSocketCount = 0;
-        agent.closeSocketCount = 0;
-        agent.timeoutSocketCount = 0;
-      }
-    }
-  }
-
-  agentStat();
-  return setInterval(agentStat, statInterval);
-}
-
-startMonitor();

config/index.js

@@ -15,13 +15,12 @@
  * Module dependencies.
  */
 
+var mkdirp = require('mkdirp');
+var copy = require('copy-to');
 var path = require('path');
 var fs = require('fs');
 var os = require('os');
-var mkdirp = require('mkdirp');
-var copy = require('copy-to');
 
-fs.existsSync = fs.existsSync || path.existsSync;
 var version = require('../package.json').version;
 
 var root = path.dirname(__dirname);
@@ -51,7 +50,7 @@ var config = {
   // max request json body size
   jsonLimit: '10mb',
   // log dir name
-  logdir: path.join(root, '.tmp', 'logs'),
+  logdir: path.join(root, '.tmp/logs'),
   // update file template dir
   uploadDir: path.join(root, '.dist'),
   // web page viewCache
@@ -115,21 +114,38 @@ var config = {
   backupFilePrefix: '/cnpm/backup/',
 
   /**
-   * mysql config
+   * database config
    */
 
-  mysqlServers: [
-    {
-      host: '127.0.0.1',
-      port: 3306,
-      user: 'root',
-      password: ''
-    }
-  ],
-  mysqlDatabase: 'cnpmjs_test',
-  mysqlMaxConnections: 4,
-  mysqlQueryTimeout: 5000,
+  database: {
+    db: 'cnpmjs_test',
+    username: 'root',
+    password: '',
 
+    // the sql dialect of the database
+    // - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
+    dialect: 'sqlite',
+
+    // custom host; default: 127.0.0.1
+    host: '127.0.0.1',
+
+    // custom port; default: 3306
+    port: 3306,
+
+    // use pooling in order to reduce db connection overload and to increase speed
+    // currently only for mysql and postgresql (since v1.5.0)
+    pool: {
+      maxConnections: 10,
+      minConnections: 0,
+      maxIdleTime: 30000
+    },
+
+    // the storage engine for 'sqlite'
+    // default store into ~/cnpmjs.org.sqlite
+    storage: path.join(process.env.HOME || root, 'cnpmjs.org.sqlite'),
+
+    logging: !!process.env.SQL_DEBUG,
+  },
 
   // redis config
   // use for koa-limit module as storage
@@ -156,10 +172,6 @@ var config = {
     '@cnpmtest'
   ],
 
-  // redirect @cnpm/private-package => private-package
-  // forward compatbility for update from lower version cnpmjs.org
-  adaptScope: true,
-
   // force user publish with scope
   // but admins still can publish without scope
   forcePublishWithScope: true,
@@ -167,7 +179,7 @@ var config = {
   // some registry already have some private packages in global scope
   // but we want to treat them as scoped private packages,
   // so you can use this white list.
-  privatePackages: ['private-package'],
+  privatePackages: [],
 
   /**
    * sync configs
@@ -179,6 +191,9 @@ var config = {
   disturl: 'http://nodejs.org/dist',
   syncDist: false,
 
+  pythonDisturl: 'https://www.python.org/ftp',
+  syncPythonDist: false,
+
   // the official npm registry
   // cnpm wont directly sync from this one
   // but sometimes will request it for some package infomations
@@ -207,6 +222,14 @@ var config = {
   // sync interval, default is 10 minutes
   syncInterval: '10m',
 
+  // sync polular modules, default to false
+  // because cnpm can't auto sync tag change for now
+  // so we want to sync popular modules to ensure their tags
+  syncPopular: false,
+  syncPopularInterval: '1h',
+  // top 100
+  topPopular: 100,
+
   // sync devDependencies or not, default is false
   syncDevDependencies: false,
 

controllers/download.js

@@ -1,83 +0,0 @@
-/**!
- * cnpmjs.org - controllers/download.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var moment = require('moment');
-var DownloadTotal = require('../proxy/download');
-
-exports.total = function (name, callback) {
-  if (typeof name === 'function') {
-    callback = name;
-    name = null;
-  }
-  var end = moment();
-  var start = end.clone().subtract(1, 'months').startOf('month');
-  var lastday = end.clone().subtract(1, 'days').format('YYYY-MM-DD');
-  var lastweekStart = end.clone().subtract(1, 'weeks').startOf('week');
-  var lastweekEnd = lastweekStart.clone().endOf('week').format('YYYY-MM-DD');
-  var lastmonthEnd = start.clone().endOf('month').format('YYYY-MM-DD');
-  var thismonthStart = end.clone().startOf('month').format('YYYY-MM-DD');
-  var thisweekStart = end.clone().startOf('week').format('YYYY-MM-DD');
-  start = start.format('YYYY-MM-DD');
-  end = end.format('YYYY-MM-DD');
-  lastweekStart = lastweekStart.format('YYYY-MM-DD');
-  var method = name ? 'getModuleTotal' : 'getTotal';
-  var args = [start, end];
-  if (name) {
-    args.unshift(name);
-  }
-  args.push(function (err, rows) {
-    if (err) {
-      return callback(err);
-    }
-
-    var download = {
-      today: 0,
-      thisweek: 0,
-      thismonth: 0,
-      lastday: 0,
-      lastweek: 0,
-      lastmonth: 0,
-    };
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (r.date === end) {
-        download.today += r.count;
-      }
-      if (r.date >= thismonthStart) {
-        download.thismonth += r.count;
-      }
-      if (r.date >= thisweekStart) {
-        download.thisweek += r.count;
-      }
-
-      if (r.date === lastday) {
-        download.lastday += r.count;
-      }
-      if (r.date >= lastweekStart && r.date <= lastweekEnd) {
-        download.lastweek += r.count;
-      }
-      if (r.date >= start && r.date <= lastmonthEnd) {
-        download.lastmonth += r.count;
-      }
-    }
-    callback(null, download);
-  });
-
-  DownloadTotal[method].apply(DownloadTotal, args);
-};
-
-thunkify(exports);

controllers/registry/package/deprecate.js

@@ -1,5 +1,5 @@
 /**!
- * cnpmjs.org - controllers/registry/deprecate.js
+ * cnpmjs.org - controllers/registry/package/deprecate.js
  *
  * Copyright(c) fengmk2 and other contributors.
  * MIT Licensed
@@ -14,20 +14,20 @@
  * Module dependencies.
  */
 
-var Module = require('../../proxy/module');
+var packageService = require('../../../services/package');
 
 module.exports = deprecateVersions;
 
 /**
  * @see https://github.com/cnpm/cnpmjs.org/issues/415
  */
-function* deprecateVersions(next) {
+function* deprecateVersions() {
   var body = this.request.body;
   var name = this.params.name || this.params[0];
 
   var tasks = [];
   for (var version in body.versions) {
-    tasks.push(Module.get(name, version));
+    tasks.push(packageService.getModule(name, version));
   }
   var rs = yield tasks;
 
@@ -46,12 +46,12 @@ function* deprecateVersions(next) {
     var data = body.versions[row.package.version];
     if (typeof data.deprecated === 'string') {
       row.package.deprecated = data.deprecated;
-      updateTasks.push(Module.updatePackage(row.id, row.package));
+      updateTasks.push(packageService.updateModulePackage(row.id, row.package));
     }
   }
   yield updateTasks;
   // update last modified
-  yield* Module.updateLastModified(name);
+  yield* packageService.updateModuleLastModified(name);
 
   this.status = 201;
   this.body = {

controllers/registry/package/download.js

@@ -0,0 +1,109 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/download.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:download');
+var mime = require('mime');
+var utility = require('utility');
+var defer = require('co-defer');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+var common = require('../../../lib/common');
+var downloadAsReadStream = require('../../utils').downloadAsReadStream;
+var packageService = require('../../../services/package');
+var downloadTotalService = require('../../../services/download_total');
+
+var _downloads = {};
+
+module.exports = function* download(next) {
+  var name = this.params.name || this.params[0];
+  var filename = this.params.filename || this.params[1];
+  var version = filename.slice(name.length + 1, -4);
+  var row = yield* packageService.getModule(name, version);
+  // can not get dist
+  var url = null;
+
+  if (typeof nfs.url === 'function') {
+    url = nfs.url(common.getCDNKey(name, filename));
+  }
+
+  debug('download %s %s %s %s', name, filename, version, url);
+
+  if (!row || !row.package || !row.package.dist) {
+    if (!url) {
+      return yield* next;
+    }
+    this.status = 302;
+    this.set('Location', url);
+    _downloads[name] = (_downloads[name] || 0) + 1;
+    return;
+  }
+
+  var dist = row.package.dist;
+  if (!dist.key) {
+    debug('get tarball by 302, url: %s', dist.tarball || url);
+    this.status = 302;
+    this.set('Location', dist.tarball || url);
+    _downloads[name] = (_downloads[name] || 0) + 1;
+    return;
+  }
+
+  // else use `dist.key` to get tarball from nfs
+  if (!nfs.download) {
+    return yield* next;
+  }
+
+  _downloads[name] = (_downloads[name] || 0) + 1;
+
+  if (typeof dist.size === 'number' && dist.size > 0) {
+    this.length = dist.size;
+  }
+  this.type = mime.lookup(dist.key);
+  this.attachment(filename);
+  this.etag = dist.shasum;
+
+  this.body = yield* downloadAsReadStream(dist.key);
+};
+
+defer.setInterval(function* () {
+  // save download count
+  var totals = [];
+  for (var name in _downloads) {
+    var count = _downloads[name];
+    totals.push([name, count]);
+  }
+  _downloads = {};
+
+  if (totals.length === 0) {
+    return;
+  }
+
+  debug('save download total: %j', totals);
+
+  var date = utility.YYYYMMDD();
+  for (var i = 0; i < totals.length; i++) {
+    var item = totals[i];
+    var name = item[0];
+    var count = item[1];
+    try {
+      yield* downloadTotalService.plusModuleTotal({ name: name, date: date, count: count });
+    } catch (err) {
+      err.message += '; name: ' + name + ', count: ' + count + ', date: ' + date;
+      logger.error(err);
+      // save back to _downloads, try again next time
+      _downloads[name] = (_downloads[name] || 0) + count;
+    }
+  }
+}, 5000);

controllers/registry/package/list.js

@@ -0,0 +1,217 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/list.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
+var packageService = require('../../../services/package');
+var npmService = require('../../../services/npm');
+var config = require('../../../config');
+var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var SyncModuleWorker = require('../../sync_module_worker');
+
+/**
+ * list all version of a module
+ * GET /:name
+ */
+module.exports = function* list() {
+  var orginalName = this.params.name || this.params[0];
+  var name = orginalName;
+  var rs = yield [
+    packageService.getModuleLastModified(name),
+    packageService.listModuleTags(name)
+  ];
+  var modifiedTime = rs[0];
+  var tags = rs[1];
+
+  debug('show %s(%s), last modified: %s, tags: %j', name, orginalName, modifiedTime, tags);
+  if (modifiedTime) {
+    // find out the latest modfied time
+    // because update tags only modfied tag, wont change module gmt_modified
+    for (var i = 0; i < tags.length; i++) {
+      var tag = tags[i];
+      if (tag.gmt_modified > modifiedTime) {
+        modifiedTime = tag.gmt_modified;
+      }
+    }
+
+    // use modifiedTime as etag
+    this.set('ETag', '"' + modifiedTime.getTime() + '"');
+
+    // must set status first
+    this.status = 200;
+    if (this.fresh) {
+      debug('%s not change at %s, 304 return', name, modifiedTime);
+      this.status = 304;
+      return;
+    }
+  }
+
+  var r = yield [
+    packageService.listModulesByName(name),
+    packageService.listStarUserNames(name),
+    packageService.listMaintainers(name),
+  ];
+  var rows = r[0];
+  var starUsers = r[1];
+  var maintainers = r[2];
+
+  debug('show %s got %d rows, %d tags, %d star users, maintainers: %j',
+    name, rows.length, tags.length, starUsers.length, maintainers);
+
+  var starUserMap = {};
+  for (var i = 0; i < starUsers.length; i++) {
+    starUserMap[starUsers[i]] = true;
+  }
+  starUsers = starUserMap;
+
+  if (rows.length === 0) {
+    // check if unpublished
+    var unpublishedInfo = yield* packageService.getUnpublishedModule(name);
+    debug('show unpublished %j', unpublishedInfo);
+    if (unpublishedInfo) {
+      this.status = 404;
+      this.body = {
+        _id: orginalName,
+        name: orginalName,
+        time: {
+          modified: unpublishedInfo.package.time,
+          unpublished: unpublishedInfo.package,
+        },
+        _attachments: {}
+      };
+      return;
+    }
+  }
+
+  // if module not exist in this registry,
+  // sync the module backend and return package info from official registry
+  if (rows.length === 0) {
+    if (!this.allowSync) {
+      this.status = 404;
+      this.body = {
+        error: 'not_found',
+        reason: 'document not found'
+      };
+      return;
+    }
+
+    // start sync
+    var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
+    debug('start sync %s, get log id %s', name, logId);
+
+    // try to get package from official registry
+    var r = yield* npmService.request('/' + name, {
+      registry: config.officialNpmRegistry
+    });
+
+    debug('requet from officialNpmRegistry response %s', r.status);
+    if (r.status !== 200) {
+      this.status = 404;
+      this.body = {
+        error: 'not_found',
+        reason: 'document not found'
+      };
+      return;
+    }
+
+    this.body = r.data;
+    return;
+  }
+
+  var latestMod = null;
+  var readme = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  var times = {};
+  var attachments = {};
+  var createdTime = null;
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    var pkg = row.package;
+    setDownloadURL(pkg, this);
+    pkg._cnpm_publish_time = row.publish_time;
+    versions[pkg.version] = pkg;
+
+    var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+      readme = pkg.readme;
+    }
+
+    delete pkg.readme;
+    if (maintainers.length > 0) {
+      pkg.maintainers = maintainers;
+    }
+
+    if (!createdTime || t < createdTime) {
+      createdTime = t;
+    }
+  }
+
+  if (modifiedTime && createdTime) {
+    var ts = {
+      modified: modifiedTime,
+      created: createdTime,
+    };
+    for (var t in times) {
+      ts[t] = times[t];
+    }
+    times = ts;
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  var rev = String(latestMod.id);
+  var pkg = latestMod.package;
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = pkg.version;
+  }
+
+  var info = {
+    _id: orginalName,
+    _rev: rev,
+    name: orginalName,
+    description: pkg.description,
+    "dist-tags": distTags,
+    maintainers: pkg.maintainers,
+    time: times,
+    users: starUsers,
+    author: pkg.author,
+    repository: pkg.repository,
+    versions: versions,
+    readme: readme,
+    _attachments: attachments,
+  };
+
+  info.readmeFilename = pkg.readmeFilename;
+  info.homepage = pkg.homepage;
+  info.bugs = pkg.bugs;
+  info.license = pkg.license;
+
+  debug('show module %s: %s, latest: %s', orginalName, rev, latestMod.version);
+  this.body = info;
+};

controllers/registry/package/list_all.js

@@ -0,0 +1,30 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/list_all.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../../services/package');
+
+// GET /-/all
+// List all packages names
+// https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L86
+module.exports = function* () {
+  var updated = Date.now();
+  var names = yield* packageService.listAllPublicModuleNames();
+  var result = { _updated: updated };
+  names.forEach(function (name) {
+    result[name] = true;
+  });
+  this.body = result;
+};

controllers/registry/package/list_since.js

@@ -0,0 +1,59 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/list_since.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../../services/package');
+
+var A_WEEK_MS = 3600000 * 24 * 7;
+
+// GET /-/all/since?stale=update_after&startkey={key}
+// List packages names since startkey
+// https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L89
+module.exports = function* () {
+  var query = this.query;
+  if (query.stale !== 'update_after') {
+    this.status = 400;
+    this.body = {
+      error: 'query_parse_error',
+      reason: 'Invalid value for `stale`.'
+    };
+    return;
+  }
+
+  var startkey = Number(query.startkey);
+  if (!startkey) {
+    this.status = 400;
+    this.body = {
+      error: 'query_parse_error',
+      reason: 'Invalid value for `startkey`.'
+    };
+    return;
+  }
+
+  var updated = Date.now();
+  if (updated - startkey > A_WEEK_MS) {
+    startkey = updated - A_WEEK_MS;
+    console.warn('[%s] list modules since time out of range: query: %j, ip: %s',
+      Date(), query, this.ip);
+  }
+
+  var names = yield* packageService.listPublicModuleNamesSince(startkey);
+  var result = { _updated: updated };
+  names.forEach(function (name) {
+    result[name] = true;
+  });
+
+  this.body = result;
+};

controllers/registry/package/remove.js

@@ -0,0 +1,67 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/remove.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:remove');
+var urlparse = require('url').parse;
+var packageService = require('../../../services/package');
+var totalService = require('../../../services/total');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+
+// DELETE /:name/-rev/:rev
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L25
+module.exports = function* remove(next) {
+  var name = this.params.name || this.params[0];
+  var rev = this.params.rev || this.params[1];
+  debug('remove all the module with name: %s, id: %s', name, rev);
+
+  var mods = yield* packageService.listModulesByName(name);
+  debug('removeAll module %s: %d', name, mods.length);
+  var mod = mods[0];
+  if (!mod) {
+    return yield* next;
+  }
+
+  yield [
+    packageService.removeModulesByName(name),
+    packageService.removeModuleTags(name),
+    totalService.plusDeleteModule(),
+  ];
+
+  var keys = [];
+  for (var i = 0; i < mods.length; i++) {
+    var row = mods[i];
+    var dist = row.package.dist;
+    var key = dist.key;
+    if (!key) {
+      key = urlparse(dist.tarball).pathname;
+    }
+    key && keys.push(key);
+  }
+
+  try {
+    yield keys.map(function (key) {
+      return nfs.remove(key);
+    });
+  } catch (err) {
+    logger.error(err);
+  }
+
+  // remove the maintainers
+  yield* packageService.removeAllMaintainers(name);
+
+  this.body = { ok: true };
+};

controllers/registry/package/remove_version.js

@@ -0,0 +1,74 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/remove_version.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:remove_version');
+var packageService = require('../../../services/package');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+var getCDNKey = require('../../../lib/common').getCDNKey;
+
+// DELETE /:name/download/:filename/-rev/:rev
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L97
+module.exports = function* removeOneVersion(next) {
+  var name = this.params.name || this.params[0];
+  var filename = this.params.filename || this.params[1];
+  var id = Number(this.params.rev || this.params[2]);
+  // cnpmjs.org-2.0.0.tgz
+  var version = filename.split(name + '-')[1];
+  if (version) {
+    // 2.0.0.tgz
+    version = version.substring(0, version.lastIndexOf('.tgz'));
+  }
+  if (!version) {
+    return yield* next;
+  }
+
+  debug('remove tarball with filename: %s, version: %s, revert to => rev id: %s', filename, version, id);
+
+  if (isNaN(id)) {
+    return yield* next;
+  }
+
+  var rs = yield [
+    packageService.getModuleById(id),
+    packageService.getModule(name, version),
+  ];
+  var revertTo = rs[0];
+  var mod = rs[1]; // module need to delete
+  if (!mod || mod.name !== name) {
+    return yield* next;
+  }
+
+  var key = mod.package && mod.package.dist && mod.package.dist.key;
+  if (!key) {
+    key = getCDNKey(mod.name, filename);
+  }
+
+  if (revertTo && revertTo.package) {
+    debug('removing key: %s from nfs, revert to %s@%s', key, revertTo.name, revertTo.package.version);
+  } else {
+    debug('removing key: %s from nfs, no revert mod', key);
+  }
+  try {
+    yield nfs.remove(key);
+  } catch (err) {
+    logger.error(err);
+  }
+  // remove version from table
+  yield* packageService.removeModulesByNameAndVersions(name, [version]);
+  debug('removed %s@%s', name, version);
+  this.body = { ok: true };
+};

controllers/registry/package/save.js

@@ -0,0 +1,237 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/save.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:save');
+var crypto = require('crypto');
+var deprecateVersions = require('./deprecate');
+var packageService = require('../../../services/package');
+var common = require('../../../lib/common');
+var nfs = require('../../../common/nfs');
+var config = require('../../../config');
+
+// old flows:
+// 1. add()
+// 2. upload()
+// 3. updateLatest()
+//
+// new flows: only one request
+// PUT /:name
+// https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
+module.exports = function* save(next) {
+  // 'dist-tags': { latest: '0.0.2' },
+  //  _attachments:
+  // { 'nae-sandbox-0.0.2.tgz':
+  //    { content_type: 'application/octet-stream',
+  //      data: 'H4sIAAAAA
+  //      length: 9883
+  var pkg = this.request.body;
+  var username = this.user.name;
+  var name = this.params.name || this.params[0];
+  var filename = Object.keys(pkg._attachments || {})[0];
+  var version = Object.keys(pkg.versions || {})[0];
+  if (!version) {
+    this.status = 400;
+    this.body = {
+      error: 'version_error',
+      reason: 'package.versions is empty'
+    };
+    return;
+  }
+
+  // check maintainers
+  var result = yield* packageService.authMaintainer(name, username);
+  if (!result.isMaintainer) {
+    this.status = 403;
+    this.body = {
+      error: 'forbidden user',
+      reason: username + ' not authorized to modify ' + name +
+        ', please contact maintainers: ' + result.maintainers.join(', ')
+    };
+    return;
+  }
+
+  if (!filename) {
+    var hasDeprecated = false;
+    for (var v in pkg.versions) {
+      var row = pkg.versions[v];
+      if (typeof row.deprecated === 'string') {
+        hasDeprecated = true;
+        break;
+      }
+    }
+    if (hasDeprecated) {
+      return yield* deprecateVersions.call(this, next);
+    }
+
+    this.status = 400;
+    this.body = {
+      error: 'attachment_error',
+      reason: 'package._attachments is empty'
+    };
+    return;
+  }
+
+  var attachment = pkg._attachments[filename];
+  var versionPackage = pkg.versions[version];
+  var maintainers = versionPackage.maintainers;
+
+  // should never happened in normal request
+  if (!maintainers) {
+    this.status = 400;
+    this.body = {
+      error: 'maintainers error',
+      reason: 'request body need maintainers'
+    };
+    return;
+  }
+
+  // notice that admins can not publish to all modules
+  // (but admins can add self to maintainers first)
+
+  // make sure user in auth is in maintainers
+  // should never happened in normal request
+  var m = maintainers.filter(function (maintainer) {
+    return maintainer.name === username;
+  });
+  if (m.length === 0) {
+    this.status = 403;
+    this.body = {
+      error: 'maintainers error',
+      reason: username + ' does not in maintainer list'
+    };
+    return;
+  }
+
+  // TODO: add this info into some table
+  versionPackage._publish_on_cnpm = true;
+  var distTags = pkg['dist-tags'] || {};
+  var tags = []; // tag, version
+  for (var t in distTags) {
+    tags.push([t, distTags[t]]);
+  }
+
+  if (tags.length === 0) {
+    this.status = 400;
+    this.body = {
+      error: 'invalid',
+      reason: 'dist-tags should not be empty'
+    };
+    return;
+  }
+
+  debug('%s publish new %s:%s, attachment size: %s, maintainers: %j, distTags: %j',
+    username, name, version, attachment.length, versionPackage.maintainers, distTags);
+
+  var exists = yield* packageService.getModule(name, version);
+  var shasum;
+  if (exists) {
+    this.status = 403;
+    this.body = {
+      error: 'forbidden',
+      reason: 'cannot modify pre-existing version: ' + version
+    };
+    return;
+  }
+
+  // upload attachment
+  var tarballBuffer;
+  tarballBuffer = new Buffer(attachment.data, 'base64');
+
+  if (tarballBuffer.length !== attachment.length) {
+    this.status = 403;
+    this.body = {
+      error: 'size_wrong',
+      reason: 'Attachment size ' + attachment.length
+        + ' not match download size ' + tarballBuffer.length,
+    };
+    return;
+  }
+
+  if (!distTags.latest) {
+    // need to check if latest tag exists or not
+    var latest = yield* packageService.getModuleByTag(name, 'latest');
+    if (!latest) {
+      // auto add latest
+      tags.push(['latest', tags[0][1]]);
+      debug('auto add latest tag: %j', tags);
+    }
+  }
+
+  shasum = crypto.createHash('sha1');
+  shasum.update(tarballBuffer);
+  shasum = shasum.digest('hex');
+
+  var options = {
+    key: common.getCDNKey(name, filename),
+    shasum: shasum
+  };
+  var uploadResult = yield nfs.uploadBuffer(tarballBuffer, options);
+  debug('upload %j', uploadResult);
+
+  var dist = {
+    shasum: shasum,
+    size: attachment.length
+  };
+
+  // if nfs upload return a key, record it
+  if (uploadResult.url) {
+    dist.tarball = uploadResult.url;
+  } else if (uploadResult.key) {
+    dist.key = uploadResult.key;
+    dist.tarball = uploadResult.key;
+  }
+
+  var mod = {
+    name: name,
+    version: version,
+    author: username,
+    package: versionPackage
+  };
+
+  mod.package.dist = dist;
+  yield* addDepsRelations(mod.package);
+
+  var addResult = yield* packageService.saveModule(mod);
+  debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
+    addResult.id, dist.tarball, dist.size, shasum, dist, version);
+
+  if (tags.length) {
+    yield tags.map(function (tag) {
+      // tag: [tagName, version]
+      return packageService.addModuleTag(name, tag[0], tag[1]);
+    });
+  }
+
+  // ensure maintainers exists
+  var maintainerNames = maintainers.map(function (item) {
+    return item.name;
+  });
+  yield* packageService.addPrivateModuleMaintainers(name, maintainerNames);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    rev: String(addResult.id)
+  };
+};
+
+function* addDepsRelations(pkg) {
+  var dependencies = Object.keys(pkg.dependencies || {});
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
+  }
+  yield packageService.addDependencies(pkg.name, dependencies);
+}

controllers/registry/package/show.js

@@ -0,0 +1,84 @@
+/**!
+ * cnpmjs.js - controllers/registry/package/show.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:show');
+var semver = require('semver');
+var packageService = require('../../../services/package');
+var npmService = require('../../../services/npm');
+var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var SyncModuleWorker = require('../../sync_module_worker');
+var config = require('../../../config');
+
+/**
+ * [deprecate] api
+ *
+ * get the special version or tag of a module
+ *
+ * GET /:name/:version
+ * GET /:name/:tag
+ */
+module.exports = function* show() {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.version || this.params[1];
+  var version = semver.valid(tag);
+  var mod;
+  if (version) {
+    mod = yield* packageService.getModule(name, version);
+  } else {
+    mod = yield* packageService.getModuleByTag(name, tag);
+  }
+  if (mod) {
+    setDownloadURL(mod.package, this);
+    mod.package._cnpm_publish_time = mod.publish_time;
+    var maintainers = yield* packageService.listMaintainers(name);
+    if (maintainers.length > 0) {
+      mod.package.maintainers = maintainers;
+    }
+    this.body = mod.package;
+    return;
+  }
+
+  // if not fond, sync from source registry
+  if (!this.allowSync) {
+    this.status = 404;
+    this.body = {
+      error: 'not exist',
+      reason: 'version not found: ' + version
+    };
+    return;
+  }
+
+  // start sync
+  var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
+  debug('start sync %s, get log id %s', name, logId);
+
+  // rty to get package from official registry
+  var r = yield npmService.request('/' + name + '/' + tag, {
+    registry: config.officialNpmRegistry
+  });
+
+  if (r.status !== 200) {
+    debug('requet from officialNpmRegistry response %s', r.status);
+    this.status = 404;
+    this.body = {
+      error: 'not exist',
+      reason: 'tag or version not found: ' + tag
+    };
+    return;
+  }
+
+  this.body = r.data;
+};

controllers/registry/package/tag.js

@@ -0,0 +1,69 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/tag.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:tag');
+var semver = require('semver');
+var util = require('util');
+var packageService = require('../../../services/package');
+
+// PUT /:name/:tag
+// https://github.com/npm/npm-registry-client/blob/master/lib/tag.js#L4
+// this.request("PUT", uri+"/"+tagName, { body : JSON.stringify(version) }, cb)
+module.exports = function* tag() {
+  var version = this.request.body;
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  debug('tag %j to %s/%s', version, name, tag);
+
+  if (!version) {
+    this.status = 400;
+    this.body = {
+      error: 'version_missed',
+      reason: 'version not found'
+    };
+    return;
+  }
+
+  if (!semver.valid(version)) {
+    this.status = 403;
+    var reason = util.format('setting tag %s to invalid version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error: 'forbidden',
+      reason: reason
+    };
+    return;
+  }
+
+  var mod = yield* packageService.getModule(name, version);
+  if (!mod) {
+    this.status = 403;
+    var reason = util.format('setting tag %s to unknown version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error: 'forbidden',
+      reason: reason
+    };
+    return;
+  }
+
+  var row = yield* packageService.addModuleTag(name, tag, version);
+  this.status = 201;
+  this.body = {
+    ok: true,
+    modified: row.gmt_modified
+  };
+};

controllers/registry/package/update.js

@@ -0,0 +1,180 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/update.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:update');
+var packageService = require('../../../services/package');
+var userService = require('../../../services/user');
+var config = require('../../../config');
+
+// PUT /:name/-rev/:rev
+//
+// * remove with versions, then will `DELETE /:name/download/:filename/-rev/:rev`
+// * ...
+module.exports = function* update(next) {
+  var name = this.params.name || this.params[0];
+  debug('update module %s, %s, %j', this.url, name, this.request.body);
+
+  var body = this.request.body;
+  if (body.versions) {
+    yield* updateVersions.call(this, next);
+  } else if (body.maintainers) {
+    yield* updateMaintainers.call(this, next);
+  } else {
+    yield* next;
+  }
+};
+
+// update with versions
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L63
+function* updateVersions(next) {
+  var name = this.params.name || this.params[0];
+  // left versions
+  var versions = this.request.body.versions;
+
+  // step1: list all the versions
+  var mods = yield* packageService.listModulesByName(name);
+  debug('removeWithVersions module %s, left versions %j, %s mods',
+    name, Object.keys(versions), mods && mods.length);
+  if (!mods || !mods.length) {
+    return yield* next;
+  }
+
+  // step3: calculate which versions need to remove and
+  // which versions need to remain
+  var removeVersions = [];
+  var removeVersionMaps = {};
+  var remainVersions = [];
+
+  for (var i = 0; i < mods.length; i++) {
+    var v = mods[i].version;
+    if (!versions[v]) {
+      removeVersions.push(v);
+      removeVersionMaps[v] = true;
+    } else {
+      remainVersions.push(v);
+    }
+  }
+
+  if (!removeVersions.length) {
+    debug('no versions need to remove');
+    this.status = 201;
+    this.body = { ok: true };
+    return;
+  }
+  debug('remove versions: %j, remain versions: %j', removeVersions, remainVersions);
+
+  // step 4: remove all the versions which need to remove
+  // let removeTar do remove versions from module table
+  var tags = yield* packageService.listModuleTags(name);
+
+  var removeTags = [];
+  var latestRemoved = false;
+  tags.forEach(function (tag) {
+    // this tag need be removed
+    if (removeVersionMaps[tag.version]) {
+      removeTags.push(tag.id);
+      if (tag.tag === 'latest') {
+        latestRemoved = true;
+      }
+    }
+  });
+
+  debug('remove tags: %j', removeTags);
+  if (removeTags.length) {
+    // step 5: remove all the tags
+    yield* packageService.removeModuleTagsByIds(removeTags);
+    if (latestRemoved && remainVersions[0]) {
+      debug('latest tags removed, generate a new latest tag with new version: %s',
+        remainVersions[0]);
+      // step 6: insert new latest tag
+      yield* packageService.addModuleTag(name, 'latest', remainVersions[0]);
+    }
+  }
+
+  // step 7: update last modified, make sure etag change
+  yield* packageService.updateModuleLastModified(name);
+
+  this.status = 201;
+  this.body = { ok: true };
+}
+
+function* updateMaintainers() {
+  var name = this.params.name || this.params[0];
+  var body = this.request.body;
+  debug('updateMaintainers module %s, %j', name, body);
+
+  var usernames = body.maintainers.map(function (user) {
+    return user.name;
+  });
+
+  if (usernames.length === 0) {
+    this.status = 403;
+    this.body = {
+      error: 'invalid operation',
+      reason: 'Can not remove all maintainers'
+    };
+    return;
+  }
+
+  if (config.customUserService) {
+    // ensure new authors are vaild
+    var maintainers = yield* packageService.listMaintainerNamesOnly(name);
+    var map = {};
+    var newNames = [];
+    for (var i = 0; i < maintainers.length; i++) {
+      map[maintainers[i]] = 1;
+    }
+    for (var i = 0; i < usernames.length; i++) {
+      var username = usernames[i];
+      if (map[username] !== 1) {
+        newNames.push(username);
+      }
+    }
+    if (newNames.length > 0) {
+      var users = yield* userService.list(newNames);
+      var map = {};
+      for (var i = 0; i < users.length; i++) {
+        var user = users[i];
+        map[user.login] = 1;
+      }
+      var invailds = [];
+      for (var i = 0; i < newNames.length; i++) {
+        var username = newNames[i];
+        if (map[username] !== 1) {
+          invailds.push(username);
+        }
+      }
+      if (invailds.length > 0) {
+        this.status = 403;
+        this.body = {
+          error: 'invalid user name',
+          reason: 'User: `' + invailds.join(', ') + '` not exists'
+        };
+        return;
+      }
+    }
+  }
+
+  var r = yield* packageService.updatePrivateModuleMaintainers(name, usernames);
+  debug('result: %j', r);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: name,
+    rev: this.params.rev || this.params[1],
+  };
+}

controllers/registry/user.js

@@ -1,291 +0,0 @@
-/**!
- * cnpmjs.org - controllers/registry/user.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:controllers:registry:user');
-var utility = require('utility');
-var crypto = require('crypto');
-var UserService = require('../../services/user');
-var User = require('../../proxy/user');
-var config = require('../../config');
-var common = require('../../lib/common');
-
-exports.show = function* (next) {
-  var name = this.params.name;
-  var isAdmin = common.isAdmin(name);
-  var scopes = config.scopes || [];
-  if (config.customUserService) {
-    var customUser = yield* UserService.get(name);
-    if (customUser) {
-      isAdmin = !!customUser.site_admin;
-      scopes = customUser.scopes;
-
-      var data = {
-        user: customUser
-      };
-      yield* User.saveCustomUser(data);
-    }
-  }
-
-  var user = yield* User.get(name);
-  if (!user) {
-    return yield* next;
-  }
-
-  var data = user.json;
-  if (!data) {
-    data = {
-      _id: 'org.couchdb.user:' + user.name,
-      _rev: user.rev,
-      name: user.name,
-      email: user.email,
-      type: 'user',
-      roles: [],
-      date: user.gmt_modified,
-    };
-  }
-
-  if (data.login) {
-    // custom user format
-    // convert to npm user format
-    data = {
-      _id: 'org.couchdb.user:' + user.name,
-      _rev: user.rev,
-      name: user.name,
-      email: user.email,
-      type: 'user',
-      roles: [],
-      date: user.gmt_modified,
-      avatar: data.avatar_url,
-      fullname: data.name || data.login,
-      homepage: data.html_url,
-    };
-  }
-
-  data._cnpm_meta = {
-    id: user.id,
-    npm_user: user.npm_user === 1,
-    custom_user: user.npm_user === 2,
-    gmt_create: user.gmt_create,
-    gmt_modified: user.gmt_modified,
-    admin: isAdmin,
-    scopes: scopes,
-  };
-
-  this.body = data;
-};
-
-function ensurePasswordSalt(user, body) {
-  if (!user.password_sha && body.password) {
-    // create password_sha on server
-    user.salt = crypto.randomBytes(30).toString('hex');
-    user.password_sha = utility.sha1(body.password + user.salt);
-  }
-}
-
-// npm 1.4.4
-// add new user first
-// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
-// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
-//   'accept-encoding': 'gzip',
-//   'user-agent': 'node/v0.11.12 darwin x64',
-//   host: '127.0.0.1:7001',
-//   'content-type': 'application/json',
-//   'content-length': '150',
-//   connection: 'close' } { name: 'mk2',
-//   password: '123456',
-//   email: 'fengmk2@gmail.com',
-//   _id: 'org.couchdb.user:mk2',
-//   type: 'user',
-//   roles: [],
-//   date: '2014-03-15T02:33:19.465Z' }
-
-// old npm flow
-// json:
-// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
-//   'user-agent': 'node/v0.8.26 darwin x64',
-//   host: '127.0.0.1:7001',
-//   'content-type': 'application/json',
-//   'content-length': '258',
-//   connection: 'keep-alive' }
-// { name: 'mk2',
-//   salt: '12351936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
-//   password_sha: '123408912a6db1d96b132a90856d99db029cef3d',
-//   email: 'fengmk2@gmail.com',
-//   _id: 'org.couchdb.user:mk2',
-//   type: 'user',
-//   roles: [],
-//   date: '2014-03-15T02:39:25.696Z' }
-exports.add = function* () {
-  var name = this.params.name;
-  var body = this.request.body || {};
-  var user = {
-    name: body.name,
-    // salt: body.salt,
-    // password_sha: body.password_sha,
-    email: body.email,
-    ip: this.ip || '0.0.0.0',
-    // roles: body.roles || [],
-  };
-
-  ensurePasswordSalt(user, body);
-
-  if (!body.password || !user.name || !user.salt || !user.password_sha || !user.email) {
-    this.status = 422;
-    this.body = {
-      error: 'paramError',
-      reason: 'params missing, name, email or password missing.'
-    };
-    return;
-  }
-
-  debug('add user: %j', body);
-
-  var loginedUser;
-  try {
-    loginedUser = yield UserService.auth(body.name, body.password);
-  } catch (err) {
-    this.status = err.status || 500;
-    this.body = {
-      error: err.name,
-      reason: err.message
-    };
-    return;
-  }
-  if (loginedUser) {
-    var rev = Date.now() + '-' + loginedUser.login;
-    if (config.customUserService) {
-      // make sure sync user meta to cnpm database
-      var data = user;
-      data.rev = rev;
-      data.user = loginedUser;
-      yield* User.saveCustomUser(data);
-    }
-    this.status = 201;
-    this.body = {
-      ok: true,
-      id: 'org.couchdb.user:' + loginedUser.login,
-      rev: rev,
-    };
-    return;
-  }
-
-  if (config.customUserService) {
-    // user login fail, not allow to add new user
-    this.status = 401;
-    this.body = {
-      error: 'unauthorized',
-      reason: 'Login fail, please check your login name and password'
-    };
-    return;
-  }
-
-  var existUser = yield User.get(name);
-  if (existUser) {
-    this.status = 409;
-    this.body = {
-      error: 'conflict',
-      reason: 'User ' + name + ' already exists.'
-    };
-    return;
-  }
-
-  var result = yield User.add(user);
-  this.etag = '"' + result.rev + '"';
-  this.status = 201;
-  this.body = {
-    ok: true,
-    id: 'org.couchdb.user:' + name,
-    rev: result.rev
-  };
-};
-
-// logined before update, no need to auth user again
-// { name: 'admin',
-// password: '123123',
-// email: 'fengmk2@gmail.com',
-// _id: 'org.couchdb.user:admin',
-// type: 'user',
-// roles: [],
-// date: '2014-08-05T16:08:22.645Z',
-// _rev: '1-1a18c3d73ba42e863523a399ff3304d8',
-// _cnpm_meta:
-//  { id: 14,
-//    npm_user: false,
-//    custom_user: false,
-//    gmt_create: '2014-08-05T15:46:58.000Z',
-//    gmt_modified: '2014-08-05T15:46:58.000Z',
-//    admin: true,
-//    scopes: [ '@cnpm', '@cnpmtest' ] } }
-exports.update = function *(next) {
-  var name = this.params.name;
-  var rev = this.params.rev;
-  if (!name || !rev) {
-    return yield* next;
-  }
-  debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
-
-  if (name !== this.user.name) {
-    // must auth user first
-    this.status = 401;
-    this.body = {
-      error: 'unauthorized',
-      reason: 'Name is incorrect.'
-    };
-    return;
-  }
-
-  var body = this.request.body || {};
-  var user = {
-    name: body.name,
-    // salt: body.salt,
-    // password_sha: body.password_sha,
-    email: body.email,
-    ip: this.ip || '0.0.0.0',
-    rev: body.rev || body._rev,
-    // roles: body.roles || [],
-  };
-
-  debug('update user %j', body);
-
-  ensurePasswordSalt(user, body);
-
-  if (!body.password || !user.name || !user.salt || !user.password_sha || !user.email) {
-    this.status = 422;
-    this.body = {
-      error: 'paramError',
-      reason: 'params missing, name, email or password missing.'
-    };
-    return;
-  }
-
-  var result = yield User.update(user);
-  if (!result) {
-    this.status = 409;
-    this.body = {
-      error: 'conflict',
-      reason: 'Document update conflict.'
-    };
-    return;
-  }
-
-  this.status = 201;
-  this.body = {
-    ok: true,
-    id: 'org.couchdb.user:' + user.name,
-    rev: result.rev
-  };
-};

controllers/registry/user/add.js

@@ -0,0 +1,128 @@
+/**!
+ * cnpmjs.org - controllers/registry/user/add.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var ensurePasswordSalt = require('./common').ensurePasswordSalt;
+var userService = require('../../../services/user');
+var config = require('../../../config');
+
+// npm 1.4.4
+// add new user first
+// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'accept-encoding': 'gzip',
+//   'user-agent': 'node/v0.11.12 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '150',
+//   connection: 'close' } { name: 'mk2',
+//   password: '123456',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:33:19.465Z' }
+
+// old npm flow
+// json:
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'user-agent': 'node/v0.8.26 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '258',
+//   connection: 'keep-alive' }
+// { name: 'mk2',
+//   salt: '12351936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
+//   password_sha: '123408912a6db1d96b132a90856d99db029cef3d',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:39:25.696Z' }
+module.exports = function* addUser() {
+  var name = this.params.name;
+  var body = this.request.body || {};
+  var user = {
+    name: body.name,
+    // salt: body.salt,
+    // password_sha: body.password_sha,
+    email: body.email,
+    ip: this.ip || '0.0.0.0',
+    // roles: body.roles || [],
+  };
+
+  ensurePasswordSalt(user, body);
+
+  if (!body.password || !user.name || !user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    this.body = {
+      error: 'paramError',
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
+  }
+
+  var loginedUser;
+  try {
+    loginedUser = yield* userService.authAndSave(body.name, body.password);
+  } catch (err) {
+    this.status = err.status || 500;
+    this.body = {
+      error: err.name,
+      reason: err.message
+    };
+    return;
+  }
+  if (loginedUser) {
+    this.status = 201;
+    this.body = {
+      ok: true,
+      id: 'org.couchdb.user:' + loginedUser.login,
+      rev: Date.now() + '-' + loginedUser.login
+    };
+    return;
+  }
+
+  if (config.customUserService) {
+    // user login fail, not allow to add new user
+    this.status = 401;
+    this.body = {
+      error: 'unauthorized',
+      reason: 'Login fail, please check your login name and password'
+    };
+    return;
+  }
+
+  var existUser = yield* userService.get(name);
+  if (existUser) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'User ' + name + ' already exists.'
+    };
+    return;
+  }
+
+  // add new user
+  var result = yield* userService.add(user);
+  this.etag = '"' + result.rev + '"';
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
+};

controllers/registry/user/common.js

@@ -0,0 +1,26 @@
+/**!
+ * cnpmjs.org - controllers/registry/user/common.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var utility = require('utility');
+
+exports.ensurePasswordSalt = function (user, body) {
+  if (!user.password_sha && body.password) {
+    // create password_sha on server
+    user.salt = crypto.randomBytes(30).toString('hex');
+    user.password_sha = utility.sha1(body.password + user.salt);
+  }
+};

controllers/registry/user/show.js

@@ -0,0 +1,69 @@
+/**!
+ * cnpmjs.org - controllers/registry/user/show.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var userService = require('../../../services/user');
+
+// GET /-/user/org.couchdb.user::name
+module.exports = function* show(next) {
+  var name = this.params.name;
+  var user = yield* userService.getAndSave(name);
+  if (!user) {
+    return yield* next;
+  }
+
+  var data = user.json;
+  if (!data) {
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
+      type: 'user',
+      roles: [],
+      date: user.gmt_modified,
+    };
+  }
+
+  if (data.login) {
+    // custom user format
+    // convert to npm user format
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
+      type: 'user',
+      roles: [],
+      date: user.gmt_modified,
+      avatar: data.avatar_url,
+      fullname: data.name || data.login,
+      homepage: data.html_url,
+      scopes: data.scopes,
+      site_admin: data.site_admin
+    };
+  }
+
+  data._cnpm_meta = {
+    id: user.id,
+    npm_user: user.isNpmUser,
+    custom_user: !!data.login,
+    gmt_create: user.gmt_create,
+    gmt_modified: user.gmt_modified,
+  };
+
+  this.body = data;
+};

controllers/registry/user/update.js

@@ -0,0 +1,97 @@
+/**!
+ * cnpmjs.org - controllers/registry/user/update.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:user:update');
+var ensurePasswordSalt = require('./common').ensurePasswordSalt;
+var userService = require('../../../services/user');
+
+// logined before update, no need to auth user again
+// { name: 'admin',
+// password: '123123',
+// email: 'fengmk2@gmail.com',
+// _id: 'org.couchdb.user:admin',
+// type: 'user',
+// roles: [],
+// date: '2014-08-05T16:08:22.645Z',
+// _rev: '1-1a18c3d73ba42e863523a399ff3304d8',
+// _cnpm_meta:
+//  { id: 14,
+//    npm_user: false,
+//    custom_user: false,
+//    gmt_create: '2014-08-05T15:46:58.000Z',
+//    gmt_modified: '2014-08-05T15:46:58.000Z',
+//    admin: true,
+//    scopes: [ '@cnpm', '@cnpmtest' ] } }
+module.exports = function* updateUser(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
+  if (!name || !rev) {
+    return yield* next;
+  }
+  debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
+
+  if (name !== this.user.name) {
+    // must auth user first
+    this.status = 401;
+    this.body = {
+      error: 'unauthorized',
+      reason: 'Name is incorrect.'
+    };
+    return;
+  }
+
+  var body = this.request.body || {};
+  var user = {
+    name: body.name,
+    // salt: body.salt,
+    // password_sha: body.password_sha,
+    email: body.email,
+    ip: this.ip || '0.0.0.0',
+    rev: body.rev || body._rev,
+    // roles: body.roles || [],
+  };
+
+  debug('update user %j', body);
+
+  ensurePasswordSalt(user, body);
+
+  if (!body.password || !user.name || !user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    this.body = {
+      error: 'paramError',
+      reason: 'params missing, name, email or password missing.'
+    };
+    return;
+  }
+
+  var result = yield* userService.update(user);
+  if (!result) {
+    this.status = 409;
+    this.body = {
+      error: 'conflict',
+      reason: 'Document update conflict.'
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
+};

controllers/registry/user_package.js

@@ -14,8 +14,9 @@
  * Module dependencies.
  */
 
-var ModuleMaintainer = require('../../proxy/module_maintainer');
+var packageService = require('../../services/package');
 
+// GET /-/by-user/:user
 exports.list = function* () {
   var users = this.params.user.split('|');
   if (users.length > 20) {
@@ -27,15 +28,23 @@ exports.list = function* () {
     return;
   }
 
+  var firstUser = users[0];
+  if (!firstUser) {
+    // params.user = '|'
+    this.body = {};
+    return;
+  }
 
-  var rows = yield* ModuleMaintainer.listByUsers(users);
-  var data = {};
-  for (var i = 0; i < rows.length; i++) {
-    var row = rows[i];
-    if (data[row.user]) {
-      data[row.user].push(row.name);
-    } else {
-      data[row.user] = [row.name];
+  var tasks = {};
+  for (var i = 0; i < users.length; i++) {
+    var username = users[i];
+    tasks[username] = packageService.listPublicModuleNamesByUser(username);
+  }
+
+  var data = yield tasks;
+  for (var k in data) {
+    if (data[k].length === 0) {
+      data[k] = undefined;
     }
   }
   this.body = data;

controllers/sync.js

@@ -15,12 +15,13 @@
  */
 
 var debug = require('debug')('cnpmjs.org:controllers:sync');
-var Log = require('../proxy/module_log');
-var SyncModuleWorker = require('../proxy/sync_module_worker');
+var Log = require('../services/module_log');
+var SyncModuleWorker = require('./sync_module_worker');
+var config = require('../config');
 
 exports.sync = function* () {
   var username = this.user.name || 'anonymous';
-  var name = this.params.name || this.params[0];
+  var name = this.params.name;
   var publish = this.query.publish === 'true';
   var noDep = this.query.nodeps === 'true';
   debug('sync %s with query: %j', name, this.query);
@@ -36,9 +37,10 @@ exports.sync = function* () {
   var options = {
     publish: publish,
     noDep: noDep,
+    syncUpstreamFirst: config.sourceNpmRegistryIsCNpm,
   };
 
-  var logId = yield SyncModuleWorker.sync(name, username, options);
+  var logId = yield* SyncModuleWorker.sync(name, username, options);
   debug('sync %s got log id %j', name, logId);
 
   this.status = 201;
@@ -49,10 +51,9 @@ exports.sync = function* () {
 };
 
 exports.getSyncLog = function* (next) {
-  // params: [$name, $id] on scope package
-  var logId = this.params.id || this.params[1];
+  var logId = this.params.id;
   var offset = Number(this.query.offset) || 0;
-  var row = yield Log.get(logId);
+  var row = yield* Log.get(logId);
   if (!row) {
     return yield* next;
   }

controllers/sync_module_worker.js

@@ -1,5 +1,5 @@
 /**!
- * cnpmjs.org - proxy/sync_module_worker.js
+ * cnpmjs.org - controllers/sync_module_worker.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -15,54 +15,54 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var co = require('co');
 var gather = require('co-gather');
 var defer = require('co-defer');
 var thunkify = require('thunkify-wrap');
-var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
 var util = require('util');
 var fs = require('fs');
 var path = require('path');
 var crypto = require('crypto');
 var sleep = require('co-sleep');
-var urllib = require('../common/urllib');
 var utility = require('utility');
-var ms = require('ms');
 var urlparse = require('url').parse;
-var nfs = require('../common/nfs');
-var npm = require('./npm');
-var common = require('../lib/common');
-var Module = require('./module');
-var ModuleDeps = require('./module_deps');
-var Log = require('./module_log');
+var urllib = require('../common/urllib');
 var config = require('../config');
-var ModuleStar = require('./module_star');
-var User = require('./user');
-var ModuleUnpublished = require('./module_unpublished');
+var nfs = require('../common/nfs');
+var logger = require('../common/logger');
+var common = require('../lib/common');
+var npmSerivce = require('../services/npm');
+var packageService = require('../services/package');
+var logService = require('../services/module_log');
+var User = require('../models').User;
 
 var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
   this._logId = options.logId;
+  this._log = '';
+  this._loging = false;
   if (!Array.isArray(options.name)) {
     options.name = [options.name];
   }
 
-  this.names = options.name || [];
+  this.names = options.name;
   this.startName = this.names[0];
 
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
-  this._publish = options.publish; // _publish_on_cnpm
+  this._publish = options.publish === true; // _publish_on_cnpm
+  this.syncUpstreamFirst = options.syncUpstreamFirst;
 
   this.syncingNames = {};
   this.nameMap = {};
   this.names.forEach(function (name) {
     this.nameMap[name] = true;
   }.bind(this));
-  this.noDep = options.noDep; //do not sync dependences
+  this.noDep = options.noDep === true; // do not sync dependences
   this.syncDevDependencies = config.syncDevDependencies;
 
   this.successes = [];
@@ -88,10 +88,39 @@ SyncModuleWorker.prototype.finish = function () {
   this.removeAllListeners();
 };
 
-SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
+// log(format, arg1, arg2, ...)
+SyncModuleWorker.prototype.log = function () {
   var str = '[' + utility.YYYYMMDDHHmmss() + '] ' + util.format.apply(util, arguments);
   debug(str);
-  this._logId && Log.append(this._logId, str, utility.noop);
+  var logId = this._logId;
+  if (logId) {
+    if (this._log) {
+      this._log += '\n';
+    }
+    this._log += str;
+    this._saveLog();
+  }
+};
+
+SyncModuleWorker.prototype._saveLog = function () {
+  var that = this;
+  if (that._loging) {
+    return;
+  }
+  that._loging = true;
+  var logstr = that._log;
+  that._log = '';
+  co(function* () {
+    yield* logService.append(that._logId, logstr);
+  })(function (err) {
+    if (err) {
+      logger.error(err);
+    }
+    that._loging = false;
+    if (that._log) {
+      that._saveLog();
+    }
+  });
 };
 
 SyncModuleWorker.prototype.start = function () {
@@ -100,8 +129,12 @@ SyncModuleWorker.prototype.start = function () {
   var that = this;
   co(function *() {
     // sync upstream
-    if (config.sourceNpmRegistryIsCNpm) {
-      yield* that.syncUpstream(that.startName);
+    if (that.syncUpstreamFirst) {
+      try {
+        yield* that.syncUpstream(that.startName);
+      } catch (err) {
+        logger.error(err);
+      }
     }
 
     var arr = [];
@@ -143,8 +176,8 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
   delete this.syncingNames[name];
   var that = this;
   // relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
-  defer.setImmediate(function *() {
-    yield *that.next(concurrencyId);
+  defer.setImmediate(function* () {
+    yield* that.next(concurrencyId);
   });
 };
 
@@ -215,7 +248,7 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
   this.log('----------------- Synced upstream %s -------------------', logURL);
 };
 
-SyncModuleWorker.prototype.next = function *(concurrencyId) {
+SyncModuleWorker.prototype.next = function* (concurrencyId) {
   var name = this.names.shift();
   if (!name) {
     return setImmediate(this.finish.bind(this));
@@ -228,9 +261,17 @@ SyncModuleWorker.prototype.next = function *(concurrencyId) {
 
   this.log('----------------- Syncing %s -------------------', name);
 
+  // ignore scoped package
+  if (name[0] === '@') {
+    this.log('[c#%d] [%s] ignore sync scoped package',
+      concurrencyId, name);
+    yield* this._doneOne(concurrencyId, name, true);
+    return;
+  }
+
   // get from npm
   try {
-    var result = yield npm.request('/' + name);
+    var result = yield* npmSerivce.request('/' + name);
     pkg = result.data;
     status = result.status;
   } catch (err) {
@@ -288,8 +329,8 @@ SyncModuleWorker.prototype.next = function *(concurrencyId) {
   yield* this._doneOne(concurrencyId, name, true);
 };
 
-function *_listStarUsers(modName) {
-  var users = yield ModuleStar.listUsers(modName);
+function* _listStarUsers(modName) {
+  var users = yield* packageService.listStarUserNames(modName);
   var userMap = {};
   users.forEach(function (user) {
     userMap[user] = true;
@@ -297,20 +338,24 @@ function *_listStarUsers(modName) {
   return userMap;
 }
 
-function *_addStar(modName, username) {
-  yield ModuleStar.add(modName, username);
-}
-
-function *_saveNpmUser(username) {
-  var user = yield *npm.getUser(username);
+function* _saveNpmUser(username) {
+  var user = yield* npmSerivce.getUser(username);
   if (!user) {
     return;
   }
-  yield User.saveNpmUser(user);
+  yield* User.saveNpmUser(user);
+}
+
+function* _saveMaintainer(modName, username, action) {
+  if (action === 'add') {
+    yield* packageService.addPublicModuleMaintainer(modName, username);
+  } else if (action === 'remove') {
+    yield* packageService.removePublicModuleMaintainer(modName, username);
+  }
 }
 
 SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
-  var mods = yield Module.listByName(name);
+  var mods = yield* packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
   if (this._isLocalModule(mods)) {
@@ -319,13 +364,13 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
     return [];
   }
 
-  var r = yield* ModuleUnpublished.add(name, unpublishedInfo);
+  var r = yield* packageService.saveUnpublishedModule(name, unpublishedInfo);
   this.log('    [%s] save unpublished info: %j to row#%s',
-    name, unpublishedInfo, r.insertId);
+    name, unpublishedInfo, r.id);
   if (mods.length === 0) {
     return;
   }
-  yield [Module.removeByName(name), Module.removeTags(name)];
+  yield [packageService.removeModulesByName(name), packageService.removeModuleTags(name)];
   var keys = [];
   for (var i = 0; i < mods.length; i++) {
     var row = mods[i];
@@ -337,18 +382,16 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
     key && keys.push(key);
   }
 
-  if (keys.length === 0) {
-    return;
-  }
-
-  try {
-    yield keys.map(function (key) {
-      return nfs.remove(key);
-    });
-  } catch (err) {
-    // ignore error here
-    this.log('    [%s] delete nfs files: %j error: %s: %s',
-      name, keys, err.name, err.message);
+  if (keys.length > 0) {
+    try {
+      yield keys.map(function (key) {
+        return nfs.remove(key);
+      });
+    } catch (err) {
+      // ignore error here
+      this.log('    [%s] delete nfs files: %j error: %s: %s',
+        name, keys, err.name, err.message);
+    }
   }
   this.log('    [%s] delete nfs files: %j success', name, keys);
 };
@@ -364,18 +407,18 @@ SyncModuleWorker.prototype._isLocalModule = function (mods) {
 };
 
 SyncModuleWorker.prototype._sync = function* (name, pkg) {
-  var username = this.username;
   var that = this;
-
   var hasModules = false;
   var result = yield [
-    Module.listByName(name),
-    Module.listTags(name),
-    _listStarUsers(name)
+    packageService.listModulesByName(name),
+    packageService.listModuleTags(name),
+    _listStarUsers(name),
+    packageService.listPublicModuleMaintainers(name),
   ];
   var moduleRows = result[0];
   var tagRows = result[1];
   var existsStarUsers = result[2];
+  var existsNpmMaintainers = result[3];
 
   if (that._isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
@@ -392,10 +435,6 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
       // package json parse error
       continue;
     }
-
-    if (r.version === 'next') {
-      continue;
-    }
     if (!map.latest) {
       map.latest = r;
     }
@@ -420,25 +459,58 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var missingStarUsers = [];
   var npmUsernames = {};
   var missingDeprecateds = [];
+  // [[user, 'add or remove'], ...]
+  var diffNpmMaintainers = [];
+
+  // find out new maintainers
+  var pkgMaintainers = pkg.maintainers || [];
+  if (Array.isArray(pkgMaintainers)) {
+    var existsMap = {};
+    var originalMap = {};
+    for (var i = 0; i < existsNpmMaintainers.length; i++) {
+      var user = existsNpmMaintainers[i];
+      existsMap[user] = true;
+    }
+    for (var i = 0; i < pkgMaintainers.length; i++) {
+      var item = pkgMaintainers[i];
+      originalMap[item.name] = item;
+      npmUsernames[item.name.toLowerCase()] = 1;
+    }
+
+    // find add users
+    for (var i = 0; i < pkgMaintainers.length; i++) {
+      var item = pkgMaintainers[i];
+      if (!existsMap[item.name]) {
+        diffNpmMaintainers.push([item.name, 'add']);
+      }
+    }
+
+    // find remove users
+    for (var i = 0; i < existsNpmMaintainers.length; i++) {
+      var user = existsNpmMaintainers[i];
+      if (!originalMap[user]) {
+        diffNpmMaintainers.push([user, 'remove']);
+      }
+    }
+  }
 
   // find out all user names
   for (var v in pkg.versions) {
     var p = pkg.versions[v];
-
     var maintainers = p.maintainers || [];
-    if (maintainers && !Array.isArray(maintainers)) {
+    if (!Array.isArray(maintainers)) {
       // http://r.cnpmjs.org/jasmine-node
       // TODO: "maintainers": "Martin H膫陇ger <martin.haeger@gmail.com>",
       maintainers = [maintainers];
     }
-
-    maintainers.forEach(pushName);
-  }
-  function pushName(m) {
-    if (m.name) {
-      npmUsernames[m.name.toLowerCase()] = 1;
+    for (var i = 0; i < maintainers.length; i++) {
+      var m = maintainers[i];
+      if (m.name) {
+        npmUsernames[m.name.toLowerCase()] = 1;
+      }
     }
   }
+
   // get the missing star users
   var starUsers = pkg.users || {};
   for (var k in starUsers) {
@@ -452,40 +524,6 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var times = pkg.time || {};
   pkg.versions = pkg.versions || {};
   var remoteVersionNames = Object.keys(pkg.versions);
-  if (remoteVersionNames.length === 0) {
-    that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
-    if (!hasModules) {
-      // save a next module
-      var maintainer = pkg.maintainers && pkg.maintainers[0];
-      if (maintainer && maintainer.name) {
-        maintainer = maintainer.name;
-      }
-      if (!maintainer) {
-        maintainer = '-';
-      }
-      var nextMod = {
-        name: name,
-        version: 'next',
-        author: maintainer,
-        package: {
-          name: name,
-          version: 'next',
-          description: pkg.description || '',
-          readme: pkg.readme || '',
-          maintainers: pkg.maintainers || {
-            name: maintainer
-          },
-        },
-      };
-      try {
-        var result = yield Module.add(nextMod);
-        that.log('  [%s] save next module, %j', name, result);
-      } catch (err) {
-        that.log('  [%s] save next module error %s', err.message);
-      }
-    }
-  }
-
   var remoteVersionNameMap = {};
 
   // find out missing versions
@@ -506,13 +544,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     if (!version.maintainers || !version.maintainers[0]) {
       version.maintainers = pkg.maintainers;
     }
-    var sourceAuthor = version.maintainers && version.maintainers[0] &&
-      version.maintainers[0].name || exists.author;
-
     if (exists.package &&
-        exists.package.dist.shasum === version.dist.shasum &&
-        exists.author === sourceAuthor) {
-      // * author make sure equal
+        exists.package.dist.shasum === version.dist.shasum) {
       // * shasum make sure equal
       if ((version.publish_time === exists.publish_time) ||
           (!version.publish_time && exists.publish_time)) {
@@ -594,7 +627,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
       continue;
     }
     try {
-      var result = yield that._syncOneVersion(index, syncModule);
+      yield* that._syncOneVersion(index, syncModule);
       syncedVersionNames.push(syncModule.version);
     } catch (err) {
       that.log('    [%s:%d] sync error, version: %s, %s: %s',
@@ -608,22 +641,21 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   } else {
     that.log('  [%s] %d versions: %j need to deleted',
       name, deletedVersionNames.length, deletedVersionNames);
-
     try {
-      yield Module.removeByNameAndVersions(name, deletedVersionNames);
+      yield* packageService.removeModulesByNameAndVersions(name, deletedVersionNames);
     } catch (err) {
       that.log('    [%s] delete error, %s: %s', name, err.name, err.message);
     }
   }
 
   // sync missing descriptions
-  function *syncDes() {
+  function* syncDes() {
     if (missingDescriptions.length === 0) {
       return;
     }
     that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
     var res = yield gather(missingDescriptions.map(function (item) {
-      return Module.updateDescription(item.id, item.description);
+      return packageService.updateModuleDescription(item.id, item.description);
     }));
 
     for (var i = 0; i < res.length; i++) {
@@ -642,7 +674,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   // sync missing tags
   function* syncTag() {
     if (deletedTags.length > 0) {
-      yield* Module.removeTagsByNames(name, deletedTags);
+      yield* packageService.removeModuleTagsByNames(name, deletedTags);
       that.log('  [%s] deleted %d tags: %j',
         name, deletedTags.length, deletedTags);
     }
@@ -652,9 +684,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     }
     that.log('  [%s] adding %d tags', name, missingTags.length);
     // sync tags
-
     var res = yield gather(missingTags.map(function (item) {
-      return Module.addTag(name, item[0], item[1]);
+      return packageService.addModuleTag(name, item[0], item[1]);
     }));
 
     for (var i = 0; i < res.length; i++) {
@@ -671,14 +702,14 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   }
 
   // sycn missing readme
-  function *syncReadme() {
+  function* syncReadme() {
     if (missingReadmes.length === 0) {
       return;
     }
     that.log('  [%s] saving %d readmes', name, missingReadmes.length);
 
     var res = yield gather(missingReadmes.map(function (item) {
-      return Module.updateReadme(item.id, item.readme);
+      return packageService.updateModuleReadme(item.id, item.readme);
     }));
 
     for (var i = 0; i < res.length; i++) {
@@ -699,7 +730,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     that.log('  [%s] saving %d Deprecated fields', name, missingDeprecateds.length);
 
     var res = yield gather(missingDeprecateds.map(function (item) {
-      return Module.updatePackageFields(item.id, {
+      return packageService.updateModulePackageFields(item.id, {
         deprecated: item.deprecated
       });
     }));
@@ -715,13 +746,13 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     }
   }
 
-  function *syncMissingUsers() {
+  function* syncMissingUsers() {
     var missingUsers = [];
     var names = Object.keys(npmUsernames);
     if (names.length === 0) {
       return;
     }
-    var rows = yield *User.listByNames(names);
+    var rows = yield* User.listByNames(names);
     var map = {};
     rows.forEach(function (r) {
       map[r.name] = r;
@@ -753,27 +784,53 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   }
 
   // sync missing star users
-  function *syncMissingStarUsers() {
+  function* syncMissingStarUsers() {
     if (missingStarUsers.length === 0) {
       return;
     }
 
     that.log('  [%s] saving %d star users', name, missingStarUsers.length);
     var res = yield gather(missingStarUsers.map(function (username) {
-      return _addStar(name, username);
+      return packageService.addStar(name, username);
     }));
 
     for (var i = 0; i < res.length; i++) {
       var r = res[i];
       if (r.error) {
-        that.log('    add star user error, %s', r.error.message);
+        that.log('    add star user error, %s', r.error.stack);
+      }
+    }
+  }
+
+  // sync diff npm package maintainers
+  function* syncNpmPackageMaintainers() {
+    if (diffNpmMaintainers.length === 0) {
+      return;
+    }
+
+    that.log('  [%s] syncing %d diff package maintainers: %j',
+      name, diffNpmMaintainers.length, diffNpmMaintainers);
+    var res = yield gather(diffNpmMaintainers.map(function (item) {
+      return _saveMaintainer(name, item[0], item[1]);
+    }));
+
+    for (var i = 0; i < res.length; i++) {
+      var r = res[i];
+      if (r.error) {
+        that.log('    save package maintainer error, %s', r.error.stack);
       }
     }
   }
 
   yield [
-    syncDes(), syncTag(), syncReadme(), syncDeprecateds(),
-    syncMissingStarUsers(), syncMissingUsers()];
+    syncDes(),
+    syncTag(),
+    syncReadme(),
+    syncDeprecateds(),
+    syncMissingStarUsers(),
+    syncMissingUsers(),
+    syncNpmPackageMaintainers(),
+  ];
   return syncedVersionNames;
 };
 
@@ -826,13 +883,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   }
 
   // add module dependence
-  try {
-    yield dependencies.map(function (depName) {
-      return ModuleDeps.add(depName, sourcePackage.name);
-    });
-  } catch (err) {
-    // ignore
-  }
+  yield* packageService.addDependencies(sourcePackage.name, dependencies);
 
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
@@ -932,7 +983,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     }
 
     mod.package.dist = dist;
-    var r = yield Module.add(mod);
+    var r = yield* packageService.saveModule(mod);
 
     that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, '
       + 'size: %d, publish_time: %j, publish on cnpm: %s',
@@ -948,13 +999,14 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
 
 SyncModuleWorker.sync = function* (name, username, options) {
   options = options || {};
-  var result = yield Log.create({name: name, username: username});
+  var result = yield* logService.create({name: name, username: username});
   var worker = new SyncModuleWorker({
     logId: result.id,
     name: name,
     username: username,
     noDep: options.noDep,
     publish: options.publish,
+    syncUpstreamFirst: options.syncUpstreamFirst,
   });
   worker.start();
   return result.id;

controllers/total.js

@@ -15,15 +15,15 @@
  * Module dependencies.
  */
 
-var Total = require('../proxy/total');
-var Download = require('./download');
+var Total = require('../services/total');
 var version = require('../package.json').version;
 var config = require('../config');
+var getDownloadTotal = require('./utils').getDownloadTotal;
 
 var startTime = '' + Date.now();
 
-exports.show = function *() {
-  var r = yield [Total.get(), Download.total()];
+module.exports = function* showTotal() {
+  var r = yield [Total.get(), getDownloadTotal()];
   var total = r[0];
   var download = r[1];
 

controllers/utils.js

@@ -18,7 +18,9 @@ var debug = require('debug')('cnpmjs.org:controllers:utils');
 var path = require('path');
 var fs = require('fs');
 var utility = require('utility');
-var ms = require('ms');
+var ms = require('humanize-ms');
+var moment = require('moment');
+var downloadTotalService = require('../services/download_total');
 var nfs = require('../common/nfs');
 var config = require('../config');
 
@@ -44,3 +46,116 @@ exports.downloadAsReadStream = function* (key) {
   tarball.once('end', cleanup);
   return tarball;
 };
+
+exports.getDownloadTotal = function* (name) {
+  var end = moment();
+  var start = end.clone().subtract(1, 'months').startOf('month');
+  var lastday = end.clone().subtract(1, 'days').format('YYYY-MM-DD');
+  var lastweekStart = end.clone().subtract(1, 'weeks').startOf('week');
+  var lastweekEnd = lastweekStart.clone().endOf('week').format('YYYY-MM-DD');
+  var lastmonthEnd = start.clone().endOf('month').format('YYYY-MM-DD');
+  var thismonthStart = end.clone().startOf('month').format('YYYY-MM-DD');
+  var thisweekStart = end.clone().startOf('week').format('YYYY-MM-DD');
+  start = start.format('YYYY-MM-DD');
+  end = end.format('YYYY-MM-DD');
+  lastweekStart = lastweekStart.format('YYYY-MM-DD');
+  var method = name ? 'getModuleTotal' : 'getTotal';
+  var args = [start, end];
+  if (name) {
+    args.unshift(name);
+  }
+
+  var rows = yield* downloadTotalService[method].apply(downloadTotalService, args);
+  var download = {
+    today: 0,
+    thisweek: 0,
+    thismonth: 0,
+    lastday: 0,
+    lastweek: 0,
+    lastmonth: 0,
+  };
+
+  for (var i = 0; i < rows.length; i++) {
+    var r = rows[i];
+    if (r.date === end) {
+      download.today += r.count;
+    }
+    if (r.date >= thismonthStart) {
+      download.thismonth += r.count;
+    }
+    if (r.date >= thisweekStart) {
+      download.thisweek += r.count;
+    }
+
+    if (r.date === lastday) {
+      download.lastday += r.count;
+    }
+    if (r.date >= lastweekStart && r.date <= lastweekEnd) {
+      download.lastweek += r.count;
+    }
+    if (r.date >= start && r.date <= lastmonthEnd) {
+      download.lastmonth += r.count;
+    }
+  }
+  return download;
+};
+
+exports.setLicense = function (pkg) {
+  var license;
+  license = pkg.license || pkg.licenses || pkg.licence || pkg.licences;
+  if (!license) {
+    return ;
+  }
+
+  if (Array.isArray(license)) {
+    license = license[0];
+  }
+
+  if (typeof license === 'object') {
+    pkg.license = {
+      name: license.name || license.type,
+      url: license.url
+    };
+  }
+
+  if (typeof license === 'string') {
+    if (license.match(/(http|https)(:\/\/)/ig)) {
+      pkg.license = {
+        name: license,
+        url: license
+      };
+    } else {
+      pkg.license = {
+        url: exports.getOssLicenseUrlFromName(license),
+        name: license
+      };
+    }
+  }
+};
+
+exports.getOssLicenseUrlFromName = function (name) {
+  var base = 'http://opensource.org/licenses/';
+
+  var licenseMap = {
+    'bsd': 'BSD-2-Clause',
+    'mit': 'MIT',
+    'x11': 'MIT',
+    'mit/x11': 'MIT',
+    'apache 2.0': 'Apache-2.0',
+    'apache2': 'Apache-2.0',
+    'apache 2': 'Apache-2.0',
+    'apache-2': 'Apache-2.0',
+    'apache': 'Apache-2.0',
+    'gpl': 'GPL-3.0',
+    'gplv3': 'GPL-3.0',
+    'gplv2': 'GPL-2.0',
+    'gpl3': 'GPL-3.0',
+    'gpl2': 'GPL-2.0',
+    'lgpl': 'LGPL-2.1',
+    'lgplv2.1': 'LGPL-2.1',
+    'lgplv2': 'LGPL-2.1'
+  };
+
+  return licenseMap[name.toLowerCase()] ?
+    base + licenseMap[name.toLowerCase()] : base + name;
+};

controllers/web/badge.js

@@ -15,13 +15,13 @@
  */
 
 var config = require('../../config');
-var Module = require('../../proxy/module');
+var packageService = require('../../services/package');
 
-exports.version = function* (next) {
+exports.version = function* () {
   var color = 'lightgrey';
   var version = 'invalid';
   var name = this.params[0];
-  var latestTag = yield* Module.getTag(name, 'latest');
+  var latestTag = yield* packageService.getModuleByTag(name, 'latest');
   if (latestTag) {
     version = latestTag.version;
     if (/^0\.0\./.test(version)) {

controllers/web/dist.js

@@ -17,7 +17,7 @@
 var debug = require('debug')('cnpmjs.org:controllers:web:dist');
 var mime = require('mime');
 var urlparse = require('url').parse;
-var Dist = require('../../proxy/dist');
+var distService = require('../../services/dist');
 var config = require('../../config');
 var downloadAsReadStream = require('../utils').downloadAsReadStream;
 
@@ -44,7 +44,7 @@ exports.list = function* (next) {
     return yield* download.call(this, next);
   }
 
-  var items = yield* Dist.listdir(url);
+  var items = yield* distService.listdir(url);
   if (url === '/') {
     // phantomjs/
     items.push({
@@ -64,7 +64,7 @@ exports.list = function* (next) {
 
 function* download(next) {
   var fullname = this.params[0];
-  var info = yield* Dist.getfile(fullname);
+  var info = yield* distService.getfile(fullname);
   debug('download %s got %j', fullname, info);
   if (!info || !info.url) {
     return yield* next;

controllers/web/package/list_privates.js

@@ -0,0 +1,37 @@
+/**!
+ * cnpmjs.org - controllers/web/package/list_privates.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../../services/package');
+var config = require('../../../config');
+
+module.exports = function* listPrivates() {
+  var tasks = {};
+  for (var i = 0; i < config.scopes.length; i++) {
+    var scope = config.scopes[i];
+    tasks[scope] = packageService.listPrivateModulesByScope(scope);
+  }
+
+  if (config.privatePackages && config.privatePackages.length > 0) {
+    tasks['no scoped'] = packageService.listModules(config.privatePackages);
+  }
+
+  var scopes = yield tasks;
+  yield this.render('private', {
+    title: 'private packages',
+    scopes: scopes
+  });
+};

controllers/web/package/search.js

@@ -0,0 +1,54 @@
+/**!
+ * cnpmjs.org - controllers/web/package/search.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:web:package:search');
+var packageService = require('../../../services/package');
+
+module.exports = function* search() {
+  var params = this.params;
+  var word = params.word || params[0];
+  debug('search %j', word);
+  var result = yield* packageService.search(word);
+
+  var match = null;
+  for (var i = 0; i < result.searchMatchs.length; i++) {
+    var p = result.searchMatchs[i];
+    if (p.name === word) {
+      match = p;
+      break;
+    }
+  }
+
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.body = {
+      keyword: word,
+      match: match,
+      packages: result.searchMatchs,
+      keywords: result.keywordMatchs,
+    };
+    this.type = 'application/json; charset=utf-8';
+    return;
+  }
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    match: match,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
+  });
+};

controllers/web/package/search_range.js

@@ -0,0 +1,49 @@
+/**!
+ * cnpmjs.org - controllers/web/package/search_range.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../../services/package');
+
+module.exports = function* searchRange() {
+  var startKey = this.query.startkey || '';
+  if (startKey[0] === '"') {
+    startKey = startKey.substring(1);
+  }
+  if (startKey[startKey.length - 1] === '"') {
+    startKey = startKey.substring(0, startKey.length - 1);
+  }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield* packageService.search(startKey, {limit: limit});
+
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
+
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
+};

controllers/web/package/show.js

@@ -1,11 +1,12 @@
-/*!
- * cnpmjs.org - controllers/web/package.js
+/**!
+ * cnpmjs.org - controllers/web/package/show.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
  */
 
 'use strict';
@@ -14,27 +15,20 @@
  * Module dependencies.
  */
 
-var debug = require('debug')('cnpmjs.org:controllers:web:package');
+var debug = require('debug')('cnpmjs.org:controllers:web:package:show');
 var bytes = require('bytes');
 var giturl = require('giturl');
 var moment = require('moment');
-var eventproxy = require('eventproxy');
 var semver = require('semver');
 var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
-var config = require('../../config');
-var Module = require('../../proxy/module');
-var down = require('../download');
-var sync = require('../sync');
-var Log = require('../../proxy/module_log');
-var ModuleDeps = require('../../proxy/module_deps');
-var setDownloadURL = require('../../lib/common').setDownloadURL;
-var ModuleStar = require('../../proxy/module_star');
-var packageService = require('../../services/package');
-var ModuleUnpublished = require('../../proxy/module_unpublished');
+var config = require('../../../config');
+var utils = require('../../utils');
+var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var packageService = require('../../../services/package');
 
-exports.display = function* (next) {
+module.exports = function* show(next) {
   var params = this.params;
   // normal: {name: $name, version: $version}
   // scope: [$name, $version]
@@ -47,25 +41,17 @@ exports.display = function* (next) {
   var getPackageArgs;
   var version = semver.valid(tag || '');
   if (version) {
-    getPackageMethod = 'get';
+    getPackageMethod = 'getModule';
     getPackageArgs = [name, version];
   } else {
-    getPackageMethod = 'getByTag';
+    getPackageMethod = 'getModuleByTag';
     getPackageArgs = [name, tag || 'latest'];
   }
 
-  var pkg = yield Module[getPackageMethod].apply(Module, getPackageArgs);
-  if (!pkg) {
-    var adaptName = yield* Module.getAdaptName(name);
-    if (adaptName) {
-      name = adaptName;
-      pkg = yield Module[getPackageMethod].apply(Module, [name, getPackageArgs[1]]);
-    }
-  }
-
+  var pkg = yield packageService[getPackageMethod].apply(packageService, getPackageArgs);
   if (!pkg || !pkg.package) {
     // check if unpublished
-    var unpublishedInfo = yield* ModuleUnpublished.get(name);
+    var unpublishedInfo = yield* packageService.getUnpublishedModule(name);
     debug('show unpublished %j', unpublishedInfo);
     if (unpublishedInfo) {
       var data = {
@@ -91,15 +77,13 @@ exports.display = function* (next) {
   }
 
   var r = yield [
-    down.total(name),
-    ModuleDeps.list(name),
-    ModuleStar.listUsers(name),
+    utils.getDownloadTotal(name),
+    packageService.listDependents(name),
+    packageService.listStarUserNames(name),
     packageService.listMaintainers(name)
   ];
   var download = r[0];
-  var dependents = (r[1] || []).map(function (item) {
-    return item.deps;
-  });
+  var dependents = r[1];
   var users = r[2];
   var maintainers = r[3];
 
@@ -124,6 +108,13 @@ exports.display = function* (next) {
     }
   }
 
+  if (pkg._npmUser) {
+    pkg.lastPublishedUser = pkg._npmUser;
+    if (pkg.lastPublishedUser.email) {
+      pkg.lastPublishedUser.gravatar = gravatar.url(pkg.lastPublishedUser.email, {s: '50', d: 'retro'}, true);
+    }
+  }
+
   if (pkg.contributors) {
     // registry.cnpmjs.org/compressible
     if (!Array.isArray(pkg.contributors)) {
@@ -144,7 +135,7 @@ exports.display = function* (next) {
     pkg.repository.weburl = giturl.parse(pkg.repository.url) || pkg.repository.url;
   }
 
-  setLicense(pkg);
+  utils.setLicense(pkg);
 
   for (var k in download) {
     download[k] = humanize(download[k]);
@@ -202,146 +193,3 @@ exports.display = function* (next) {
     download: download
   });
 };
-
-exports.search = function *(next) {
-  var params = this.params;
-  var word = params.word || params[0];
-  debug('search %j', word);
-  var result = yield Module.search(word);
-
-  var match = null;
-  for (var i = 0; i < result.searchMatchs.length; i++) {
-    var p = result.searchMatchs[i];
-    if (p.name === word) {
-      match = p;
-      break;
-    }
-  }
-
-  // return a json result
-  if (this.query && this.query.type === 'json') {
-    this.body = {
-      keyword: word,
-      match: match,
-      packages: result.searchMatchs,
-      keywords: result.keywordMatchs,
-    };
-    this.type = 'application/json; charset=utf-8';
-    return;
-  }
-  yield this.render('search', {
-    title: 'Keyword - ' + word,
-    keyword: word,
-    match: match,
-    packages: result.searchMatchs,
-    keywords: result.keywordMatchs,
-  });
-};
-
-exports.rangeSearch = function *(next) {
-  var startKey = this.query.startkey || '';
-  if (startKey[0] === '"') {
-    startKey = startKey.substring(1);
-  }
-  if (startKey[startKey.length - 1] === '"') {
-    startKey = startKey.substring(0, startKey.length - 1);
-  }
-  var limit = Number(this.query.limit) || 20;
-  var result = yield Module.search(startKey, {limit: limit});
-
-  var packages = result.searchMatchs.concat(result.keywordMatchs);
-
-  var rows = [];
-  for (var i = 0; i < packages.length; i++) {
-    var p = packages[i];
-    var row = {
-      key: p.name,
-      count: 1,
-      value: {
-        name: p.name,
-        description: p.description,
-      }
-    };
-    rows.push(row);
-  }
-  this.body = {
-    rows: rows
-  };
-};
-
-exports.displaySync = function* (next) {
-  var name = this.params.name || this.params[0] || this.query.name;
-  yield this.render('sync', {
-    name: name,
-    title: 'Sync - ' + name,
-  });
-};
-
-exports.listPrivates = function* () {
-  var packages = yield Module.listPrivates();
-  yield this.render('private', {
-      title: 'private packages',
-      packages: packages
-    });
-};
-
-function setLicense(pkg) {
-  var license;
-  license = pkg.license || pkg.licenses || pkg.licence || pkg.licences;
-  if (!license) {
-    return ;
-  }
-
-  if (Array.isArray(license)) {
-    license = license[0];
-  }
-
-  if (typeof license === 'object') {
-    pkg.license = {
-      name: license.name || license.type,
-      url: license.url
-    };
-  }
-
-  if (typeof license === 'string') {
-    if (license.match(/(http|https)(:\/\/)/ig)) {
-      pkg.license = {
-        name: license,
-        url: license
-      };
-    } else {
-      pkg.license = {
-        url: getOssLicenseUrlFromName(license),
-        name: license
-      };
-    }
-  }
-}
-exports.setLicense = setLicense;
-
-function getOssLicenseUrlFromName(name) {
-  var base = 'http://opensource.org/licenses/';
-
-  var licenseMap = {
-    'bsd': 'BSD-2-Clause',
-    'mit': 'MIT',
-    'x11': 'MIT',
-    'mit/x11': 'MIT',
-    'apache 2.0': 'Apache-2.0',
-    'apache2': 'Apache-2.0',
-    'apache 2': 'Apache-2.0',
-    'apache-2': 'Apache-2.0',
-    'apache': 'Apache-2.0',
-    'gpl': 'GPL-3.0',
-    'gplv3': 'GPL-3.0',
-    'gplv2': 'GPL-2.0',
-    'gpl3': 'GPL-3.0',
-    'gpl2': 'GPL-2.0',
-    'lgpl': 'LGPL-2.1',
-    'lgplv2.1': 'LGPL-2.1',
-    'lgplv2': 'LGPL-2.1'
-  };
-
-  return licenseMap[name.toLowerCase()] ?
-    base + licenseMap[name.toLowerCase()] : base + name;
-}

controllers/web/package/show_sync.js

@@ -0,0 +1,27 @@
+/**!
+ * cnpmjs.org - controllers/web/package/show_sync.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+module.exports = function* showSync() {
+  var name = this.params.name || this.params[0] || this.query.name;
+  if (!name) {
+    return this.redirect('/');
+  }
+  yield this.render('sync', {
+    name: name,
+    title: 'Sync - ' + name,
+  });
+};

controllers/web/user/show.js

@@ -1,5 +1,5 @@
 /**!
- * cnpmjs.org - controllers/web/package.js
+ * cnpmjs.org - controllers/web/user/show.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -15,30 +15,18 @@
  * Module dependencies.
  */
 
-var config = require('../../config');
-var Module = require('../../proxy/module');
-var User = require('../../proxy/user');
-var UserService = require('../../services/user');
-var common = require('../../lib/common');
+var config = require('../../../config');
+var packageService = require('../../../services/package');
+var userService = require('../../../services/user');
+var common = require('../../../lib/common');
 
-exports.display = function* (next) {
+module.exports = function* showUser(next) {
   var name = this.params.name;
   var isAdmin = common.isAdmin(name);
   var scopes = config.scopes || [];
-  if (config.customUserService) {
-    var customUser = yield* UserService.get(name);
-    if (customUser) {
-      isAdmin = !!customUser.site_admin;
-      scopes = customUser.scopes;
-      var data = {
-        user: customUser
-      };
-      yield* User.saveCustomUser(data);
-    }
-  }
-
-  var r = yield [Module.listByAuthor(name), User.get(name)];
-  var packages = r[0] || [];
+  var user;
+  var r = yield [packageService.listModulesByUser(name), userService.getAndSave(name)];
+  var packages = r[0];
   var user = r[1];
   if (!user && !packages.length) {
     return yield* next;
@@ -75,7 +63,7 @@ exports.display = function* (next) {
     title: 'User - ' + name,
     packages: packages,
     user: data,
-    lastModified: user && user.gmt_modified,
+    lastModified: user.gmt_modified,
     isAdmin: isAdmin,
     scopes: scopes
   });

dispatch.js

@@ -15,13 +15,12 @@
  * Module dependencies.
  */
 
+var childProcess = require('child_process');
 var path = require('path');
 var util = require('util');
-var cluster = require('cluster');
 var cfork = require('cfork');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
-var childProcess = require('child_process');
 var syncPath = path.join(__dirname, 'sync');
 
 if (config.enableCluster) {

docs/db.sql

@@ -1,4 +1,4 @@
-CREATE TABLE `user` (
+CREATE TABLE IF NOT EXISTS `user` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -19,7 +19,7 @@ CREATE TABLE `user` (
 --   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
 --   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
 
-CREATE TABLE `module_keyword` (
+CREATE TABLE IF NOT EXISTS `module_keyword` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
@@ -30,7 +30,7 @@ CREATE TABLE `module_keyword` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
 
-CREATE TABLE `module_star` (
+CREATE TABLE IF NOT EXISTS `module_star` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
@@ -40,7 +40,7 @@ CREATE TABLE `module_star` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
 
-CREATE TABLE `module_maintainer` (
+CREATE TABLE IF NOT EXISTS `module_maintainer` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
@@ -48,9 +48,19 @@ CREATE TABLE `module_maintainer` (
  PRIMARY KEY (`id`),
  UNIQUE KEY `user_module_name` (`user`,`name`),
  KEY `name` (`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='private module maintainers';
 
-CREATE TABLE `module` (
+CREATE TABLE IF NOT EXISTS `npm_module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='npm original module maintainers';
+
+CREATE TABLE IF NOT EXISTS `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -76,7 +86,7 @@ CREATE TABLE `module` (
 -- show create table module\G
 -- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
-CREATE TABLE `module_log` (
+CREATE TABLE IF NOT EXISTS `module_log` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -88,7 +98,7 @@ CREATE TABLE `module_log` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
 -- ALTER TABLE  `module_log` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
-CREATE TABLE `tag` (
+CREATE TABLE IF NOT EXISTS `tag` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -104,7 +114,7 @@ CREATE TABLE `tag` (
 -- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 -- ALTER TABLE `tag` ADD KEY `gmt_modified` (`gmt_modified`);
 
-CREATE TABLE `module_unpublished` (
+CREATE TABLE IF NOT EXISTS `module_unpublished` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -115,7 +125,7 @@ CREATE TABLE `module_unpublished` (
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module unpublished info';
 
-CREATE TABLE `total` (
+CREATE TABLE IF NOT EXISTS `total` (
  `name` varchar(100) NOT NULL COMMENT 'total name',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `module_delete` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'module delete count',
@@ -129,7 +139,9 @@ CREATE TABLE `total` (
  `last_sync_module` varchar(100) COMMENT 'last sync success module name',
  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
-INSERT INTO total(name, gmt_modified) VALUES('total', now());
+-- init `total` count
+INSERT INTO total(name, gmt_modified) VALUES('total', now())
+  ON DUPLICATE KEY UPDATE gmt_modified=now();
 -- ALTER TABLE `total` ADD `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry'
 -- ALTER TABLE `total` ADD `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry'
 -- ALTER TABLE `total` ADD `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status'
@@ -139,7 +151,7 @@ INSERT INTO total(name, gmt_modified) VALUES('total', now());
 -- ALTER TABLE `total` ADD `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync'
 -- ALTER TABLE `total` ADD `last_sync_module` varchar(100) NOT NULL COMMENT 'last sync success module name';
 
-CREATE TABLE `download_total` (
+CREATE TABLE IF NOT EXISTS `download_total` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
@@ -151,7 +163,7 @@ CREATE TABLE `download_total` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
 -- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
-CREATE TABLE `module_deps` (
+CREATE TABLE IF NOT EXISTS `module_deps` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
@@ -161,11 +173,11 @@ CREATE TABLE `module_deps` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
 
-CREATE TABLE `dist_dir` (
+CREATE TABLE IF NOT EXISTS `dist_dir` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(200) NOT NULL COMMENT 'user name',
+ `name` varchar(200) NOT NULL COMMENT 'dir name',
  `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
  `date` varchar(20) COMMENT '02-May-2014 01:06',
  PRIMARY KEY (`id`),
@@ -173,11 +185,11 @@ CREATE TABLE `dist_dir` (
  KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
 
-CREATE TABLE `dist_file` (
+CREATE TABLE IF NOT EXISTS `dist_file` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'user name',
+ `name` varchar(100) NOT NULL COMMENT 'file name',
  `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
  `date` varchar(20) COMMENT '02-May-2014 01:06',
  `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',

docs/update_sqls/v1.6.x-upgrade.sql

@@ -1,4 +1,4 @@
-CREATE TABLE `module_maintainer` (
+CREATE TABLE IF NOT EXISTS `npm_module_maintainer` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
@@ -6,4 +6,4 @@ CREATE TABLE `module_maintainer` (
  PRIMARY KEY (`id`),
  UNIQUE KEY `user_module_name` (`user`,`name`),
  KEY `name` (`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='npm original module maintainers';

docs/web/readme.md

@@ -53,57 +53,7 @@ So `cnpm` is meaning: **Company npm**.
 
 Running on [Node.js](http://nodejs.org), version <span id="node-version"></span>.
 
-<script>
-$(function () {
-  function humanize(n, options) {
-    options = options || {};
-    var d = options.delimiter || ',';
-    var s = options.separator || '.';
-    n = n.toString().split('.');
-    n[0] = n[0].replace(/(\d)(?=(\d\d\d)+(?!\d))/g, '$1' + d);
-    return n.join(s);
-  }
-
-  $.getJSON('/total', function (data) {
-    $('#total-packages').html(humanize(data.doc_count));
-    $('#total-versions').html(humanize(data.doc_version_count));
-    $('#total-deletes').html(humanize(data.doc_del_count));
-
-    var downloads = $('table.downloads');
-    downloads.find('td.count:eq(3)').html(humanize(data.download.today));
-    downloads.find('td.count:eq(4)').html(humanize(data.download.thisweek));
-    downloads.find('td.count:eq(5)').html(humanize(data.download.thismonth));
-    downloads.find('td.count:eq(6)').html(humanize(data.download.lastday));
-    downloads.find('td.count:eq(7)').html(humanize(data.download.lastweek));
-    downloads.find('td.count:eq(8)').html(humanize(data.download.lastmonth));
-
-    $('#node-version').html(data.node_version || 'v0.10.22');
-    $('#app-version').html(data.app_version || '0.0.0');
-
-    if (data.sync_model === 'all') {
-      $('#sync-model').html('This registry will sync all packages from official registry.');
-      $('#last-sync-time').html(new Date(data.last_sync_time));
-    } else if (data.sync_model === 'exist') {
-      $('#sync-model').html('This registry will only update exist packages from official registry.');
-      $('#last-sync-time').html(new Date(data.last_exist_sync_time));
-    }
-
-    $('#need-sync').html(data.need_sync_num);
-    $('#success-sync').html(data.success_sync_num);
-    $('#fail-sync').html(data.fail_sync_num);
-    $('#left-sync').html(data.left_sync_num);
-    $('#percent-sync').html(Math.floor(data.success_sync_num / data.need_sync_num * 100));
-    $('#last-success-name').html('<a target="_blank" href="/package/' + data.last_sync_module + '">' +
-      data.last_sync_module + '</a>');
-
-    if (!data.sync_status) {
-      $('.syncing').html('');
-    }
-
-    $('.sync').show();
-  });
-});
-</script>
+<script src="/js/readme.js"></script>
 
 ## Version Badge
 

middleware/auth.js

@@ -22,7 +22,7 @@ var UserService = require('../services/user');
  * get the real user
  */
 
-module.exports = function (options) {
+module.exports = function () {
   return function* auth(next) {
     this.user = {};
 

middleware/editable.js

@@ -0,0 +1,42 @@
+/**!
+ * cnpmjs.org - middleware/editable.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../services/package');
+
+// admin or module's maintainer can modified the module
+module.exports = function* editable(next) {
+  var username = this.user && this.user.name;
+  var moduleName = this.params.name || this.params[0];
+  if (username && moduleName) {
+    if (this.user.isAdmin) {
+      return yield* next;
+    }
+    var isMaintainer = yield* packageService.isMaintainer(moduleName, username);
+    if (isMaintainer) {
+      return yield* next;
+    }
+  }
+
+  this.status = 403;
+  var message = 'not authorized to modify ' + moduleName;
+  if (username) {
+    message = username + ' ' + message;
+  }
+  this.body = {
+    error: 'forbidden user',
+    reason: message
+  };
+};

middleware/login.js

@@ -31,7 +31,6 @@ module.exports = function *login(next) {
   }
 
   if (!this.user.name) {
-
     this.status = 401;
     this.body = {
       error: 'unauthorized',

middleware/opensearch.js

@@ -22,8 +22,6 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
     <Url method="get" type="text/html" template="http://${host}/browse/keyword/{searchTerms}"/>\
  </OpenSearchDescription>';
 
-var lastModifyDate = new Date();
-
 module.exports = function *publishable(next) {
   if (this.path === '/opensearch.xml') {
     this.type = 'text/xml';

middleware/publishable.js

@@ -21,7 +21,6 @@ var debug = require('debug')('cnpmjs.org:middlewares/publishable');
 module.exports = function *publishable(next) {
   // private mode, only admin user can publish
   if (config.enablePrivate && !this.user.isAdmin) {
-
     this.status = 403;
     this.body = {
       error: 'no_perms',
@@ -71,7 +70,7 @@ function checkScope(name, ctx) {
     ctx.status = 400;
     ctx.body = {
       error: 'invalid scope',
-      reason: util.format('scope %s not match legal scopes %j', scope, ctx.user.scopes)
+      reason: util.format('scope %s not match legal scopes: %s', scope, ctx.user.scopes.join(', '))
     };
     return false;
   }
@@ -98,6 +97,6 @@ function checkNoneScope(ctx) {
   ctx.status = 403;
   ctx.body = {
     error: 'no_perms',
-    reason: 'only allow publish with ' + ctx.user.scopes.join(',') + ' scope(s)'
+    reason: 'only allow publish with ' + ctx.user.scopes.join(', ') + ' scope(s)'
   };
 }

middleware/registry_not_found.js

@@ -14,8 +14,8 @@
  * Module dependencies.
  */
 
-module.exports = function *notFound(next) {
-  yield *next;
+module.exports = function* notFound(next) {
+  yield* next;
 
   if (this.status && this.status !== 404) {
     return;

middleware/sync_by_install.js

@@ -21,6 +21,7 @@ var config = require('../config');
  */
 
 module.exports = function* syncByInstall(next) {
+  this.allowSync = false;
   if (!config.syncByInstall || !config.enablePrivate) {
     // only config.enablePrivate should enable sync on install
     return yield* next;
@@ -36,6 +37,13 @@ module.exports = function* syncByInstall(next) {
     return yield* next;
   }
 
+  var name = this.params.name || this.params[0];
+
+  // scoped package dont sync
+  if (name && name[0] === '@') {
+    return yield* next;
+  }
+
   this.allowSync = true;
   yield* next;
 };

middleware/web_not_found.js

@@ -16,8 +16,8 @@
 
 var debug = require('debug')('cnpmjs.org:middleware:web_not_found');
 
-module.exports = function *notFound(next) {
-  yield *next;
+module.exports = function* notFound(next) {
+  yield* next;
 
   if (this.status && this.status !== 404) {
     return;
@@ -38,16 +38,16 @@ module.exports = function *notFound(next) {
 
   // package not found
   m = /\/package\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  var name = null;
+  var title = '404: Page Not Found';
   if (m) {
-    var name = m[1];
-    this.status = 404;
-    yield* this.render('404', {
-      title: 'Package - ' + name,
-      name: name
-    });
-    return;
+    name = m[1];
+    title = name + ' Not Found';
   }
 
   this.status = 404;
-  this.body = 'Cannot GET ' + this.path;
+  yield* this.render('404', {
+    title: title,
+    name: name
+  });
 };

models/_module_maintainer_class_methods.js

@@ -0,0 +1,159 @@
+/**!
+ * cnpmjs.org - models/_module_maintainer_class_methods.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/**
+ * list all module names by user
+ * @param {String} user
+ */
+
+exports.listModuleNamesByUser = function* (user) {
+  var rows = yield this.findAll({
+    attributrs: ['name'],
+    where: {
+      user: user
+    }
+  });
+  return rows.map(function (row) {
+    return row.name;
+  });
+};
+
+/**
+ * list all maintainers of module `name`
+ * @param {String} name
+ */
+
+exports.listMaintainers = function* (name) {
+  var rows = yield this.findAll({
+    attributrs: ['user'],
+    where: {
+      name: name
+    }
+  });
+  return rows.map(function (row) {
+    return row.user;
+  });
+};
+
+/**
+ * add a maintainer for module `name`
+ * @param {String} name
+ * @param {String} user
+ */
+
+exports.addMaintainer = function* (name, user) {
+  var row = yield this.find({
+    where: {
+      user: user,
+      name: name
+    }
+  });
+  if (!row) {
+    row = yield this.build({
+      user: user,
+      name: name
+    }).save();
+  }
+  return row;
+};
+
+/**
+ * add maintainers for module `name`
+ * @param {String} name
+ * @param {Array} users
+ */
+
+exports.addMaintainers = function* (name, users) {
+  return yield users.map(function (user) {
+    return this.addMaintainer(name, user);
+  }.bind(this));
+};
+
+/**
+ * remove maintainers for module `name`
+ * @param {String} name
+ * @param {Array} users
+ */
+
+exports.removeMaintainers = function* (name, users) {
+  // removeMaintainers(name, oneUserName)
+  if (typeof users === 'string') {
+    users = [users];
+  }
+  if (users.length === 0) {
+    return;
+  }
+  yield this.destroy({
+    where: {
+      name: name,
+      user: users,
+    }
+  });
+};
+
+/**
+ * remove all maintainers for module `name`
+ * @param {String} name
+ */
+
+exports.removeAllMaintainers = function* (name) {
+  yield this.destroy({
+    where: {
+      name: name
+    }
+  });
+};
+
+/**
+ * add maintainers to module
+ * @param {String} name
+ * @param {Array} users
+ */
+
+exports.updateMaintainers = function* (name, users) {
+  // maintainers should be [username1, username2, ...] format
+  // find out the exists maintainers
+  // then remove all the users not present and add all the left
+
+  if (users.length === 0) {
+    return {
+      add: [],
+      remove: []
+    };
+  }
+  var exists = yield* this.listMaintainers(name);
+
+  var addUsers = users.filter(function (username) {
+    // add user which in `users` but do not in `exists`
+    return exists.indexOf(username) === -1;
+  });
+
+  var removeUsers = exists.filter(function (username) {
+    // remove user which in `exists` by not in `users`
+    return users.indexOf(username) === -1;
+  });
+
+  yield [
+    this.addMaintainers(name, addUsers),
+    this.removeMaintainers(name, removeUsers),
+  ];
+
+  return {
+    add: addUsers,
+    remove: removeUsers
+  };
+};
+

models/dist_dir.js

@@ -0,0 +1,64 @@
+/**!
+ * cnpmjs.org - models/dist_dir.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `dist_dir` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(200) NOT NULL COMMENT 'dir name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `dist_dir_parent_name` (`parent`, `name`),
+ KEY `dist_dir_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('DistDir', {
+    name: {
+      type: DataTypes.STRING(200),
+      allowNull: false,
+      comment: 'dir name',
+    },
+    parent: {
+      type: DataTypes.STRING(200),
+      allowNull: false,
+      defaultValue: '/',
+      comment: 'parent dir',
+    },
+    date: {
+      type: DataTypes.STRING(20),
+      allowNull: false,
+      comment: '02-May-2014 01:06'
+    }
+  }, {
+    tableName: 'dist_dir',
+    comment: 'dist dir info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['parent', 'name']
+      },
+      {
+        fields: ['gmt_modified']
+      }
+    ],
+    classMethods: {
+    }
+  });
+};

models/dist_file.js

@@ -0,0 +1,82 @@
+/**!
+ * cnpmjs.org - models/dist_file.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `dist_file` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) NOT NULL COMMENT 'file name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
+ `sha1` varchar(40) COMMENT 'sha1 hex value',
+ `url` varchar(2048),
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `dist_file_parent_name` (`parent`, `name`),
+ KEY `dist_file_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('DistFile', {
+    name: {
+      type: DataTypes.STRING(200),
+      allowNull: false,
+      comment: 'dir name',
+    },
+    parent: {
+      type: DataTypes.STRING(200),
+      allowNull: false,
+      defaultValue: '/',
+      comment: 'parent dir',
+    },
+    date: {
+      type: DataTypes.STRING(20),
+      allowNull: false,
+      comment: '02-May-2014 01:06'
+    },
+    size: {
+      type: DataTypes.INTEGER(10).UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'file size'
+    },
+    sha1: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+      comment: 'sha1 hex value'
+    },
+    url: {
+      type: DataTypes.STRING(2048),
+      allowNull: false
+    }
+  }, {
+    tableName: 'dist_file',
+    comment: 'dist file info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['parent', 'name']
+      },
+      {
+        fields: ['gmt_modified']
+      }
+    ],
+    classMethods: {
+    }
+  });
+};

models/download_total.js

@@ -0,0 +1,59 @@
+/**!
+ * cnpmjs.org - models/download_total.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+// CREATE TABLE IF NOT EXISTS `download_total` (
+//  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+//  `gmt_create` datetime NOT NULL COMMENT 'create time',
+//  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+//  `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
+//  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+//  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
+//  PRIMARY KEY (`id`),
+//  UNIQUE KEY `download_total_date_name` (`date`, `name`)
+// ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('DownloadTotal', {
+    date: {
+      type: DataTypes.STRING(10),
+      allowNull: false,
+      comment: 'YYYY-MM-DD format',
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    count: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'download count',
+    }
+  }, {
+    tableName: 'download_total',
+    comment: 'module download total info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['date', 'name']
+      }
+    ],
+    classMethods: {
+
+    }
+  });
+};

models/index.js

@@ -0,0 +1,49 @@
+/**!
+ * cnpmjs.org - models/index.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var path = require('path');
+var sequelize = require('../common/sequelize');
+
+function load(name) {
+  return sequelize.import(path.join(__dirname, name));
+}
+
+module.exports = {
+  sequelize: sequelize,
+  Module: load('module'),
+  ModuleLog: load('module_log'),
+  ModuleStar: load('module_star'),
+  ModuleKeyword: load('module_keyword'),
+  ModuleDependency: load('module_deps'),
+  ModuleMaintainer: load('module_maintainer'),
+  ModuleUnpublished: load('module_unpublished'),
+  NpmModuleMaintainer: load('npm_module_maintainer'),
+
+  Tag: load('tag'),
+  User: load('user'),
+  Total: load('total'),
+  DownloadTotal: load('download_total'),
+  DistFile: load('dist_file'),
+  DistDir: load('dist_dir'),
+
+  query: function* (sql, args) {
+    return yield this.sequelize.query(sql, null, {raw: true}, args);
+  },
+  queryOne: function* (sql, args) {
+    var rows = yield* this.query(sql, args);
+    return rows && rows[0];
+  }
+};

models/init_script.js

@@ -0,0 +1,48 @@
+/**!
+ * cnpmjs.org - models/init_script.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+
+config.database.logging = console.log;
+
+// $ node --harmony models/init_script.js <force> <dialect>
+var force = process.argv[2] === 'true';
+var dialect = process.argv[3];
+if (dialect) {
+  config.database.dialect = dialect;
+}
+
+var models = require('./');
+
+models.sequelize.sync({ force: force })
+  .then(function () {
+    models.Total.init(function (err) {
+      if (err) {
+        console.error('[models/init_script.js] sequelize init fail');
+        console.error(err);
+        throw err;
+      } else {
+        console.log('[models/init_script.js] `%s` sequelize sync and init success',
+          config.database.dialect);
+        process.exit(0);
+      }
+    });
+  })
+  .catch(function (err) {
+    console.error('[models/init_script.js] sequelize sync fail');
+    console.error(err);
+    throw err;
+  });

models/module.js

@@ -0,0 +1,110 @@
+/**!
+ * cnpmjs.org - models/module.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var utils = require('./utils');
+
+/*
+CREATE TABLE IF NOT EXISTS `module` (
+  `id` INTEGER NOT NULL auto_increment ,
+  `author` VARCHAR(100) NOT NULL,
+  `name` VARCHAR(100) NOT NULL,
+  `version` VARCHAR(30) NOT NULL,
+  `description` LONGTEXT,
+  `package` LONGTEXT,
+  `dist_shasum` VARCHAR(100),
+  `dist_tarball` VARCHAR(2048),
+  `dist_size` INTEGER UNSIGNED NOT NULL DEFAULT 0,
+  `publish_time` BIGINT(20) UNSIGNED,
+  `gmt_create` DATETIME NOT NULL,
+  `gmt_modified` DATETIME NOT NULL,
+  PRIMARY KEY (`id`)
+)
+COMMENT 'module info' ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE utf8_general_ci;
+
+CREATE UNIQUE INDEX `module_name_version` ON `module` (`name`, `version`);
+CREATE INDEX `module_gmt_modified` ON `module` (`gmt_modified`);
+CREATE INDEX `module_publish_time` ON `module` (`publish_time`);
+CREATE INDEX `module_author` ON `module` (`author`);
+*/
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('Module', {
+    author: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'first maintainer name'
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name'
+    },
+    version: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'module version'
+    },
+    description: {
+      type: DataTypes.LONGTEXT,
+    },
+    package: {
+      type: DataTypes.LONGTEXT,
+      comment: 'package.json',
+    },
+    dist_shasum: {
+      type: DataTypes.STRING(100),
+      allowNull: true,
+    },
+    dist_tarball: {
+      type: DataTypes.STRING(2048),
+      allowNull: true,
+    },
+    dist_size: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+    },
+    publish_time: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: true,
+    }
+  }, {
+    tableName: 'module',
+    comment: 'module info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name', 'version']
+      },
+      {
+        fields: ['gmt_modified']
+      },
+      {
+        fields: ['publish_time']
+      },
+      {
+        fields: ['author']
+      }
+    ],
+    classMethods: {
+      findByNameAndVersion: function* (name, version) {
+        return yield this.find({
+          where: { name: name, version: version }
+        });
+      }
+    }
+  });
+};

models/module_deps.js

@@ -0,0 +1,58 @@
+/**!
+ * cnpmjs.org - models/module_deps.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '`name` is deped by `deps`',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `module_deps_name_deps` (`name`,`deps`),
+ KEY `deps` (`module_deps_deps`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleDependency', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    dependent: {
+      field: 'deps',
+      type: DataTypes.STRING(100),
+      comment: '`name` is depended by `deps`. `deps` == depend => `name`'
+    }
+  }, {
+    tableName: 'module_deps',
+    comment: 'module deps',
+    // no need update timestamp
+    updatedAt: false,
+    indexes: [
+      {
+        unique: true,
+        fields: ['name', 'deps']
+      },
+      {
+        fields: ['deps']
+      }
+    ],
+    classMethods: {
+    }
+  });
+};

models/module_keyword.js

@@ -0,0 +1,69 @@
+/**!
+ * cnpmjs.org - models/module_keyword.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `module_keyword` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `description` longtext,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `keyword_module_name` (`keyword`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleKeyword', {
+    keyword: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    description: {
+      type: DataTypes.LONGTEXT,
+      allowNull: true,
+    }
+  }, {
+    tableName: 'module_keyword',
+    comment: 'module keyword',
+    updatedAt: false,
+    indexes: [
+      {
+        unique: true,
+        fields: ['keyword', 'name']
+      },
+      {
+        fields: ['name']
+      }
+    ],
+    classMethods: {
+      findByKeywordAndName: function* (keyword, name) {
+        return yield this.find({
+          where: {
+            keyword: keyword,
+            name: name
+          }
+        });
+      }
+    }
+  });
+};

models/module_log.js

@@ -0,0 +1,56 @@
+/**!
+ * cnpmjs.org - models/module_log.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `module_log` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `username` varchar(100) NOT NULL,
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `log` longtext,
+ PRIMARY KEY (`id`),
+ KEY `module_log_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleLog', {
+    username: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user name'
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    log: {
+      type: DataTypes.LONGTEXT
+    }
+  }, {
+    tableName: 'module_log',
+    comment: 'module sync log',
+    indexes: [
+      {
+        fields: ['name']
+      }
+    ],
+    classMethods: {
+    }
+  });
+};

models/module_maintainer.js

@@ -0,0 +1,56 @@
+/**!
+ * cnpmjs.org - models/module_maintainer.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `module_maintainer_user_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='private module maintainers';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleMaintainer', {
+    user: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user name'
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    }
+  }, {
+    tableName: 'module_maintainer',
+    comment: 'private module maintainers',
+    updatedAt: false,
+    indexes: [
+      {
+        unique: true,
+        fields: ['user', 'name']
+      },
+      {
+        fields: ['name']
+      }
+    ],
+    classMethods: require('./_module_maintainer_class_methods'),
+  });
+};

models/module_star.js

@@ -0,0 +1,57 @@
+/**!
+ * cnpmjs.org - models/module_star.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `module_star_user_name` (`user`,`name`),
+ KEY `module_star_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleStar', {
+    user: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user name'
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    }
+  }, {
+    tableName: 'module_star',
+    comment: 'module star',
+    updatedAt: false,
+    indexes: [
+      {
+        unique: true,
+        fields: ['user', 'name']
+      },
+      {
+        fields: ['name']
+      }
+    ],
+    classMethods: {
+    }
+  });
+};

models/module_unpublished.js

@@ -0,0 +1,84 @@
+/**!
+ * cnpmjs.org - models/module_unpublished.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var utils = require('./utils');
+
+/*
+CREATE TABLE IF NOT EXISTS `module_unpublished` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'base info: tags, time, maintainers, description, versions',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `module_unpublished_name` (`name`),
+ KEY `module_unpublished_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module unpublished info';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('ModuleUnpublished', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    package: {
+      type: DataTypes.LONGTEXT,
+      comment: 'base info: tags, time, maintainers, description, versions',
+      get: utils.JSONGetter('package'),
+      set: utils.JSONSetter('package'),
+    }
+  }, {
+    tableName: 'module_unpublished',
+    comment: 'module unpublished info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name']
+      },
+      {
+        fields: ['gmt_modified']
+      }
+    ],
+    classMethods: {
+      findByName: function* (name) {
+        return yield this.find({
+          where: {
+            name: name
+          }
+        });
+      },
+      save: function* (name, pkg) {
+        var row = yield this.find({
+          where: {
+            name: name
+          }
+        });
+        if (row) {
+          row.package = pkg;
+          return yield row.save(['package']);
+        }
+
+        row = this.build({
+          name: name,
+          package: pkg,
+        });
+        return yield row.save();
+      }
+    }
+  });
+};

models/npm_module_maintainer.js

@@ -0,0 +1,56 @@
+/**!
+ * cnpmjs.org - models/npm_module_maintainer.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `npm_module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `npm_module_maintainer_user_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='npm original module maintainers';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('NpmModuleMaintainer', {
+    user: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user name'
+    },
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    }
+  }, {
+    tableName: 'npm_module_maintainer',
+    comment: 'npm original module maintainers',
+    updatedAt: false,
+    indexes: [
+      {
+        unique: true,
+        fields: ['user', 'name']
+      },
+      {
+        fields: ['name']
+      }
+    ],
+    classMethods: require('./_module_maintainer_class_methods'),
+  });
+};

models/tag.js

@@ -0,0 +1,72 @@
+/**!
+ * cnpmjs.org - models/tag.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+/*
+CREATE TABLE IF NOT EXISTS `tag` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `tag` varchar(30) NOT NULL COMMENT 'tag name',
+ `version` varchar(30) NOT NULL COMMENT 'module version',
+ `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `tag_name_tag` (`name`, `tag`),
+ KEY `tag_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
+ */
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('Tag', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'module name',
+    },
+    tag: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'tag name',
+    },
+    version: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'module version',
+    },
+    module_id: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: false,
+      comment: 'module id'
+    }
+  }, {
+    tableName: 'tag',
+    comment: 'module tag',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name', 'tag']
+      },
+      {
+        fields: ['gmt_modified']
+      }
+    ],
+    classMethods: {
+      findByNameAndTag: function* (name, tag) {
+        return yield this.find({ where: { name: name, tag: tag } });
+      }
+    }
+  });
+};

models/total.js

@@ -0,0 +1,118 @@
+/**!
+ * cnpmjs.org - models/total.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+// CREATE TABLE IF NOT EXISTS `total` (
+//  `name` varchar(100) NOT NULL COMMENT 'total name',
+//  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+//  `module_delete` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'module delete count',
+//  `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry',
+//  `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry',
+//  `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status',
+//  `need_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages need to be sync',
+//  `success_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync success at this time',
+//  `fail_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync fail at this time',
+//  `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync',
+//  `last_sync_module` varchar(100) COMMENT 'last sync success module name',
+//  PRIMARY KEY (`name`)
+// ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
+// -- init `total` count
+// INSERT INTO total(name, gmt_modified) VALUES('total', now())
+//   ON DUPLICATE KEY UPDATE gmt_modified=now();
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('Total', {
+    name: {
+      type: DataTypes.STRING(100),
+      primaryKey: true,
+      comment: 'total name'
+    },
+    module_delete: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'module delete count',
+    },
+    last_sync_time: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'last timestamp sync from official registry',
+    },
+    last_exist_sync_time: {
+      type: DataTypes.BIGINT(20).UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'last timestamp sync exist packages from official registry',
+    },
+    sync_status: {
+      type: 'TINYINT',
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'system sync from official registry status',
+    },
+    need_sync_num: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'how many packages need to be sync',
+    },
+    success_sync_num: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'how many packages sync success at this time',
+    },
+    fail_sync_num: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'how many packages sync fail at this time',
+    },
+    left_sync_num: {
+      type: DataTypes.INTEGER.UNSIGNED,
+      allowNull: false,
+      defaultValue: 0,
+      comment: 'how many packages left to be sync',
+    },
+    last_sync_module: {
+      type: DataTypes.STRING(100),
+      allowNull: true,
+      comment: 'last sync success module name',
+    },
+  }, {
+    tableName: 'total',
+    comment: 'total info',
+    createdAt: false,
+    classMethods: {
+      init: function (callback) {
+        var that = this;
+        that.find({
+          where: { name: 'total' }
+        }).then(function (row) {
+          if (!row) {
+            that.build({name: 'total'}).save()
+              .then(function () {
+                callback();
+              })
+              .catch(callback);
+            return;
+          }
+          callback();
+        }).catch(callback);
+      }
+    }
+  });
+};

models/user.js

@@ -0,0 +1,241 @@
+/**!
+ * cnpmjs.org - models/user.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var utility = require('utility');
+var utils = require('./utils');
+
+/*
+CREATE TABLE IF NOT EXISTS `user` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) NOT NULL COMMENT 'user name',
+ `salt` varchar(100) NOT NULL,
+ `password_sha` varchar(100) NOT NULL COMMENT 'user password hash',
+ `ip` varchar(64) NOT NULL COMMENT 'user last request ip',
+ `roles` varchar(200) NOT NULL DEFAULT '[]',
+ `rev` varchar(40) NOT NULL,
+ `email` varchar(400) NOT NULL,
+ `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+ `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_name` (`name`),
+ KEY `user_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
+*/
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('User', {
+    name: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user name',
+    },
+    salt: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+    },
+    password_sha: {
+      type: DataTypes.STRING(100),
+      allowNull: false,
+      comment: 'user password hash',
+    },
+    ip: {
+      type: DataTypes.STRING(64),
+      allowNull: false,
+      comment: 'user last request ip',
+    },
+    roles: {
+      type: DataTypes.STRING(200),
+      allowNull: false,
+      defaultValue: '[]',
+    },
+    rev: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+    },
+    email: {
+      type: DataTypes.STRING(400),
+      allowNull: false,
+    },
+    json: {
+      type: DataTypes.LONGTEXT,
+      allowNull: true,
+      get: utils.JSONGetter('json'),
+      set: utils.JSONSetter('json'),
+    },
+    isNpmUser: {
+      field: 'npm_user',
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false,
+      comment: 'user sync from npm or not, 1: true, other: false',
+    }
+  }, {
+    tableName: 'user',
+    comment: 'user base info',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name']
+      },
+      {
+        fields: ['gmt_modified']
+      }
+    ],
+    classMethods: {
+      // utils
+      createPasswordSha: function (password, salt) {
+        return utility.sha1(password + salt);
+      },
+
+      // read
+      auth: function* (name, password) {
+        var user = yield* this.findByName(name);
+        if (user) {
+          var sha = this.createPasswordSha(password, user.salt);
+          if (user.password_sha !== sha) {
+            user = null;
+          }
+        }
+        return user;
+      },
+      findByName: function* (name) {
+        return yield this.find({ where: { name: name } });
+      },
+      listByNames: function* (names) {
+        return yield this.findAll({
+          where: {
+            name: {
+              in: names
+            }
+          }
+        });
+      },
+      search: function* (query, options) {
+        return yield this.findAll({
+          where: {
+            name: {
+              like: query + '%'
+            }
+          },
+          limit: options.limit
+        });
+      },
+
+      // write
+      saveNpmUser: function* (data) {
+        var user = yield* this.findByName(data.name);
+        if (!user) {
+          user = this.build({
+            isNpmUser: true,
+            name: data.name,
+            salt: '0',
+            password_sha: '0',
+            ip: '0',
+          });
+        }
+        user.isNpmUser = true;
+        user.json = data;
+        user.email = data.email || '';
+        user.rev = data._rev || '';
+        return yield user.save();
+      },
+      saveCustomUser: function* (data) {
+        var name = data.user.login;
+        var user = yield* this.findByName(name);
+        if (!user) {
+          user = this.build({
+            isNpmUser: false,
+            name: name,
+          });
+        }
+
+        var rev = '1-' + data.user.login;
+        var salt = data.salt || '0';
+        var passwordSha = data.password_sha || '0';
+        var ip = data.ip || '0';
+
+        user.isNpmUser = false;
+        user.email = data.user.email;
+        user.ip = ip;
+        user.json = data.user;
+        user.rev = rev;
+        user.salt = salt;
+        user.password_sha = passwordSha;
+        return yield user.save();
+      },
+
+      // add cnpm user
+      add: function* (user) {
+        var roles = user.roles || [];
+        try {
+          roles = JSON.stringify(roles);
+        } catch (e) {
+          roles = '[]';
+        }
+        var rev = '1-' + utility.md5(JSON.stringify(user));
+
+        var row = this.build({
+          rev: rev,
+          name: user.name,
+          email: user.email,
+          salt: user.salt,
+          password_sha: user.password_sha,
+          ip: user.ip,
+          roles: roles,
+          isNpmUser: false,
+        });
+
+        return yield row.save();
+      },
+
+      update: function* (user) {
+        var rev = user.rev || user._rev;
+        var revNo = Number(rev.split('-', 1));
+        if (!revNo) {
+          var err = new Error(rev + ' format error');
+          err.name = 'RevFormatError';
+          err.data = {user: user};
+          throw err;
+        }
+        revNo++;
+        var newRev = revNo + '-' + utility.md5(JSON.stringify(user));
+        var roles = user.roles || [];
+        try {
+          roles = JSON.stringify(roles);
+        } catch (e) {
+          roles = '[]';
+        }
+
+        var row = yield* this.findByName(user.name);
+        if (!row) {
+          return null;
+        }
+
+        row.rev = newRev;
+        row.email = user.email;
+        row.salt = user.salt;
+        row.password_sha = user.password_sha;
+        row.ip = user.ip;
+        row.roles = roles;
+        row.isNpmUser = false;
+
+        return yield row.save(['rev', 'email', 'salt', 'password_sha', 'ip', 'roles', 'isNpmUser']);
+      }
+    }
+  });
+};

models/utils.js

@@ -0,0 +1,34 @@
+/**!
+ * cnpmjs.org - models/utils.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+exports.JSONGetter = function (propertyName) {
+  return function JSONGetter() {
+    var value = this.getDataValue(propertyName);
+    if (value && typeof value === 'string') {
+      value = JSON.parse(value);
+    }
+    return value;
+  };
+};
+
+exports.JSONSetter = function (propertyName) {
+  return function JSONSetter(value) {
+    if (typeof value !== 'string') {
+      value = JSON.stringify(value);
+    }
+    this.setDataValue(propertyName, value);
+  };
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "1.6.0",
+  "version": "2.0.0-beta3",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -11,60 +11,57 @@
   },
   "dependencies": {
     "agentkeepalive": "~1.2.0",
-    "bluebird": "~2.3.5",
+    "bluebird": "~2.3.10",
     "bytes": "~1.0.0",
-    "cfork": "~1.2.0",
+    "cfork": "~1.2.1",
     "cheerio": "~0.17.0",
     "co": "~3.1.0",
     "co-defer": "~0.1.1",
     "co-gather": "~0.0.1",
-    "co-read": "~0.1.0",
     "co-redis": "~1.1.0",
     "co-sleep": "~0.0.1",
-    "co-write": "~0.3.0",
-    "copy-to": "~1.0.1",
-    "debug": "~2.0.0",
+    "copy-to": "~2.0.1",
+    "debug": "~2.1.0",
     "error-formater": "~1.0.3",
-    "eventproxy": "~0.3.1",
     "fs-cnpm": "~1.1.0",
     "giturl": "~0.0.3",
     "graceful": "~0.1.0",
     "gravatar": "~1.1.0",
+    "humanize-ms": "~1.0.0",
     "humanize-number": "~0.0.2",
-    "koa": "~0.12.2",
+    "koa": "~0.13.0",
     "koa-limit": "~1.0.2",
-    "koa-markdown": "~0.0.3",
+    "koa-markdown": "~1.0.0",
     "koa-middlewares": "~1.4.1",
     "marked": "~0.3.2",
     "mime": "~1.2.11",
     "mini-logger": "~1.0.0",
     "mkdirp": "~0.5.0",
     "moment": "~2.8.3",
-    "ms": "~0.6.2",
-    "multiline": "~1.0.1",
-    "mysql": "~2.5.1",
+    "mysql": "~2.5.2",
     "nodemailer": "~1.3.0",
-    "ready": "~0.1.1",
     "redis": "~0.12.1",
-    "semver": "~4.0.3",
-    "thunkify-wrap": "~1.0.2",
-    "urllib": "~1.5.2",
+    "semver": "~4.1.0",
+    "sequelize": "~2.0.0-rc2",
+    "thunkify-wrap": "~1.0.3",
+    "urllib": "~2.0.2",
     "utility": "~1.2.0"
   },
   "devDependencies": {
-    "autod": "^1.0.0",
-    "chunkstream": "~0.0.1",
-    "co-mocha": "0.0.2",
+    "autod": "*",
+    "chunkstream": "*",
+    "co-mocha": "*",
     "contributors": "*",
-    "istanbul-harmony": "~0.3.0",
-    "jshint": "~2.5.6",
-    "mm": "~0.2.1",
-    "mocha": "~1.21.4",
+    "istanbul-harmony": "*",
+    "jshint": "*",
+    "mm": "*",
+    "mocha": "*",
     "node-dev": "*",
-    "pedding": "~1.0.0",
+    "pedding": "*",
     "should": "~4.0.4",
-    "should-http": "0.0.1",
-    "supertest": "~0.14.0"
+    "should-http": "*",
+    "sqlite3": "*",
+    "supertest": "*"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
@@ -84,7 +81,7 @@
     "registry"
   ],
   "engines": {
-    "node": ">= 0.11.9"
+    "node": ">= 0.11.14"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

proxy/dist.js

@@ -1,86 +0,0 @@
-/**!
- * cnpmjs.org - proxy/dist.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var path = require('path');
-var multiline = require('multiline');
-var mysql = require('../common/mysql');
-
-var SAVE_FILE_SQL = multiline(function () {;/*
-  INSERT INTO
-    dist_file(gmt_create, gmt_modified, name, parent, date, size, url, sha1)
-  VALUES
-    (now(), now(), ?, ?, ?, ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    name=VALUES(name),
-    parent=VALUES(parent),
-    date=VALUES(date),
-    size=VALUES(size),
-    url=VALUES(url),
-    sha1=VALUES(sha1);
-*/});
-
-exports.savefile = function* (info) {
-  return yield mysql.query(SAVE_FILE_SQL, [
-    info.name, info.parent, info.date, info.size, info.url, info.sha1
-  ]);
-};
-
-var SAVE_DIR_SQL = multiline(function () {;/*
-  INSERT INTO
-    dist_dir(gmt_create, gmt_modified, name, parent, date)
-  VALUES
-    (now(), now(), ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    name=VALUES(name),
-    parent=VALUES(parent),
-    date=VALUES(date);
-*/});
-
-exports.savedir = function* (info) {
-  return yield mysql.query(SAVE_DIR_SQL, [
-    info.name, info.parent, info.date
-  ]);
-};
-
-var LIST_DIRS_SQL = multiline(function () {;/*
-  SELECT name, parent, date FROM dist_dir WHERE parent = ?;
-*/});
-var LIST_FILES_SQL = multiline(function () {;/*
-  SELECT name, parent, date, size, url, sha1
-  FROM dist_file WHERE parent = ?;
-*/});
-
-exports.listdir = function* (name) {
-  var rs = yield [
-    mysql.query(LIST_DIRS_SQL, [name]),
-    mysql.query(LIST_FILES_SQL, [name]),
-  ];
-  return rs[0].concat(rs[1]);
-};
-
-var GET_FILE_SQL = multiline(function () {;/*
-  SELECT name, parent, date, url, size, sha1 FROM dist_file WHERE parent = ? AND name = ?;
-*/});
-
-exports.getfile = function* (fullname) {
-  var name = path.basename(fullname);
-  var parent = path.dirname(fullname);
-  if (parent !== '/') {
-    parent += '/';
-  }
-  return yield mysql.queryOne(GET_FILE_SQL, [parent, name]);
-};

proxy/download.js

@@ -1,64 +0,0 @@
-/**!
- * cnpmjs.org - proxy/download.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var config = require('../config');
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-var PLUS_SQL = multiline(function () {;/*
-  INSERT INTO
-    download_total(gmt_create, gmt_modified, date, name, count)
-  VALUES
-    (now(), now(), ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    count=count + VALUES(count),
-    name=VALUES(name),
-    date=VALUES(date);
-*/});
-exports.plusTotal = function (data, callback) {
-  mysql.query(PLUS_SQL, [data.date, data.name, data.count], callback);
-};
-
-var SELECT_ONE_TOTAL_SQL = multiline(function () {;/*
-  SELECT
-    date, count
-  FROM
-    download_total
-  WHERE
-    date>=? AND date<=? AND name=?;
-*/});
-exports.getModuleTotal = function (name, start, end, callback) {
-  mysql.query(SELECT_ONE_TOTAL_SQL, [start, end, name], callback);
-};
-
-var SELECT_ALL_TOTAL_SQL = multiline(function () {;/*
-  SELECT
-    date, sum(count) AS count
-  FROM
-    download_total
-  WHERE
-    date>=? AND date<=?
-  GROUP BY
-    date;
-*/});
-exports.getTotal = function (start, end, callback) {
-  mysql.query(SELECT_ALL_TOTAL_SQL, [start, end], callback);
-};
-
-thunkify(exports);

proxy/module.js

@@ -1,768 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var utility = require('utility');
-var eventproxy = require('eventproxy');
-var config = require('../config');
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-var INSERT_MODULE_SQL = multiline(function () {;/*
-  INSERT INTO
-    module(gmt_create, gmt_modified, publish_time, author, name, version,
-      package, dist_tarball, dist_shasum, dist_size, description)
-  VALUES
-    (now(), now(), ?, ?, ?, ?, ?, ?, ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    gmt_modified=now(),
-    publish_time=VALUES(publish_time),
-    description=VALUES(description),
-    author=VALUES(author),
-    name=VALUES(name),
-    version=VALUES(version),
-    package=VALUES(package),
-    dist_tarball=VALUES(dist_tarball),
-    dist_shasum=VALUES(dist_shasum),
-    dist_size=VALUES(dist_size);
-*/});
-
-exports.add = function (mod, callback) {
-  var keywords = mod.package.keywords;
-  var pkg;
-  try {
-    pkg = stringifyPackage(mod.package);
-  } catch (e) {
-    return callback(e);
-  }
-  var description = mod.package && mod.package.description || '';
-
-  var dist = mod.package.dist || {};
-  dist.tarball = '';
-  dist.shasum = '';
-  dist.size = 0;
-  var publish_time = mod.publish_time || Date.now();
-  var values = [
-    publish_time, mod.author, mod.name, mod.version, pkg,
-    dist.tarball, dist.shasum, dist.size, description
-  ];
-  mysql.query(INSERT_MODULE_SQL, values, function (err, result) {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, {id: result.insertId, gmt_modified: new Date()});
-
-    if (typeof keywords === 'string') {
-      keywords = [keywords];
-    }
-
-    if (!Array.isArray(keywords)) {
-      return;
-    }
-
-    var words = [];
-    for (var i = 0; i < keywords.length; i++) {
-      var w = keywords[i];
-      if (typeof w === 'string') {
-        w = w.trim();
-        if (w) {
-          words.push(w);
-        }
-      }
-    }
-
-    if (words.length === 0) {
-      return;
-    }
-
-    // add keywords
-    exports.addKeywords(mod, description, words, utility.noop);
-  });
-};
-
-var GET_KEYWORD_SQL = multiline(function () {;/*
-  SELECT
-    keyword
-  FROM
-    module_keyword
-  WHERE
-    name = ?
-  ORDER BY
-    keyword;
-*/});
-
-exports.getKeywords = function (name, callback) {
-  mysql.query(GET_KEYWORD_SQL, [name], function (err, rows) {
-    var keywords = [];
-    if (rows && rows.length) {
-      keywords = rows.map(function (r) {
-        return r.keyword;
-      });
-    }
-    callback(err, keywords);
-  });
-};
-
-var ADD_KEYWORD_SQL = multiline(function () {;/*
-  INSERT INTO
-    module_keyword(gmt_create, keyword, name, description)
-  VALUES
-    (now(), ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    description=VALUES(description);
-*/});
-
-exports.addKeywords = function (name, description, keywords, callback) {
-  var sql = '';
-  var values = [];
-  for (var i = 0; i < keywords.length; i++) {
-    sql += ADD_KEYWORD_SQL;
-    values.push(keywords[i]);
-    values.push(name);
-    values.push(description);
-  }
-  mysql.query(sql, values, function (err, results) {
-    if (err) {
-      return callback(err);
-    }
-    var ids = [];
-    for (var i = 0; i < results.length; i++) {
-      var r = results[i];
-      if (r.insertId) {
-        ids.push(r.insertId);
-      }
-    }
-    callback(null, ids);
-  });
-};
-
-var UPDATE_DESC_SQL = multiline(function () {;/*
-  UPDATE
-    module
-  SET
-    description=?
-  WHERE
-    id=?;
-*/});
-exports.updateDescription = function (id, description, callback) {
-  mysql.query(UPDATE_DESC_SQL, [description, id], callback);
-};
-
-var UPDATE_DIST_SQL = 'UPDATE module SET ? WHERE id=?';
-exports.update = function (mod, callback) {
-  var pkg;
-  try {
-    pkg = stringifyPackage(mod.package);
-  } catch (e) {
-    return callback(e);
-  }
-  var dist = mod.package.dist;
-
-  var arg = {
-    publish_time: mod.publish_time,
-    version: mod.version,
-    package: pkg,
-    dist_tarball: dist.tarball,
-    dist_shasum: dist.shasum,
-    dist_size: dist.size
-  };
-
-  mysql.query(UPDATE_DIST_SQL,
-    [arg, mod.id],
-  function (err, result) {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, {id: mod.id, gmt_modified: new Date()});
-  });
-};
-
-function parseRow(row) {
-  if (row && row.package) {
-    try {
-      if (row.package.indexOf('%7B%22') === 0) {
-        // now store package will encodeURIComponent() after JSON.stringify
-        row.package = decodeURIComponent(row.package);
-      }
-      row.package = JSON.parse(row.package);
-    } catch (e) {
-      console.warn('parse package error: %s, id: %s version: %s, error: %s', row.name, row.id, row.version, e);
-    }
-  }
-}
-exports.parseRow = parseRow;
-
-function stringifyPackage(pkg) {
-  return encodeURIComponent(JSON.stringify(pkg));
-}
-
-
-var SELECT_MODULE_BY_ID_SQL = multiline(function () {;/*
-  SELECT
-    id, publish_time, gmt_create, gmt_modified, author, name,
-    version, description, package, dist_tarball, dist_shasum, dist_size
-  FROM
-    module
-  WHERE
-    id=?;
-*/});
-exports.getById = function (id, callback) {
-  id = Number(id);
-  mysql.queryOne(SELECT_MODULE_BY_ID_SQL, [id], function (err, row) {
-    if (err || !row) {
-      return callback(err, row);
-    }
-    try {
-      parseRow(row);
-    } catch (e) {
-      e.data = row;
-      return callback(e);
-    }
-    callback(null, row);
-  });
-};
-
-var SELECT_MODULE_SQL = multiline(function () {;/*
-  SELECT
-    id, publish_time, gmt_create, gmt_modified, author, name,
-    version, description, package, dist_tarball, dist_shasum, dist_size
-  FROM
-    module
-  WHERE
-    name=? AND version=?;
-*/});
-exports.get = function (name, version, callback) {
-  mysql.queryOne(SELECT_MODULE_SQL, [name, version], function (err, row) {
-    if (err || !row) {
-      return callback(err, row);
-    }
-    try {
-      parseRow(row);
-    } catch (e) {
-      e.data = row;
-      return callback(e);
-    }
-    callback(null, row);
-  });
-};
-
-var SELECT_MODULE_ID_SQL = multiline(function () {;/*
-  SELECT
-    id
-  FROM
-    module
-  WHERE
-    name=? AND version=?;
-*/});
-var INSERT_TAG_SQL = multiline(function () {;/*
-  INSERT INTO
-    tag(gmt_create, gmt_modified, name, tag, version, module_id)
-  VALUES
-    (now(), now(), ?, ?, ?, ?)
-  ON DUPLICATE KEY UPDATE
-    gmt_modified=now(),
-    module_id=VALUES(module_id),
-    name=VALUES(name),
-    tag=VALUES(tag),
-    version=VALUES(version);
-*/});
-exports.addTag = function (name, tag, version, callback) {
-  mysql.queryOne(SELECT_MODULE_ID_SQL, [name, version], function (err, row) {
-    if (err) {
-      return callback(err);
-    }
-    var module_id = row && row.id || 0;
-    mysql.query(INSERT_TAG_SQL, [name, tag, version, module_id], function (err, result) {
-      if (err) {
-        return callback(err);
-      }
-      callback(null, {id: result.insertId, gmt_modified: new Date(), module_id: module_id});
-    });
-  });
-};
-
-var SELECT_TAG_SQL = multiline(function () {;/*
-  SELECT
-    tag, version, gmt_modified, module_id
-  FROM
-    tag
-  WHERE
-    name=? AND tag=?;
-*/});
-exports.getByTag = function (name, tag, callback) {
-  mysql.queryOne(SELECT_TAG_SQL, [name, tag], function (err, row) {
-    if (err || !row) {
-      return callback(err, row);
-    }
-    exports.get(name, row.version, callback);
-  });
-};
-
-var DELETE_TAGS_SQL = multiline(function () {;/*
-  DELETE FROM
-    tag
-  WHERE
-    name=?;
-*/});
-exports.removeTags = function (name, callback) {
-  mysql.query(DELETE_TAGS_SQL, [name], callback);
-};
-
-var DELETE_TAGS_BY_IDS_SQL = multiline(function () {;/*
-  DELETE FROM
-    tag
-  WHERE
-    id IN (?);
-*/});
-exports.removeTagsByIds = function (ids, callback) {
-  mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
-};
-
-var SELECT_ALL_TAGS_SQL = multiline(function () {;/*
-  SELECT
-    id, tag, version, gmt_modified, module_id
-  FROM
-    tag
-  WHERE
-    name=?;
-*/});
-exports.listTags = function (name, callback) {
-  mysql.query(SELECT_ALL_TAGS_SQL, [name], callback);
-};
-
-var SELECT_LATEST_MODULE_SQL = multiline(function () {;/*
-  SELECT
-    id, publish_time, gmt_create, gmt_modified, author, name,
-    version, description, package, dist_tarball, dist_shasum, dist_size
-  FROM
-    module
-  WHERE
-    name=? AND version <> "next"
-  ORDER BY
-    publish_time DESC
-  LIMIT
-    1;
-*/});
-exports.getLatest = function (name, callback) {
-  exports.getByTag(name, 'latest', function (err, row) {
-    if (err || row) {
-      return callback(err, row);
-    }
-
-    // get latest order by id
-    mysql.queryOne(SELECT_LATEST_MODULE_SQL, [name], function (err, row) {
-      if (err || !row) {
-        return callback(err, row);
-      }
-      try {
-        parseRow(row);
-      } catch (e) {
-        e.data = row;
-        return callback(e);
-      }
-      callback(null, row);
-    });
-  });
-};
-
-var LIST_MODULE_SQL = multiline(function () {;/*
-  SELECT
-    id, publish_time, gmt_create, gmt_modified, author, name,
-    version, description, package, dist_tarball, dist_shasum, dist_size
-  FROM
-    module
-  WHERE
-    name=?
-  ORDER BY
-    id DESC;
-*/});
-exports.listByName = function (name, callback) {
-  mysql.query(LIST_MODULE_SQL, [name], function (err, rows) {
-    if (err) {
-      return callback(err);
-    }
-
-    rows = rows || [];
-    try {
-      for (var i = 0; i < rows.length; i++) {
-        parseRow(rows[i]);
-      }
-    } catch (e) {
-      err = e;
-      err.data = rows;
-    }
-    callback(err, rows);
-  });
-};
-
-var LIST_SINCE_SQL = multiline(function () {;/*
-  SELECT
-    distinct(name)
-  FROM
-    tag
-  WHERE
-    gmt_modified > ?;
-*/});
-exports.listSince = function (start, callback) {
-  mysql.query(LIST_SINCE_SQL, [new Date(start)], callback);
-};
-
-var LIST_ALL_NAME_SQL = multiline(function () {;/*
-  SELECT
-    distinct(name)
-  FROM
-    module;
-*/});
-exports.listAllNames = function (callback) {
-  mysql.query(LIST_ALL_NAME_SQL, [], callback);
-};
-
-var LIST_SHORT_SQL = multiline(function () {;/*
-  SELECT
-    distinct(name)
-  FROM
-    tag
-  ORDER BY
-    name;
-*/});
-exports.listShort = function (callback) {
-  mysql.query(LIST_SHORT_SQL, callback);
-};
-
-var LIST_ALL_MODULE_NAMES_SQL = multiline(function () {;/*
-  SELECT
-    distinct(name)
-  FROM
-    module
-  ORDER BY
-    name;
-*/});
-exports.listAllModuleNames = function (callback) {
-  mysql.query(LIST_ALL_MODULE_NAMES_SQL, callback);
-};
-
-var DELETE_MODULE_BY_NAME_SQL = multiline(function () {;/*
-  DELETE FROM
-    module
-  WHERE
-    name=?;
-*/});
-exports.removeByName = function (name, callback) {
-  mysql.query(DELETE_MODULE_BY_NAME_SQL, [name], callback);
-};
-
-var DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL = multiline(function () {;/*
-  DELETE FROM
-    module
-  WHERE
-    name=? AND version in(?);
-*/});
-exports.removeByNameAndVersions = function (name, versions, callback) {
-  mysql.query(DELETE_MODULE_BY_NAME_AND_VERSIONS_SQL, [name, versions], callback);
-};
-
-var SEARCH_MODULES_SQL = multiline(function () {;/*
-  SELECT
-    module_id
-  FROM
-    tag
-  WHERE
-    LOWER(name) LIKE LOWER(?) AND tag="latest"
-  ORDER BY
-    name
-  LIMIT
-    ?;
-*/});
-var SEARCH_MODULES_BY_KEYWORD_SQL = multiline(function () {;/*
-  SELECT
-    name, description
-  FROM
-    module_keyword
-  WHERE
-    keyword=?
-  ORDER BY
-    id DESC
-  LIMIT
-    ?;
-*/});
-var QUERY_MODULES_BY_ID_SQL = multiline(function () {;/*
-  SELECT
-    name, description
-  FROM
-    module
-  WHERE
-    id IN (?)
-  ORDER BY
-    name;
-*/});
-exports.search = function (word, options, callback) {
-  if (typeof options === 'function') {
-    callback = options;
-    options = null;
-  }
-  options = options || {};
-  var limit = options.limit || 100;
-  word = word.replace(/^%/, ''); //ignore prefix %
-  var ep = eventproxy.create();
-  ep.fail(callback);
-
-  // search flows:
-  // 1. prefix search by name
-  // 2. like search by name
-  // 3. keyword equal search
-  var ids = {};
-
-  mysql.query(SEARCH_MODULES_SQL, [word + '%', limit ], ep.done(function (rows) {
-    rows = rows || [];
-    if (rows.length > 0) {
-      for (var i = 0; i < rows.length; i++) {
-        ids[rows[i].module_id] = 1;
-      }
-    }
-    if (rows.length >= 20) {
-      return ep.emit('ids', Object.keys(ids));
-    }
-
-    mysql.query(SEARCH_MODULES_SQL, [ '%' + word + '%', limit ], ep.done('likeSearch'));
-  }));
-
-  mysql.query(SEARCH_MODULES_BY_KEYWORD_SQL, [ word, limit ], ep.done('keywordRows'));
-
-  ep.on('likeSearch', function (rows) {
-    rows = rows || [];
-    if (rows.length > 0) {
-      for (var i = 0; i < rows.length; i++) {
-        ids[rows[i].module_id] = 1;
-      }
-    }
-
-    ep.emit('ids', Object.keys(ids));
-  });
-
-  ep.all('ids', 'keywordRows', function (ids, keywordRows) {
-    keywordRows = keywordRows || [];
-    var data = {
-      keywordMatchs: keywordRows,
-      searchMatchs: []
-    };
-    if (ids.length === 0) {
-      return callback(null, data);
-    }
-
-    mysql.query(QUERY_MODULES_BY_ID_SQL, [ids], ep.done(function (modules) {
-      data.searchMatchs = modules;
-      callback(null, data);
-    }));
-  });
-};
-
-thunkify(exports);
-
-var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
-  SELECT
-    id, gmt_modified
-  FROM
-    module
-  WHERE
-    name=?
-  ORDER BY
-    gmt_modified DESC
-  LIMIT 1;
-*/});
-exports.getLastModified = function* (name) {
-  var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
-  return row && row.gmt_modified;
-};
-
-var UPDATE_LAST_MODIFIED_SQL = 'UPDATE module SET gmt_modified=now() WHERE id=?;';
-exports.updateLastModified = function* (name) {
-  var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
-  if (row) {
-    yield mysql.query(UPDATE_LAST_MODIFIED_SQL, [row.id]);
-  }
-};
-
-var DELETE_TAGS_BY_NAMES_SQL = 'DELETE FROM tag WHERE name=? AND tag IN (?);';
-exports.removeTagsByNames = function* (moduleName, tagNames) {
-  return yield mysql.query(DELETE_TAGS_BY_NAMES_SQL, [moduleName, tagNames]);
-};
-
-/**
- * forward compatbility for update from lower version cnpmjs.org
- * redirect @scope/name => name
- */
-exports.getAdaptName = function* (name) {
-  if (!config.scopes
-    || !config.scopes.length
-    || !config.adaptScope) {
-    return;
-  }
-
-  var tmp = name.split('/');
-  var scope = tmp[0];
-  name = tmp[1];
-
-  if (config.scopes.indexOf(scope) === -1) {
-    return;
-  }
-
-  var pkg = yield exports.getByTag(name, 'latest');
-  // only private module can adapt
-  if (pkg && pkg.package._publish_on_cnpm) {
-    return name;
-  }
-  return;
-};
-
-exports.listPrivates = function* () {
-  var scopes = config.scopes;
-  if (!scopes || !scopes.length) {
-    return [];
-  }
-  var privatePackages = config.privatePackages || [];
-
-  var args = [];
-  var sql = 'SELECT module_id AS id FROM tag WHERE tag="latest" AND (';
-  var wheres = [];
-
-  scopes.forEach(function (scope) {
-    wheres.push('name LIKE ?');
-    args.push(scope + '%');
-  });
-
-  if (privatePackages.length) {
-    wheres.push('name in (?)');
-    args.push(privatePackages);
-  }
-
-  sql = sql + wheres.join(' OR ') + ')';
-
-  var ids = yield mysql.query(sql, args);
-  ids = ids.map(function (row) {
-    return row.id;
-  });
-
-  if (!ids.length) {
-    return [];
-  }
-
-  return yield mysql.query(QUERY_MODULES_BY_ID_SQL, [ids]);
-};
-
-var LIST_BY_AUTH_SQLS = [];
-LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
-  SELECT
-    distinct(name) AS name
-  FROM
-    module
-  WHERE
-    author=?
-  ORDER BY
-    publish_time DESC
-  LIMIT
-    100;
-*/}));
-LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
-  SELECT
-    name
-  FROM
-    module_maintainer
-  WHERE
-    user = ?
-*/}));
-LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
-  SELECT
-    module_id
-  FROM
-    tag
-  WHERE
-    tag="latest" AND name IN (?);
-*/}));
-LIST_BY_AUTH_SQLS.push(multiline(function () {;/*
-  SELECT
-    name, description
-  FROM
-    module
-  WHERE
-    id IN (?)
-  ORDER BY
-    publish_time DESC;
-*/}));
-exports.listByAuthor = function* (author, callback) {
-  var names = yield [
-    mysql.query(LIST_BY_AUTH_SQLS[0], [author]),
-    mysql.query(LIST_BY_AUTH_SQLS[1], [author])
-  ];
-
-  names = names[0].concat(names[1]).map(function (n) {
-    return n.name;
-  }).sort();
-
-  if (!names.length) {
-    return [];
-  }
-
-  var ids = yield mysql.query(LIST_BY_AUTH_SQLS[2], [names]);
-  if (!ids.length) {
-    return [];
-  }
-
-  ids = ids.map(function (i) {
-    return i.module_id;
-  });
-  return yield mysql.query(LIST_BY_AUTH_SQLS[3], [ids]);
-};
-
-var UPDATE_PACKAGE_SQL = multiline(function () {;/*
-  UPDATE
-    module
-  SET
-    package=?
-  WHERE
-    id=?;
-*/});
-
-exports.updatePackage = function* (id, pkg) {
-  pkg = stringifyPackage(pkg);
-  return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
-};
-
-exports.updatePackageFields = function* (id, fields) {
-  var data = yield exports.getById(id);
-  if (!data) {
-    throw new Error('module#' + id + ' not exists');
-  }
-  data.package = data.package || {};
-  for (var k in fields) {
-    data.package[k] = fields[k];
-  }
-  return yield* exports.updatePackage(id, data.package);
-};
-
-exports.updateReadme = function* (id, readme) {
-  var data = yield exports.getById(id);
-  if (!data) {
-    throw new Error('module#' + id + ' not exists');
-  }
-  data.package = data.package || {};
-  data.package.readme = readme;
-  return yield* exports.updatePackage(id, data.package);
-};
-
-exports.getTag = function* (name, tag) {
-  return yield mysql.queryOne(SELECT_TAG_SQL, [name, tag]);
-};

proxy/module_deps.js

@@ -1,59 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module_deps.js
- *
- * Copyright(c) 2014
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-"use strict";
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-var LIST_DEPS_SQL = multiline(function () {;/*
-  SELECT
-    deps
-  FROM
-    module_deps
-  WHERE
-    name=?;
-*/});
-exports.list = function (name, callback) {
-  mysql.query(LIST_DEPS_SQL, [name], callback);
-};
-
-var INSERT_DEPS_SQL = multiline(function () {;/*
-  INSERT INTO
-    module_deps(gmt_create, name, deps)
-  VALUES
-    (now(), ?, ?);
-*/});
-exports.add = function (name, deps, callback) {
-  mysql.query(INSERT_DEPS_SQL, [name, deps], function (err, result) {
-    if (err && err.code === 'ER_DUP_ENTRY') {
-      err = null;
-    }
-    callback(err);
-  });
-};
-
-var DELETE_DEPS_SQL = multiline(function () {;/*
-  DELETE FROM
-    module_deps
-  WHERE
-    name=? AND deps=?;
-*/});
-exports.remove = function (name, deps, callback) {
-  mysql.query(DELETE_DEPS_SQL, [name, deps], callback);
-};
-
-thunkify(exports);

proxy/module_log.js

@@ -1,52 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module_log.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var mysql = require('../common/mysql');
-
-var INSERT_LOG_SQL = 'INSERT INTO module_log SET ?';
-exports.create = function (data, callback) {
-  var now = new Date();
-  var args = {
-    gmt_create: now,
-    gmt_modified: now,
-    name: data.name,
-    username: data.username,
-    log: ''
-  };
-  mysql.query(INSERT_LOG_SQL, [args], function (err, result) {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, {id: result.insertId, gmt_modified: new Date()});
-  });
-};
-
-var APPEND_SQL = 'UPDATE module_log SET log=CONCAT(log, ?), gmt_modified=now() WHERE id=?';
-exports.append = function (id, log, callback) {
-  log = '\n' + log;
-  mysql.query(APPEND_SQL, [log, id], function (err) {
-    callback(null, {id: id, gmt_modified: new Date()});
-  });
-};
-
-var SELECT_SQL = 'SELECT * FROM module_log WHERE id=?';
-exports.get = function (id, callback) {
-  mysql.queryOne(SELECT_SQL, [id], callback);
-};
-
-thunkify(exports);

proxy/module_maintainer.js

@@ -1,102 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module_maintainer.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:proxy:module_maintainer');
-var mysql = require('../common/mysql');
-var Module = require('./module');
-
-var GET_MAINTANINERS_SQL = 'SELECT user FROM module_maintainer WHERE name = ?;';
-
-exports.get = function* (name) {
-  var users = yield mysql.query(GET_MAINTANINERS_SQL, [name]);
-  return users.map(function (row) {
-    return row.user;
-  });
-};
-
-var ADD_SQL = 'INSERT INTO module_maintainer(name, user, gmt_create) \
-  VALUES (?, ?, now());';
-function* add(name, username) {
-  try {
-    yield mysql.query(ADD_SQL, [name, username]);
-  } catch (err) {
-    if (err.code !== 'ER_DUP_ENTRY') {
-      throw err;
-    }
-  }
-}
-
-var REMOVE_SQL = 'DELETE FROM module_maintainer WHERE name = ? AND user IN (?);';
-function* remove(name, usernames) {
-  return yield mysql.query(REMOVE_SQL, [name, usernames]);
-}
-
-var REMOVE_ALL_SQL = 'DELETE FROM module_maintainer WHERE name = ?';
-
-exports.removeAll = function* (name) {
-  return yield mysql.query(REMOVE_ALL_SQL, [name]);
-};
-
-exports.addMulti = function* (name, usernames) {
-  var tasks = [];
-  for (var i = 0; i < usernames.length; i++) {
-    tasks.push(add(name, usernames[i]));
-  }
-  return yield tasks;
-};
-
-exports.update = function* (name, maintainers) {
-  // maintainers should be [name1, name2, ...] format
-  // find out the exists maintainers then remove the deletes and add the left
-  if (maintainers.length === 0) {
-    return {
-      add: [],
-      remove: []
-    };
-  }
-  var exists = yield* exports.get(name);
-  var addUsers = maintainers;
-  var removeUsers = [];
-  if (exists.length > 0) {
-    for (var i = 0; i < exists.length; i++) {
-      var username = exists[i];
-      if (addUsers.indexOf(username) === -1) {
-        removeUsers.push(username);
-      }
-    }
-  }
-
-  yield* exports.addMulti(name, addUsers);
-  // make sure all add users success then remove users
-  if (removeUsers.length > 0) {
-    yield* remove(name, removeUsers);
-  }
-  debug('add %d users, remove %d users', addUsers.length, removeUsers.length);
-  return {
-    add: addUsers,
-    remove: removeUsers
-  };
-};
-
-exports.listByUsers = function* (users) {
-  var sql = 'SELECT name, user FROM module_maintainer WHERE user = ?;';
-  var args = users;
-  if (users.length > 1) {
-    sql = 'SELECT name, user FROM module_maintainer WHERE user in (?);';
-    args = [users];
-  }
-  return yield mysql.query(sql, args);
-};

proxy/module_star.js

@@ -1,74 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module_star.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-var ADD_SQL = multiline(function () {;/*
-  INSERT INTO
-    module_star(name, user, gmt_create)
-  VALUES
-    (?, ?, now());
-*/});
-exports.add = function *add(name, user) {
-  try {
-    yield mysql.query(ADD_SQL, [name, user]);
-  } catch (err) {
-    if (err.code !== 'ER_DUP_ENTRY') {
-      throw err;
-    }
-  }
-};
-
-var REMOVE_SQL = multiline(function () {;/*
-  DELETE FROM
-    module_star
-  WHERE
-    name = ? AND user = ?;
-*/});
-exports.remove = function *(name, user) {
-  return yield mysql.query(REMOVE_SQL, [name, user]);
-};
-
-var LIST_USERS_SQL = multiline(function () {;/*
-  SELECT
-    user
-  FROM
-    module_star
-  WHERE
-    name = ?;
-*/});
-exports.listUsers = function *(name) {
-  var rows = yield mysql.query(LIST_USERS_SQL, [name]);
-  return rows.map(function (r) {
-    return r.user;
-  });
-};
-
-var LIST_USER_MODULES_SQL = multiline(function () {;/*
-  SELECT
-    name
-  FROM
-    module_star
-  WHERE
-    user = ?;
-*/});
-exports.listUserModules = function *(user) {
-  return (yield mysql.query(LIST_USER_MODULES_SQL, [user])).map(function (r) {
-    return r.name;
-  });
-};

proxy/module_unpublished.js

@@ -1,44 +0,0 @@
-/**!
- * cnpmjs.org - proxy/module_unpublished.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var multiline = require('multiline');
-var mysql = require('../common/mysql');
-
-var SAVE_SQL = multiline(function () {;/*
-  INSERT INTO
-    module_unpublished(gmt_create, gmt_modified, name, package)
-  VALUES
-    (now(), now(), ?, ?)
-  ON DUPLICATE KEY UPDATE
-    gmt_modified=now(),
-    name=VALUES(name),
-    package=VALUES(package);
-*/});
-
-exports.add = function* (name, pkg) {
-  return yield mysql.query(SAVE_SQL, [name, JSON.stringify(pkg)]);
-};
-
-var GET_SQL = 'SELECT gmt_modified, name, package FROM module_unpublished WHERE name=?;';
-
-exports.get = function* (name) {
-  var row = yield mysql.queryOne(GET_SQL, [name]);
-  if (row) {
-    row.package = JSON.parse(row.package);
-  }
-  return row;
-};

proxy/total.js

@@ -1,170 +0,0 @@
-/**!
- * cnpmjs.org - proxy/total.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var eventproxy = require('eventproxy');
-var config = require('../config');
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-var DB_SIZE_SQL = multiline(function () {;/*
-  SELECT
-    TABLE_NAME AS name, data_length, index_length
-  FROM
-    information_schema.tables
-  WHERE
-    TABLE_SCHEMA = ?
-  GROUP BY
-    TABLE_NAME
-  ORDER BY
-    data_length DESC
-  LIMIT
-    0, 200;
-*/});
-
-var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM module;';
-var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM module;';
-var TOTAL_USER_SQL = 'SELECT count(name) AS count FROM user;';
-var TOTAL_INFO_SQL = 'SELECT * FROM total WHERE name="total";';
-
-exports.get = function (callback) {
-  var ep = eventproxy.create();
-  ep.fail(callback);
-
-  mysql.queryOne(TOTAL_MODULE_SQL, ep.done('module'));
-  mysql.queryOne(TOTAL_USER_SQL, ep.done('user'));
-  mysql.queryOne(TOTAL_VERSION_SQL, ep.done('version'));
-  mysql.query(DB_SIZE_SQL, [config.mysqlDatabase], ep.done('db_sizes'));
-  mysql.queryOne(TOTAL_INFO_SQL, ep.done('info'));
-
-  ep.all('db_sizes', 'module', 'version', 'user', 'info',
-  function (sizes, mc, vc, uc, info) {
-    info = info || {};
-    var total = {
-      data_tables: {},
-      disk_size: 0,
-      data_size: 0,
-      index_size: 0,
-      disk_format_version: 0,
-      committed_update_seq: 0,
-      update_seq: 0,
-      purge_seq: 0,
-      compact_running: false,
-      doc_count: mc.count,
-      doc_del_count: info.module_delete || 0,
-      doc_version_count: vc.count,
-      user_count: uc.count,
-      store_engine: 'mysql',
-      sync_status: info.sync_status,
-      need_sync_num: info.need_sync_num || 0,
-      success_sync_num: info.success_sync_num || 0,
-      fail_sync_num: info.fail_sync_num || 0,
-      left_sync_num: info.left_sync_num || 0,
-      last_sync_time: info.last_sync_time || 0,
-      last_exist_sync_time: info.last_exist_sync_time || 0,
-      last_sync_module: info.last_sync_module || '',
-    };
-
-    for (var i = 0; i < sizes.length; i++) {
-      var row = sizes[i];
-      total.data_tables[row.name] = {
-        data_size: row.data_length,
-        index_size: row.index_length,
-      };
-      total.data_size += row.data_length;
-      total.index_size += row.index_length;
-    }
-
-    total.disk_size = total.data_size + total.index_size;
-    callback(null, total);
-  });
-};
-
-var PLUS_DELETE_MODULE_SQL = multiline(function () {;/*
-  UPDATE
-    total
-  SET
-    module_delete=module_delete+1
-  WHERE
-    name="total";
-*/});
-exports.plusDeleteModule = function (callback) {
-  mysql.query(PLUS_DELETE_MODULE_SQL, callback);
-};
-
-exports.getTotalInfo = function (callback) {
-  mysql.queryOne(TOTAL_INFO_SQL, callback);
-};
-
-var SET_LAST_SYNC_TIME_SQL = multiline(function () {;/*
-  UPDATE
-    total
-  SET
-    last_sync_time=?
-  WHERE
-    name="total";
-*/});
-exports.setLastSyncTime = function (time, callback) {
-  mysql.query(SET_LAST_SYNC_TIME_SQL, Number(time), callback);
-};
-
-var SET_LAST_EXIST_SYNC_TIME_SQL = multiline(function () {;/*
-  UPDATE
-    total
-  SET
-    last_exist_sync_time=?
-  WHERE
-    name="total";
-*/});
-exports.setLastExistSyncTime = function (time, callback) {
-  mysql.query(SET_LAST_EXIST_SYNC_TIME_SQL, Number(time), callback);
-};
-
-var UPDATE_SYNC_STATUS_SQL = multiline(function () {;/*
-  UPDATE
-    total
-  SET
-    sync_status=?
-  WHERE
-    name="total";
-*/});
-exports.updateSyncStatus = function (status, callback) {
-  mysql.query(UPDATE_SYNC_STATUS_SQL, [status], callback);
-};
-
-var UPDATE_SYNC_NUM_SQL = multiline(function () {;/*
-  UPDATE
-    total
-  SET
-    ?
-  WHERE
-    name="total";
-*/});
-exports.updateSyncNum = function (params, callback) {
-  var arg = {
-    sync_status: params.syncStatus,
-    need_sync_num: params.need || 0,
-    success_sync_num: params.success || 0,
-    fail_sync_num: params.fail || 0,
-    left_sync_num: params.left || 0,
-    last_sync_module: params.lastSyncModule
-  };
-
-  mysql.query(UPDATE_SYNC_NUM_SQL, [arg], callback);
-};
-
-thunkify(exports);

proxy/user.js

@@ -1,196 +0,0 @@
-/**!
- * cnpmjs.org - proxy/user.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var thunkify = require('thunkify-wrap');
-var utility = require('utility');
-var config = require('../config');
-var mysql = require('../common/mysql');
-var multiline = require('multiline');
-
-function passwordSha(password, salt) {
-  return utility.sha1(password + salt);
-}
-
-var INSERT_USER_SQL = 'INSERT INTO user SET ?';
-exports.add = function (user, callback) {
-  var roles = user.roles || [];
-  try {
-    roles = JSON.stringify(roles);
-  } catch (e) {
-    roles = '[]';
-  }
-  var rev = '1-' + utility.md5(JSON.stringify(user));
-
-  var now = new Date();
-
-  var arg = {
-    rev: rev,
-    name: user.name,
-    email: user.email,
-    salt: user.salt,
-    password_sha: user.password_sha,
-    ip: user.ip,
-    roles: roles,
-    gmt_create: now,
-    gmt_modified: now
-  };
-
-  mysql.query(INSERT_USER_SQL, [arg], function (err) {
-    callback(err, {rev: rev});
-  });
-};
-
-var UPDATE_USER_SQL = multiline(function () {;/*
-  UPDATE
-    user
-  SET
-    ?
-  WHERE
-    name=? AND rev=?;
-*/});
-exports.update = function (user, callback) {
-  var rev = user.rev || user._rev;
-  var revNo = Number(rev.split('-', 1));
-  if (!revNo) {
-    var err = new Error(rev + ' format error');
-    err.name = 'RevFormatError';
-    err.data = {user: user};
-    return callback(err);
-  }
-  revNo++;
-  var newRev = revNo + '-' + utility.md5(JSON.stringify(user));
-  var roles = user.roles || [];
-  try {
-    roles = JSON.stringify(roles);
-  } catch (e) {
-    roles = '[]';
-  }
-
-  var arg = {
-    rev: newRev,
-    email: user.email,
-    salt: user.salt,
-    password_sha: user.password_sha,
-    ip: user.ip,
-    roles: roles,
-    gmt_modified: new Date()
-  };
-
-  mysql.query(UPDATE_USER_SQL, [arg, user.name, rev], function (err, data) {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, {rev: newRev, result: data});
-  });
-};
-
-thunkify(exports);
-
-exports.passwordSha = passwordSha;
-
-var SELECT_USER_SQL = 'SELECT \
-    id, rev, name, email, salt, password_sha, ip, \
-    roles, json, npm_user, gmt_create, gmt_modified \
-  FROM \
-    user \
-  WHERE \
-    name=?;';
-exports.get = function* (name) {
-  var row = yield mysql.queryOne(SELECT_USER_SQL, [name]);
-  if (row) {
-    try {
-      row.roles = row.roles ? JSON.parse(row.roles) : [];
-    } catch (e) {
-      row.roles = [];
-    }
-    try {
-      row.json = row.json ? JSON.parse(row.json) : null;
-    } catch (e) {
-      row.json = null;
-    }
-  }
-  return row;
-};
-
-exports.auth = function* (name, password) {
-  var row = yield* exports.get(name);
-  if (row) {
-    var sha = passwordSha(password, row.salt);
-    if (row.password_sha !== sha) {
-      row = null;
-    }
-  }
-  return row;
-};
-
-exports.saveNpmUser = function* (user) {
-  var sql = 'SELECT id, json FROM user WHERE name=?;';
-  var row = yield mysql.queryOne(sql, [user.name]);
-  if (!row) {
-    sql = 'INSERT INTO user(npm_user, json, rev, name, email, salt, password_sha, ip, gmt_create, gmt_modified) \
-      VALUES(1, ?, ?, ?, ?, "0", "0", "0", now(), now());';
-    yield mysql.query(sql, [JSON.stringify(user), user._rev, user.name, user.email]);
-  } else {
-    sql = 'UPDATE user SET json=?, rev=? WHERE id=?;';
-    yield mysql.query(sql, [JSON.stringify(user), user._rev, row.id]);
-  }
-};
-
-exports.saveCustomUser = function* (data) {
-  var sql = 'SELECT id, json FROM user WHERE name=?;';
-  var row = yield mysql.queryOne(sql, [data.user.login]);
-  var salt = data.salt || '0';
-  var password_sha = data.password_sha || '0';
-  var ip = data.ip || '0';
-  var rev = rev || '1-' + data.user.login;
-  var json = JSON.stringify(data.user);
-  if (!row) {
-    sql = 'INSERT INTO user(npm_user, json, rev, name, email, salt, password_sha, ip, gmt_create, gmt_modified) \
-      VALUES(2, ?, ?, ?, ?, ?, ?, ?, now(), now());';
-    yield mysql.query(sql, [
-      json, rev, data.user.login, data.user.email,
-      salt, password_sha, ip
-    ]);
-  } else {
-    sql = 'UPDATE user SET json=?, rev=?, salt=?, password_sha=?, ip=? WHERE id=?;';
-    yield mysql.query(sql, [
-      json, rev,
-      salt, password_sha, ip,
-      row.id
-    ]);
-  }
-};
-
-var LIST_BY_NAMES_SQL = 'SELECT \
-    id, name, email, json \
-  FROM \
-    user \
-  WHERE \
-    name in (?);';
-exports.listByNames = function* (names) {
-  if (names.length === 0) {
-    return [];
-  }
-  return yield mysql.query(LIST_BY_NAMES_SQL, [names]);
-};
-
-var SEARCH_SQL = 'SELECT id, name, email, json FROM user WHERE name LIKE ? LIMIT ?;';
-exports.search = function* (query, options) {
-  var limit = options.limit;
-  query = query + '%';
-  return yield mysql.query(SEARCH_SQL, [query, limit]);
-};

public/js/readme.js

@@ -0,0 +1,49 @@
+$(function () {
+  function humanize(n, options) {
+    options = options || {};
+    var d = options.delimiter || ',';
+    var s = options.separator || '.';
+    n = n.toString().split('.');
+    n[0] = n[0].replace(/(\d)(?=(\d\d\d)+(?!\d))/g, '$1' + d);
+    return n.join(s);
+  }
+
+  $.getJSON('/total', function (data) {
+    $('#total-packages').html(humanize(data.doc_count));
+    $('#total-versions').html(humanize(data.doc_version_count));
+    $('#total-deletes').html(humanize(data.doc_del_count));
+
+    var downloads = $('table.downloads');
+    downloads.find('td.count:eq(3)').html(humanize(data.download.today));
+    downloads.find('td.count:eq(4)').html(humanize(data.download.thisweek));
+    downloads.find('td.count:eq(5)').html(humanize(data.download.thismonth));
+    downloads.find('td.count:eq(6)').html(humanize(data.download.lastday));
+    downloads.find('td.count:eq(7)').html(humanize(data.download.lastweek));
+    downloads.find('td.count:eq(8)').html(humanize(data.download.lastmonth));
+
+    $('#node-version').html(data.node_version || 'v0.10.22');
+    $('#app-version').html(data.app_version || '0.0.0');
+
+    if (data.sync_model === 'all') {
+      $('#sync-model').html('This registry will sync all packages from official registry.');
+      $('#last-sync-time').html(new Date(data.last_sync_time));
+    } else if (data.sync_model === 'exist') {
+      $('#sync-model').html('This registry will only update exist packages from official registry.');
+      $('#last-sync-time').html(new Date(data.last_exist_sync_time));
+    }
+
+    $('#need-sync').html(data.need_sync_num);
+    $('#success-sync').html(data.success_sync_num);
+    $('#fail-sync').html(data.fail_sync_num);
+    $('#left-sync').html(data.left_sync_num);
+    $('#percent-sync').html(Math.floor(data.success_sync_num / data.need_sync_num * 100));
+    $('#last-success-name').html('<a target="_blank" href="/package/' + data.last_sync_module + '">' +
+      data.last_sync_module + '</a>');
+
+    if (!data.sync_status) {
+      $('.syncing').html('');
+    }
+
+    $('.sync').show();
+  });
+});

routes/registry.js

@@ -19,9 +19,25 @@ var limit = require('../middleware/limit');
 var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
-var total = require('../controllers/total');
-var mod = require('../controllers/registry/module');
-var user = require('../controllers/registry/user');
+var editable = require('../middleware/editable');
+
+var showTotal = require('../controllers/total');
+
+var listAll = require('../controllers/registry/package/list_all');
+var listSince = require('../controllers/registry/package/list_since');
+var listAllVersions = require('../controllers/registry/package/list');
+var getOneVersion = require('../controllers/registry/package/show');
+var savePackage = require('../controllers/registry/package/save');
+var tag = require('../controllers/registry/package/tag');
+var removePackage = require('../controllers/registry/package/remove');
+var removeOneVersion = require('../controllers/registry/package/remove_version');
+var updatePackage = require('../controllers/registry/package/update');
+var downloadPackage = require('../controllers/registry/package/download');
+
+var addUser = require('../controllers/registry/user/add');
+var showUser = require('../controllers/registry/user/show');
+var updateUser = require('../controllers/registry/user/update');
+
 var sync = require('../controllers/sync');
 var userPackage = require('../controllers/registry/user_package');
 
@@ -34,56 +50,60 @@ function routes(app) {
     }
   }
 
-  app.get('/', jsonp, total.show);
+  app.get('/', jsonp, showTotal);
 
-  //before /:name/:version
-  //get all modules, for npm search
-  app.get('/-/all', mod.listAllModules);
-  app.get('/-/all/since', mod.listAllModulesSince);
-  //get all module names, for auto completion
-  app.get('/-/short', mod.listAllModuleNames);
+  // before /:name/:version
+  // get all modules, for npm search
+  app.get('/-/all', listAll);
+  app.get('/-/all/since', listSince);
+  // get all module names, for auto completion
+  // app.get('/-/short', mod.listAllModuleNames);
 
   // module
   // scope package: params: [$name]
-  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)$/, syncByInstall, mod.show);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)$/, syncByInstall, listAllVersions);
   // scope package: params: [$name, $version]
-  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\.\-]+)$/, syncByInstall, mod.get);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\.\-]+)$/, syncByInstall, getOneVersion);
+
+  app.get('/:name', syncByInstall, listAllVersions);
+  app.get('/:name/:version', syncByInstall, getOneVersion);
 
-  app.get('/:name', syncByInstall, mod.show);
-  app.get('/:name/:version', syncByInstall, mod.get);
   // try to add module
-  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)$/, login, publishable, mod.addPackageAndDist);
-  app.put('/:name', login, publishable, mod.addPackageAndDist);
+  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)$/, login, publishable, savePackage);
+  app.put('/:name', login, publishable, savePackage);
 
   // sync from source npm
   app.put('/:name/sync', sync.sync);
   app.get('/:name/sync/log/:id', sync.getSyncLog);
 
-  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\-\.]+)$/, login, mod.updateTag);
-  app.put('/:name/:tag', login, mod.updateTag);
+  // add tag
+  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\-\.]+)$/, login, editable, tag);
+  app.put('/:name/:tag', login, editable, tag);
 
   // need limit by ip
-  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/download\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, mod.download);
-  app.get('/:name/download/:filename', limit, mod.download);
-  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, mod.download);
-  app.get('/:name/-/:filename', limit, mod.download);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/download\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, downloadPackage);
+  app.get('/:name/download/:filename', limit, downloadPackage);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, downloadPackage);
+  app.get('/:name/-/:filename', limit, downloadPackage);
 
-  // delete tarball
+  // delete tarball and remove one version
   app.delete(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/download\/(@[\w\-\.]+\/[\w\-\.]+)\/\-rev\/([\w\-\.]+)$/,
-    login, publishable, mod.removeTar);
-  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, mod.removeTar);
+    login, publishable, editable, removeOneVersion);
+  app.delete('/:name/download/:filename/-rev/:rev', login, publishable, editable, removeOneVersion);
 
   // update module, unpublish will PUT this
-  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-rev\/([\w\-\.]+)$/, login, publishable, mod.updateOrRemove);
-  app.delete(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-rev\/([\w\-\.]+)$/, login, publishable, mod.removeAll);
-  app.put('/:name/-rev/:rev', login, publishable, mod.updateOrRemove);
-  app.delete('/:name/-rev/:rev', login, publishable, mod.removeAll);
+  app.put(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-rev\/([\w\-\.]+)$/, login, publishable, editable, updatePackage);
+  app.put('/:name/-rev/:rev', login, publishable, editable, updatePackage);
+
+  // remove all versions
+  app.delete(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-rev\/([\w\-\.]+)$/, login, publishable, editable, removePackage);
+  app.delete('/:name/-rev/:rev', login, publishable, editable, removePackage);
 
   // try to create a new user
   // https://registry.npmjs.org/-/user/org.couchdb.user:fengmk2
-  app.put('/-/user/org.couchdb.user::name', user.add);
-  app.get('/-/user/org.couchdb.user::name', user.show);
-  app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, user.update);
+  app.put('/-/user/org.couchdb.user::name', addUser);
+  app.get('/-/user/org.couchdb.user::name', showUser);
+  app.put('/-/user/org.couchdb.user::name/-rev/:rev', login, updateUser);
 
   // list all packages of user
   app.get('/-/by-user/:user', userPackage.list);

routes/web.js

@@ -15,41 +15,42 @@
  * Module dependencies.
  */
 
-var pkg = require('../controllers/web/package');
-var user = require('../controllers/web/user');
+var showPackage = require('../controllers/web/package/show');
+var searchPackage = require('../controllers/web/package/search');
+var searchRange = require('../controllers/web/package/search_range');
+var listPrivates = require('../controllers/web/package/list_privates');
+var showSync = require('../controllers/web/package/show_sync');
+var showUser = require('../controllers/web/user/show');
+
 var sync = require('../controllers/sync');
-var total = require('../controllers/total');
+var showTotal = require('../controllers/total');
 var dist = require('../controllers/web/dist');
 var badge = require('../controllers/web/badge');
 
 function routes(app) {
-  app.get('/total', total.show);
+  app.get('/total', showTotal);
 
   // scope package without version
-  app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)$/, pkg.display);
+  app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)$/, showPackage);
   // scope package with version
-  app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\d\.]+)$/, pkg.display);
-  app.get('/package/:name', pkg.display);
-  app.get('/package/:name/:version', pkg.display);
+  app.get(/\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/([\w\d\.]+)$/, showPackage);
+  app.get('/package/:name', showPackage);
+  app.get('/package/:name/:version', showPackage);
 
-  app.get('/privates', pkg.listPrivates);
+  app.get('/privates', listPrivates);
 
-  app.get(/\/browse\/keyword\/(@[\w\-\.]+\/[\w\-\.]+)$/, pkg.search);
-  app.get('/browse/keyword/:word', pkg.search);
+  app.get(/\/browse\/keyword\/(@[\w\-\.]+\/[\w\-\.]+)$/, searchPackage);
+  app.get('/browse/keyword/:word', searchPackage);
 
-  app.get('/~:name', user.display);
-
-  app.get('/sync/:name', pkg.displaySync);
+  app.get('/~:name', showUser);
 
+  app.get('/sync/:name', showSync);
+  app.get('/sync', showSync);
   app.put('/sync/:name', sync.sync);
 
-  // params: [$name, $id]
-  app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
 
-  app.get('/sync', pkg.displaySync);
-
-  app.get('/_list/search/search', pkg.rangeSearch);
+  app.get('/_list/search/search', searchRange);
 
   app.get(/^\/dist(\/.*)?/, dist.list);
 

servers/registry.js

@@ -55,7 +55,8 @@ routes(app);
  */
 
 app.on('error', function (err, ctx) {
-  // console.log(err.stack)
+  console.log(err);
+  console.log(err.stack);
   err.url = err.url || ctx.request.url;
   logger.error(err);
 });

servers/web.js

@@ -77,7 +77,10 @@ app.use(markdown({
   layout: layoutFile,
   titleHolder: '<%- locals.title %>',
   bodyHolder: '<%- locals.body %>',
-  indexName: '_readme'
+  indexName: '_readme',
+  remarkableOptions: {
+    html: true
+  }
 }));
 
 var locals = {

services/common.js

@@ -0,0 +1,29 @@
+/**!
+ * cnpmjs.org - services/common.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+
+config.privatePackages = config.privatePackages || [];
+
+exports.isPrivatePackage = function* (name) {
+  if (name[0] === '@') {
+    return true;
+  }
+  if (config.privatePackages.indexOf(name) >= 0) {
+    return true;
+  }
+  return false;
+};

services/default_user_service.js

@@ -15,7 +15,8 @@
  */
 
 var gravatar = require('gravatar');
-var User = require('../proxy/user');
+// var User = require('../proxy/user');
+var User = require('../models').User;
 var isAdmin = require('../lib/common').isAdmin;
 var config = require('../config');
 
@@ -42,7 +43,7 @@ function convertToUser(row) {
     avatar_url: '',
     im_url: '',
     site_admin: isAdmin(row.name),
-    scopes: config.scopes
+    scopes: config.scopes,
   };
   if (row.json) {
     var data = row.json;
@@ -95,7 +96,7 @@ proto.auth = function* (login, password) {
  * @return {User}
  */
 proto.get = function* (login) {
-  var row = yield* User.get(login);
+  var row = yield User.findByName(login);
   if (!row) {
     return null;
   }

services/dist.js

@@ -0,0 +1,91 @@
+/**!
+ * cnpmjs.org - services/dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+/* jshint -W032 */
+
+/**
+ * Module dependencies.
+ */
+
+var path = require('path');
+var models = require('../models');
+var File = models.DistFile;
+var Dir = models.DistDir;
+
+exports.savefile = function* (info) {
+  var row = yield File.find({
+    where: {
+      parent: info.parent,
+      name: info.name
+    }
+  });
+  if (!row) {
+    row = File.build({
+      parent: info.parent,
+      name: info.name
+    });
+  }
+  row.date = info.date;
+  row.size = info.size;
+  row.url = info.url;
+  row.sha1 = info.sha1;
+  return yield row.save();
+};
+
+exports.savedir = function* (info) {
+  var row = yield Dir.find({
+    where: {
+      parent: info.parent,
+      name: info.name
+    }
+  });
+  if (!row) {
+    row = Dir.build({
+      parent: info.parent,
+      name: info.name
+    });
+  }
+  row.date = info.date;
+  return yield row.save();
+};
+
+exports.listdir = function* (name) {
+  var rs = yield [
+    File.findAll({
+      attributrs: ['name', 'parent', 'date'],
+      where: {
+        parent: name
+      }
+    }),
+    Dir.findAll({
+      attributrs: ['name', 'parent', 'date', 'size', 'url', 'sha1'],
+      where: {
+        parent: name
+      }
+    })
+  ];
+  return rs[0].concat(rs[1]);
+};
+
+exports.getfile = function* (fullname) {
+  var name = path.basename(fullname);
+  var parent = path.dirname(fullname);
+  if (parent !== '/') {
+    parent += '/';
+  }
+  return yield File.find({
+    attributrs: ['name', 'parent', 'date', 'size', 'url', 'sha1'],
+    where: {
+      parent: parent,
+      name: name
+    }
+  });
+};

services/download_total.js

@@ -0,0 +1,54 @@
+/**!
+ * cnpmjs.org - services/download_total.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var models = require('../models');
+var DownloadTotal = models.DownloadTotal;
+
+exports.getModuleTotal = function* (name, start, end) {
+  return yield DownloadTotal.findAll({
+    where: {
+      date: {
+        gte: start,
+        lte: end
+      },
+      name: name
+    }
+  });
+};
+
+exports.plusModuleTotal = function* (data) {
+  var row = yield DownloadTotal.find({
+    where: {
+      date: data.date,
+      name: data.name
+    }
+  });
+  if (!row) {
+    row = DownloadTotal.build({
+      date: data.date,
+      name: data.name
+    });
+  }
+  row.count += data.count;
+  return yield row.save();
+};
+
+
+exports.getTotal = function* (start, end) {
+  var sql = 'SELECT date, sum(count) AS count FROM download_total \
+    WHERE date>=? AND date<=? GROUP BY date;';
+  return yield models.query(sql, [start, end]);
+};

services/module_log.js

@@ -0,0 +1,54 @@
+/**!
+ * cnpmjs.org - services/module_log.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var models = require('../models');
+var ModuleLog = models.ModuleLog;
+
+exports.create = function* (data) {
+  var row = ModuleLog.build({
+    name: data.name,
+    username: data.username || 'anonymous',
+    log: ''
+  });
+  return yield row.save();
+};
+
+var ONE_MB = 1024 * 1024;
+
+exports.append = function* (id, log) {
+  if (!log) {
+    return null;
+  }
+
+  var row = yield* exports.get(id);
+  if (!row) {
+    return null;
+  }
+
+  if (row.log) {
+    row.log += '\n' + log;
+  } else {
+    row.log = log;
+  }
+  if (row.log.length >= ONE_MB) {
+    row.log = '...\n' + row.log.substring(ONE_MB / 2);
+  }
+  return yield row.save(['log']);
+};
+
+exports.get = function* (id) {
+  return yield ModuleLog.find(Number(id));
+};

services/npm.js

@@ -1,11 +1,12 @@
 /**!
- * cnpmjs.org - proxy/npm.js
+ * cnpmjs.org - services/npm.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   dead_horse <dead_horse@qq.com> (http://deadhorse.me)
  */
 
 'use strict';
@@ -30,7 +31,7 @@ function* request(url, options) {
   url = registry + url;
   var r;
   try {
-    r = yield urllib.request(url, options);
+    r = yield urllib.requestThunk(url, options);
   } catch (err) {
     var statusCode = err.status || -1;
     var data = err.data || '[empty]';
@@ -45,9 +46,9 @@ function* request(url, options) {
 
 exports.request = request;
 
-exports.getUser = function *(name) {
+exports.getUser = function* (name) {
   var url = '/-/user/org.couchdb.user:' + name;
-  var r = yield *request(url);
+  var r = yield* request(url);
   var data = r.data;
   if (data && !data.name) {
     data = null;
@@ -55,8 +56,8 @@ exports.getUser = function *(name) {
   return data;
 };
 
-exports.get = function *(name) {
-  var r = yield *request('/' + name);
+exports.get = function* (name) {
+  var r = yield* request('/' + name);
   var data = r.data;
   if (r.status === 404) {
     data = null;
@@ -64,17 +65,39 @@ exports.get = function *(name) {
   return data;
 };
 
-exports.getAllSince = function *(startkey) {
-  var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, {
-    timeout: 300000
+exports.getAllSince = function* (startkey, timeout) {
+  var r = yield* request('/-/all/since?stale=update_after&startkey=' + startkey, {
+    timeout: timeout || 300000
   });
   return r.data;
 };
 
-exports.getShort = function *() {
-  var r = yield *request('/-/short', {
-    timeout: 300000,
-    registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now.
+exports.getShort = function* (timeout) {
+  var r = yield* request('/-/short', {
+    timeout: timeout || 300000,
+    // registry.npmjs.org/-/short is 404 now therefore have a fallback
+    registry: config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'http://r.cnpmjs.org',
   });
   return r.data;
 };
+
+exports.getPopular = function* (top, timeout) {
+  var r = yield* request('/-/_view/dependedUpon?group_level=1', {
+    registry: config.officialNpmRegistry,
+    timeout: timeout || 60000
+  });
+  if (!r.data || !r.data.rows || !r.data.rows.length) {
+    return [];
+  }
+
+  return r.data.rows.sort(function (a, b) {
+    return b.value - a.value;
+  })
+  .slice(0, top)
+  .map(function (r) {
+    return r.key && r.key[0];
+  })
+  .filter(function (r) {
+    return r;
+  });
+};

services/package.js

@@ -14,16 +14,494 @@
  * Module dependencies.
  */
 
-var Module = require('../proxy/module');
-var ModuleMaintainer = require('../proxy/module_maintainer');
-var User = require('../proxy/user');
+var models = require('../models');
+var common = require('./common');
+var Tag = models.Tag;
+var User = models.User;
+var Module = models.Module;
+var ModuleStar = models.ModuleStar;
+var ModuleKeyword = models.ModuleKeyword;
+var PrivateModuleMaintainer = models.ModuleMaintainer;
+var ModuleDependency = models.ModuleDependency;
+var ModuleUnpublished = models.ModuleUnpublished;
+var NpmModuleMaintainer = models.NpmModuleMaintainer;
+
+// module
+
+// module:read
+function parseRow(row) {
+  if (row && row.package) {
+    try {
+      if (row.package.indexOf('%7B%22') === 0) {
+        // now store package will encodeURIComponent() after JSON.stringify
+        row.package = decodeURIComponent(row.package);
+      }
+      row.package = JSON.parse(row.package);
+    } catch (e) {
+      console.warn('parse package error: %s, id: %s version: %s, error: %s', row.name, row.id, row.version, e);
+    }
+  }
+}
+exports.parseRow = parseRow;
+
+function stringifyPackage(pkg) {
+  return encodeURIComponent(JSON.stringify(pkg));
+}
+
+exports.getModuleById = function* (id) {
+  var row = yield Module.find(Number(id));
+  parseRow(row);
+  return row;
+};
+
+exports.getModule = function* (name, version) {
+  var row = yield* Module.findByNameAndVersion(name, version);
+  parseRow(row);
+  return row;
+};
+
+exports.getModuleByTag = function* (name, tag) {
+  var tag = yield* Tag.findByNameAndTag(name, tag);
+  if (!tag) {
+    return null;
+  }
+  return yield* exports.getModule(tag.name, tag.version);
+};
+
+exports.getLatestModule = function* (name) {
+  return yield* exports.getModuleByTag(name, 'latest');
+};
+
+// module:list
+
+exports.listPrivateModulesByScope = function* (scope) {
+  var tags = yield Tag.findAll({
+    where: {
+      tag: 'latest',
+      name: {
+        like: scope + '/%'
+      }
+    }
+  });
+
+  if (tags.length === 0) {
+    return [];
+  }
+
+  var ids = tags.map(function (tag) {
+    return tag.module_id;
+  });
+
+  return yield Module.findAll({
+    where: {
+      id: ids
+    }
+  });
+};
+
+exports.listModules = function* (names) {
+  if (names.length === 0) {
+    return [];
+  }
+
+  // fetch latest module tags
+  var tags = yield Tag.findAll({
+    where: {
+      name: names,
+      tag: 'latest'
+    }
+  });
+  if (tags.length === 0) {
+    return [];
+  }
+
+  var ids = tags.map(function (tag) {
+    return tag.module_id;
+  });
+
+  var rows = yield Module.findAll({
+    where: {
+      id: ids
+    },
+    attributes: [
+      'name', 'description'
+    ]
+  });
+  return rows;
+};
+
+exports.listModulesByUser = function* (username) {
+  var names = yield* exports.listModuleNamesByUser(username);
+  return yield* exports.listModules(names);
+};
+
+exports.listModuleNamesByUser = function* (username) {
+  var sql = 'SELECT distinct(name) AS name FROM module WHERE author=?;';
+  var rows = yield* models.query(sql, [username]);
+  var map = {};
+  var names = rows.map(function (r) {
+    return r.name;
+  });
+
+  // find from npm module maintainer table
+  var moduleNames = yield* NpmModuleMaintainer.listModuleNamesByUser(username);
+  moduleNames.forEach(function (name) {
+    if (!map[name]) {
+      names.push(name);
+    }
+  });
+
+  // find from private module maintainer table
+  moduleNames = yield* PrivateModuleMaintainer.listModuleNamesByUser(username);
+  moduleNames.forEach(function (name) {
+    if (!map[name]) {
+      names.push(name);
+    }
+  });
+  return names;
+};
+
+exports.listPublicModulesByUser = function* (username) {
+  var names = yield* exports.listPublicModuleNamesByUser(username);
+  return yield* exports.listModules(names);
+};
+
+// return user all public package names
+exports.listPublicModuleNamesByUser = function* (username) {
+  var sql = 'SELECT distinct(name) AS name FROM module WHERE author=?;';
+  var rows = yield* models.query(sql, [username]);
+  var map = {};
+  var names = rows.map(function (r) {
+    return r.name;
+  }).filter(function (name) {
+    var matched = name[0] !== '@';
+    if (matched) {
+      map[name] = 1;
+    }
+    return matched;
+  });
+
+  // find from npm module maintainer table
+  var moduleNames = yield* NpmModuleMaintainer.listModuleNamesByUser(username);
+  moduleNames.forEach(function (name) {
+    if (!map[name]) {
+      names.push(name);
+    }
+  });
+  return names;
+};
+
+// start must be a date or timestamp
+exports.listPublicModuleNamesSince = function* (start) {
+  if (!(start instanceof Date)) {
+    start = new Date(Number(start));
+  }
+  var rows = yield Tag.findAll({
+    attributes: ['name'],
+    where: {
+      gmt_modified: {
+        gt: start
+      }
+    },
+  });
+  var names = {};
+  for (var i = 0; i < rows.length; i++) {
+    names[rows[i].name] = 1;
+  }
+  return Object.keys(names);
+};
+
+exports.listAllPublicModuleNames = function* () {
+  var sql = 'SELECT DISTINCT(name) AS name FROM tag ORDER BY name';
+  var rows = yield models.query(sql);
+  return rows.filter(function (row) {
+    return row.name[0] !== '@';
+  }).map(function (row) {
+    return row.name;
+  });
+};
+
+exports.listModulesByName = function* (moduleName) {
+  var mods = yield Module.findAll({
+    where: {
+      name: moduleName
+    },
+    order: [ ['id', 'DESC'] ]
+  });
+  return mods.map(function (mod) {
+    parseRow(mod);
+    return mod;
+  });
+};
+
+exports.getModuleLastModified = function* (name) {
+  var mod = yield Module.find({
+    where: {
+      name: name,
+    },
+    order: [
+      ['gmt_modified', 'DESC']
+    ],
+    attributes: [ 'gmt_modified' ]
+  });
+  return mod && mod.gmt_modified || null;
+};
+
+// module:update
+exports.saveModule = function* (mod) {
+  var keywords = mod.package.keywords;
+  if (typeof keywords === 'string') {
+    keywords = [keywords];
+  }
+  var pkg = stringifyPackage(mod.package);
+  var description = mod.package && mod.package.description || '';
+  var dist = mod.package.dist || {};
+  // dist.tarball = '';
+  // dist.shasum = '';
+  // dist.size = 0;
+  var publish_time = mod.publish_time || Date.now();
+  var item = yield* Module.findByNameAndVersion(mod.name, mod.version);
+  if (!item) {
+    item = Module.build({
+      name: mod.name,
+      version: mod.version
+    });
+  }
+  item.publish_time = publish_time;
+  // meaning first maintainer, more maintainers please check module_maintainer table
+  item.author = mod.author;
+  item.package = pkg;
+  item.dist_tarball = dist.tarball;
+  item.dist_shasum = dist.shasum;
+  item.dist_size = dist.size;
+  item.description = description;
+
+  var newItem = yield item.save();
+  var result = {
+    id: newItem.id,
+    gmt_modified: newItem.gmt_modified
+  };
+
+  if (!Array.isArray(keywords)) {
+    return result;
+  }
+
+  var words = [];
+  for (var i = 0; i < keywords.length; i++) {
+    var w = keywords[i];
+    if (typeof w === 'string') {
+      w = w.trim();
+      if (w) {
+        words.push(w);
+      }
+    }
+  }
+
+  if (words.length > 0) {
+    // add keywords
+    yield* exports.addKeywords(mod.name, description, words);
+  }
+
+  return result;
+};
+
+exports.updateModulePackage = function* (id, pkg) {
+  var mod = yield Module.find(Number(id));
+  if (!mod) {
+    // not exists
+    return null;
+  }
+  mod.package = stringifyPackage(pkg);
+  return yield mod.save(['package']);
+};
+
+exports.updateModulePackageFields = function* (id, fields) {
+  var mod = yield* exports.getModuleById(id);
+  if (!mod) {
+    return null;
+  }
+  var pkg = mod.package || {};
+  for (var k in fields) {
+    pkg[k] = fields[k];
+  }
+  return yield* exports.updateModulePackage(id, pkg);
+};
+
+exports.updateModuleReadme = function* (id, readme) {
+  var mod = yield* exports.getModuleById(id);
+  if (!mod) {
+    return null;
+  }
+  var pkg = mod.package || {};
+  pkg.readme = readme;
+  return yield* exports.updateModulePackage(id, pkg);
+};
+
+exports.updateModuleDescription = function* (id, description) {
+  var mod = yield* exports.getModuleById(id);
+  if (!mod) {
+    return null;
+  }
+  mod.description = description;
+  // also need to update package.description
+  var pkg = mod.package || {};
+  pkg.description = description;
+  mod.package = stringifyPackage(pkg);
+
+  return yield mod.save(['description', 'package']);
+};
+
+exports.updateModuleLastModified = function* (name) {
+  var row = yield Module.find({
+    where: { name: name },
+    order: [ [ 'gmt_modified', 'DESC' ] ],
+  });
+  if (!row) {
+    return null;
+  }
+  row.gmt_modified = new Date();
+  return yield row.save(['gmt_modified']);
+};
+
+exports.removeModulesByName = function* (name) {
+  yield Module.destroy({
+    where: {
+      name: name
+    }
+  });
+};
+
+exports.removeModulesByNameAndVersions = function* (name, versions) {
+  yield Module.destroy({
+    where: {
+      name: name,
+      version: versions
+    }
+  });
+};
+
+// tags
+
+exports.addModuleTag = function* (name, tag, version) {
+  var mod = yield* exports.getModule(name, version);
+  if (!mod) {
+    return null;
+  }
+
+  var row = yield* Tag.findByNameAndTag(name, tag);
+  if (!row) {
+    row = Tag.build({
+      name: name,
+      tag: tag
+    });
+  }
+  row.module_id = mod.id;
+  row.version = version;
+  return yield row.save();
+};
+
+exports.getModuleTag = function* (name, tag) {
+  return yield Tag.findByNameAndTag(name, tag);
+};
+
+exports.removeModuleTags = function* (name) {
+  return yield Tag.destroy({where: {name: name}});
+};
+
+exports.removeModuleTagsByIds = function* (ids) {
+  return yield Tag.destroy({where: {id: ids}});
+};
+
+exports.removeModuleTagsByNames = function* (moduleName, tagNames) {
+  return yield Tag.destroy({
+    where: {
+      name: moduleName,
+      tag: tagNames
+    }
+  });
+};
+
+exports.listModuleTags = function* (name) {
+  return yield Tag.findAll({ where: { name: name } });
+};
+
+// dependencies
+
+// name => dependency
+exports.addDependency = function* (name, dependency) {
+  var row = yield ModuleDependency.find({
+    where: {
+      name: dependency,
+      dependent: name
+    }
+  });
+  if (row) {
+    return row;
+  }
+  return yield ModuleDependency.build({
+    name: dependency,
+    dependent: name
+  }).save();
+};
+
+exports.addDependencies = function* (name, dependencies) {
+  var tasks = [];
+  for (var i = 0; i < dependencies.length; i++) {
+    tasks.push(exports.addDependency(name, dependencies[i]));
+  }
+  return yield tasks;
+};
+
+exports.listDependents = function* (dependency) {
+  var items = yield ModuleDependency.findAll({
+    where: {
+      name: dependency
+    }
+  });
+  return items.map(function (item) {
+    return item.dependent;
+  });
+};
+
+// maintainers
+
+exports.listPublicModuleMaintainers = function* (name) {
+  return yield* NpmModuleMaintainer.listMaintainers(name);
+};
+
+exports.addPublicModuleMaintainer = function* (name, user) {
+  return yield* NpmModuleMaintainer.addMaintainer(name, user);
+};
+
+exports.removePublicModuleMaintainer = function* (name, user) {
+  return yield* NpmModuleMaintainer.removeMaintainers(name, user);
+};
+
+// only can add to cnpm maintainer table
+exports.addPrivateModuleMaintainers = function* (name, usernames) {
+  return yield* PrivateModuleMaintainer.addMaintainers(name, usernames);
+};
+
+exports.updatePrivateModuleMaintainers = function* (name, usernames) {
+  var result = yield* PrivateModuleMaintainer.updateMaintainers(name, usernames);
+  if (result.add.length > 0 || result.remove.length > 0) {
+    yield* exports.updateModuleLastModified(name);
+  }
+  return result;
+};
+
+function* getMaintainerModel(name) {
+  var isPrivatePackage = yield* common.isPrivatePackage(name);
+  return isPrivatePackage ? PrivateModuleMaintainer : NpmModuleMaintainer;
+}
 
 exports.listMaintainers = function* (name) {
-  var names = yield* ModuleMaintainer.get(name);
-  if (names.length === 0) {
-    return names;
+  var mod = yield* getMaintainerModel(name);
+  var usernames = yield* mod.listMaintainers(name);
+  if (usernames.length === 0) {
+    return usernames;
   }
-  var users = yield* User.listByNames(names);
+  var users = yield* User.listByNames(usernames);
   return users.map(function (user) {
     return {
       name: user.name,
@@ -33,29 +511,22 @@ exports.listMaintainers = function* (name) {
 };
 
 exports.listMaintainerNamesOnly = function* (name) {
-  return yield* ModuleMaintainer.get(name);
-};
-
-exports.addMaintainers = function* (name, usernames) {
-  return yield* ModuleMaintainer.addMulti(name, usernames);
-};
-
-exports.updateMaintainers = function* (name, usernames) {
-  var rs = yield [
-    ModuleMaintainer.update(name, usernames),
-    Module.updateLastModified(name),
-  ];
-  return rs[0];
+  var mod = yield* getMaintainerModel(name);
+  return yield* mod.listMaintainers(name);
 };
 
 exports.removeAllMaintainers = function* (name) {
-  return yield* ModuleMaintainer.removeAll(name);
+  return yield [
+    PrivateModuleMaintainer.removeAllMaintainers(name),
+    NpmModuleMaintainer.removeAllMaintainers(name),
+  ];
 };
 
 exports.authMaintainer = function* (packageName, username) {
+  var mod = yield* getMaintainerModel(packageName);
   var rs = yield [
-    ModuleMaintainer.get(packageName),
-    Module.getLatest(packageName)
+    mod.listMaintainers(packageName),
+    exports.getLatestModule(packageName)
   ];
   var maintainers = rs[0];
   var latestMod = rs[1];
@@ -70,7 +541,6 @@ exports.authMaintainer = function* (packageName, username) {
   }
 
   var isMaintainer = false;
-
   if (latestMod && !latestMod.package._publish_on_cnpm) {
     // no one can update public package maintainers
     // public package only sync from source npm registry
@@ -92,3 +562,141 @@ exports.isMaintainer = function* (name, username) {
   var result = yield* exports.authMaintainer(name, username);
   return result.isMaintainer;
 };
+
+// module keywords
+
+exports.addKeyword = function* (data) {
+  var item = yield ModuleKeyword.findByKeywordAndName(data.keyword, data.name);
+  if (!item) {
+    item = ModuleKeyword.build(data);
+  }
+  item.description = data.description;
+  return yield item.save();
+};
+
+exports.addKeywords = function* (name, description, keywords) {
+  var tasks = [];
+  keywords.forEach(function (keyword) {
+    tasks.push(exports.addKeyword({
+      name: name,
+      keyword: keyword,
+      description: description
+    }));
+  });
+  return yield tasks;
+};
+
+// search
+
+exports.search = function* (word, options) {
+  options = options || {};
+  var limit = options.limit || 100;
+  word = word.replace(/^%/, ''); //ignore prefix %
+
+  // search flows:
+  // 1. prefix search by name
+  // 2. like search by name
+  // 3. keyword equal search
+  var ids = {};
+
+  var sql = 'SELECT module_id FROM tag WHERE LOWER(name) LIKE LOWER(?) AND tag="latest" \
+    ORDER BY name LIMIT ?;';
+  var rows = yield* models.query(sql, [word + '%', limit ]);
+  for (var i = 0; i < rows.length; i++) {
+    ids[rows[i].module_id] = 1;
+  }
+
+  if (rows.length < 20) {
+    rows = yield* models.query(sql, [ '%' + word + '%', limit ]);
+    for (var i = 0; i < rows.length; i++) {
+      ids[rows[i].module_id] = 1;
+    }
+  }
+
+  var keywordRows = yield ModuleKeyword.findAll({
+    attributes: [ 'name', 'description' ],
+    where: {
+      keyword: word
+    },
+    limit: limit,
+    order: [ [ 'id', 'DESC' ] ]
+  });
+
+  var data = {
+    keywordMatchs: keywordRows,
+    searchMatchs: []
+  };
+
+  ids = Object.keys(ids);
+  if (ids.length > 0) {
+    data.searchMatchs = yield Module.findAll({
+      attributes: [ 'name', 'description' ],
+      where: {
+        id: ids
+      },
+      order: 'name'
+    });
+  }
+
+  return data;
+};
+
+// module star
+
+exports.addStar = function* add(name, user) {
+  var row = yield ModuleStar.find({
+    where: {
+      name: name,
+      user: user
+    }
+  });
+  if (row) {
+    return row;
+  }
+
+  row = ModuleStar.build({
+    name: name,
+    user: user
+  });
+  return yield row.save();
+};
+
+exports.removeStar = function* (name, user) {
+  return yield ModuleStar.destroy({
+    where: {
+      name: name,
+      user: user
+    }
+  });
+};
+
+exports.listStarUserNames = function* (moduleName) {
+  var rows = yield ModuleStar.findAll({
+    where: {
+      name: moduleName
+    }
+  });
+  return rows.map(function (row) {
+    return row.user;
+  });
+};
+
+exports.listUserStarModuleNames = function* (user) {
+  var rows = yield ModuleStar.findAll({
+    where: {
+      user: user
+    }
+  });
+  return rows.map(function (row) {
+    return row.name;
+  });
+};
+
+// unpublish info
+exports.saveUnpublishedModule = function* (name, pkg) {
+  return yield* ModuleUnpublished.save(name, pkg);
+};
+
+exports.getUnpublishedModule = function* (name) {
+  return yield* ModuleUnpublished.findByName(name);
+};

services/total.js

@@ -0,0 +1,123 @@
+/**!
+ * cnpmjs.org - services/total.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('../config');
+var models = require('../models');
+var Total = models.Total;
+
+exports.get = function* () {
+  // var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
+  //   FROM information_schema.tables WHERE TABLE_SCHEMA = ? \
+  //   GROUP BY TABLE_NAME \
+  //   ORDER BY data_length DESC \
+  //   LIMIT 0, 200';
+  var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM `module`;';
+  var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM `module`;';
+  var TOTAL_USER_SQL = 'SELECT count(name) AS count FROM `user`;';
+  var rs = yield [
+    // models.query(DB_SIZE_SQL, [config.db]),
+    models.queryOne(TOTAL_MODULE_SQL),
+    models.queryOne(TOTAL_VERSION_SQL),
+    models.queryOne(TOTAL_USER_SQL),
+    exports.getTotalInfo(),
+  ];
+
+  // var sizes = rs[0];
+  var mc = rs[0];
+  var vc = rs[1];
+  var uc = rs[2];
+  var info = rs[3] || {};
+
+  var total = {
+    data_tables: {},
+    disk_size: 0,
+    data_size: 0,
+    index_size: 0,
+    disk_format_version: 0,
+    committed_update_seq: 0,
+    update_seq: 0,
+    purge_seq: 0,
+    compact_running: false,
+    doc_count: mc.count,
+    doc_del_count: info.module_delete || 0,
+    doc_version_count: vc.count,
+    user_count: uc.count,
+    store_engine: config.database.dialect,
+    sync_status: info.sync_status,
+    need_sync_num: info.need_sync_num || 0,
+    success_sync_num: info.success_sync_num || 0,
+    fail_sync_num: info.fail_sync_num || 0,
+    left_sync_num: info.left_sync_num || 0,
+    last_sync_time: info.last_sync_time || 0,
+    last_exist_sync_time: info.last_exist_sync_time || 0,
+    last_sync_module: info.last_sync_module || '',
+  };
+
+  // for (var i = 0; i < sizes.length; i++) {
+  //   var row = sizes[i];
+  //   total.data_tables[row.name] = {
+  //     data_size: row.data_length,
+  //     index_size: row.index_length,
+  //   };
+  //   total.data_size += row.data_length;
+  //   total.index_size += row.index_length;
+  // }
+
+  total.disk_size = total.data_size + total.index_size;
+
+  return total;
+};
+
+exports.getTotalInfo = function* () {
+  return yield Total.find({
+    where: {
+      name: 'total'
+    }
+  });
+};
+
+exports.plusDeleteModule = function* () {
+  var sql = 'UPDATE total SET module_delete=module_delete+1 WHERE name="total"';
+  return yield* models.query(sql);
+};
+
+exports.setLastSyncTime = function* (time) {
+  var sql = 'UPDATE total SET last_sync_time=? WHERE name="total"';
+  return yield* models.query(sql, [Number(time)]);
+};
+
+exports.setLastExistSyncTime = function* (time) {
+  var sql = 'UPDATE total SET last_exist_sync_time=? WHERE name="total"';
+  return yield* models.query(sql, [Number(time)]);
+};
+
+exports.updateSyncStatus = function* (status) {
+  var sql = 'UPDATE total SET sync_status=? WHERE name="total"';
+  return yield* models.query(sql, [status]);
+};
+
+exports.updateSyncNum = function* (params) {
+  var arg = {
+    sync_status: params.syncStatus,
+    need_sync_num: params.need || 0,
+    success_sync_num: params.success || 0,
+    fail_sync_num: params.fail || 0,
+    left_sync_num: params.left || 0,
+    last_sync_module: params.lastSyncModule
+  };
+  var sql = 'UPDATE total SET ? WHERE name="total"';
+  return yield* models.query(sql, [arg]);
+};

services/user.js

@@ -15,6 +15,8 @@
  */
 
 var config = require('../config');
+var User = require('../models').User;
+
 if (!config.userService) {
   var DefaultUserService = require('./default_user_service');
   config.userService = new DefaultUserService();
@@ -54,3 +56,39 @@ exports.search = function* (query, options) {
   var users = yield* config.userService.search(query, options);
   return users.map(convertUser);
 };
+
+exports.getAndSave = function* (login) {
+  if (config.customUserService) {
+    var user = yield* exports.get(login);
+    if (user) {
+      var data = {
+        user: user
+      };
+      yield* User.saveCustomUser(data);
+    }
+  }
+  return yield* User.findByName(login);
+};
+
+exports.authAndSave = function* (login, password) {
+  var user = yield* exports.auth(login, password);
+  if (user) {
+    if (config.customUserService) {
+      // make sure sync user meta to cnpm database
+      var data = {
+        rev: Date.now() + '-' + user.login,
+        user: user
+      };
+      yield* User.saveCustomUser(data);
+    }
+  }
+  return user;
+};
+
+exports.add = function* (user) {
+  return yield* User.add(user);
+};
+
+exports.update = function* (user) {
+  return yield* User.update(user);
+};

sync/index.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - sync/index.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -16,14 +16,13 @@
 
 var debug = require('debug')('cnpmjs.org:sync:index');
 var co = require('co');
-var ms = require('ms');
+var ms = require('humanize-ms');
 var util = require('util');
-var utility = require('utility');
 var config = require('../config');
 var mail = require('../common/mail');
-var Total = require('../proxy/total');
 var logger = require('../common/logger');
-var syncDistWorker = require('./sync_dist');
+var totalService = require('../services/total');
+var DistSyncer = require('./sync_dist');
 
 var sync = null;
 
@@ -45,12 +44,23 @@ console.log('[%s] [sync_worker:%s] syncing with %s mode',
   Date(), process.pid, config.syncModel);
 
 //set sync_status = 0 at first
-Total.updateSyncStatus(0, utility.noop);
+co(function* () {
+  yield* totalService.updateSyncStatus(0);
+})(function (err) {
+  if (err) {
+    logger.error(err);
+  }
+});
+
+var syncInterval = ms(config.syncInterval);
+var minSyncInterval = ms('5m');
+if (!syncInterval || syncInterval < minSyncInterval) {
+  syncInterval = minSyncInterval;
+}
 
 // the same time only sync once
 var syncing = false;
-
-var handleSync = co(function *() {
+var handleSync = co(function* () {
   debug('mode: %s, syncing: %s', config.syncModel, syncing);
   if (!syncing) {
     syncing = true;
@@ -58,7 +68,7 @@ var handleSync = co(function *() {
     var data;
     var error;
     try {
-      var data = yield *sync();
+      data = yield* sync();
     } catch (err) {
       error = err;
       error.message += ' (sync package error)';
@@ -74,19 +84,26 @@ var handleSync = co(function *() {
 
 if (sync) {
   handleSync();
-  setInterval(handleSync, ms(config.syncInterval));
+  setInterval(handleSync, syncInterval);
 }
 
+/**
+ * sync dist(node.js and phantomjs)
+ */
+
 var syncingDist = false;
-var syncDist = co(function* syncDist() {
+var syncDist = co(function* () {
   if (syncingDist) {
     return;
   }
   syncingDist = true;
   logger.syncInfo('Start syncing dist...');
+  var distSyncer = new DistSyncer({
+    disturl: config.disturl
+  });
   try {
-    yield* syncDistWorker();
-    yield* syncDistWorker.syncPhantomjsDir();
+    yield* distSyncer.start();
+    yield* distSyncer.syncPhantomjsDir();
   } catch (err) {
     err.message += ' (sync dist error)';
     logger.syncError(err);
@@ -99,11 +116,82 @@ var syncDist = co(function* syncDist() {
 
 if (config.syncDist) {
   syncDist();
-  setInterval(syncDist, ms(config.syncInterval));
+  setInterval(syncDist, syncInterval);
 } else {
   logger.syncInfo('sync dist disable');
 }
 
+/**
+ * sync python dist
+ */
+
+var syncingPythonDist = false;
+var syncPythonDist = co(function* () {
+  if (syncingPythonDist) {
+    return;
+  }
+  syncingPythonDist = true;
+  logger.syncInfo('Start syncing python dist...');
+  var distSyncer = new DistSyncer({
+    disturl: config.pythonDisturl
+  });
+  try {
+    yield* distSyncer.start();
+  } catch (err) {
+    err.message += ' (sync python dist error)';
+    logger.syncError(err);
+    if (config.noticeSyncDistError) {
+      sendMailToAdmin(err, null, new Date());
+    }
+  }
+  syncingPythonDist = false;
+});
+
+if (config.syncPythonDist) {
+  syncPythonDist();
+  setInterval(syncPythonDist, syncInterval);
+} else {
+  logger.syncInfo('sync python dist disable');
+}
+
+/**
+ * sync popular modules
+ */
+
+var syncingPopular = false;
+var syncPopular = co(function* syncPopular() {
+  if (syncingPopular) {
+    return;
+  }
+  syncingPopular = true;
+  logger.syncInfo('Start syncing popular modules...');
+  var data;
+  var error;
+  try {
+    data = yield* require('./sync_popular');
+  } catch (err) {
+    error = err;
+    error.message += ' (sync package error)';
+    logger.syncError(error);
+  }
+
+  if (data) {
+    logger.syncInfo(data);
+  }
+  if (!config.debug) {
+    sendMailToAdmin(error, data, new Date());
+  }
+
+  syncingPopular = false;
+});
+
+if (config.syncPopular) {
+  syncPopular();
+  setInterval(syncPopular, ms(config.syncPopularInterval));
+} else {
+  logger.syncInfo('sync popular module disable');
+}
+
 function sendMailToAdmin(err, result, syncTime) {
   result = result || {};
   var to = [];

sync/status.js

@@ -15,8 +15,8 @@
  */
 
 var debug = require('debug')('cnpmjs.org:sync:status');
-var utility = require('utility');
-var Total = require('../proxy/total');
+var co = require('co');
+var Total = require('../services/total');
 
 function Status(options) {
   this.need = options.need;
@@ -35,7 +35,9 @@ Status.prototype.log = function (syncDone) {
     left: this.left,
     lastSyncModule: this.lastSyncModule,
   };
-  Total.updateSyncNum(params, utility.noop);
+  co(function* () {
+    yield* Total.updateSyncNum(params);
+  })();
 };
 
 Status.prototype.start = function () {