Release 2.1.0

feat(web): search support jsonp

.npmignore

@@ -22,3 +22,4 @@ config/web_readme.md
 config/config.js
 *.sqlite
 .node-dev.json
+nohup.out

.travis.yml

@@ -1,6 +1,8 @@
 language: node_js
 node_js:
-  - '0.11.14'
-before_install: "npm install --build-from-source"
-script: "make test-travis-all"
-after_script: "npm install coveralls@2 && cat ./coverage/lcov.info | coveralls"
+  - 'iojs-1.2'
+  - '0.12'
+addons:
+  - postgresql: '9.3'
+script: 'make test-travis-all'
+after_script: 'npm install coveralls@2 && cat ./coverage/lcov.info | coveralls'

AUTHORS

@@ -1,8 +1,7 @@
 # Ordered by date of first contribution.
-# Auto-generated by 'contributors' on Mon, 03 Mar 2014 13:01:28 GMT.
-# https://github.com/xingrz/node-contributors
 
 fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)
 dead-horse <dead_horse@qq.com> (https://github.com/dead-horse)
 alsotang <alsotang@gmail.com> (https://github.com/alsotang)
 4simple <wondger@qq.com> (https://github.com/4simple)
+afc163 <afc163@gmail.com> (https://github.com/afc163)

History.md

@@ -1,4 +1,131 @@
 
+2.1.0 / 2015-07-08
+==================
+
+ * feat(web): search support jsonp
+ * fix function name
+
+2.0.0 / 2015-05-11
+==================
+
+ * fix: real download as stream
+ * add custom ad banner config
+ * add sponsors: ucloud.cn
+ * fix small typo
+ * feat(urllib): support http_proxy
+ * force using https links
+
+2.0.0-rc.15 / 2015-02-15
+==================
+
+ * fix(markdown): filter xss after markdown render
+ * feat(database): support PostgreSQL
+
+2.0.0-rc.14 / 2015-02-14
+==================
+
+ * feat: support always-auth
+ * fix mysql select args = [] bug
+ * fix #597 sequelize raw query.
+ * fix(markdown): hotfix markdown-it cpu problem
+ * feat: upgrade to co4
+ * use kcors fixes #594
+
+2.0.0-rc.13 / 2015-02-04
+==================
+
+ * docs: Deploy a private npm registry in 5 minutes
+ * refactor(config): move application data to ~/.cnpmjs.org/
+ * fix(sync): make get popular pakcage faster
+ * feat(sync): web page also redirect to npm www
+ * refactor(config): make syncModel to none by default
+ * test: fix admin can not publish non-scoped package test cases
+ * docs: add chinese mirror link
+ * fix: admin can not publish non scoped package on "none" sync model
+ * feat(sync): enable none syncModel proxy all public packages
+ * fix: ignore username start with " or '
+ * fix(bin): fix stop not work on iojs
+
+2.0.0-rc.12 / 2015-02-01
+==================
+
+ * feat(syncer): add hostname ua
+ * fix(web): remove pkg.contributors logic
+
+2.0.0-rc.11 / 2015-02-01
+==================
+
+ * fix xss tests
+ * fix(markdown): revert marky-markdown
+
+2.0.0-rc.10 / 2015-01-31
+==================
+
+ * feat(middleware): CORS headers for GET and HEAD requests
+ * fix(readme): fix index page markdown
+ * feat(markdown): use npm same markdown parser
+ * feat(download): support download redirect to nfs
+ * feat(syncer): request npm registry with gzip
+ * change(sync): remove dist syncer
+ * feat(registry): add dist tag api
+ * refactor(common): remove redis store
+
+2.0.0-rc.9 / 2015-01-22 
+==================
+
+  * hotfix reame render error, pin xss
+  * fix registry user auth api
+
+2.0.0-rc.8 / 2015-01-10
+==================
+
+ * fix(markdown): readme.md allow scripts
+ * fix(style) flexbox compatibility for both chrome and firefox (@afc163)
+ * feat(sync): default sync exist packages
+
+2.0.0-rc.7 / 2015-01-07
+==================
+
+ * install sync dont check `enablePrivate`
+ * fix(markdown): filter xss readme before markdown render
+
+2.0.0-rc.6 / 2015-01-05
+==================
+
+ * fix(markdown): use markdown-it
+ * add userService options on config
+ * add upload to nfs sync info log
+
+2.0.0-rc.5 / 2015-01-03
+==================
+
+ * fix(markdown): use marked instead of remarkable
+ * fix(package): pkg.readme is not a string, dont remarkable it
+ * feat(sync): sync user profile
+
+2.0.0-rc.4 / 2014-12-25
+==================
+
+ * refactor(download): try to use nsf.url() first
+ * use __all__ for full downloads
+ * refactor(download_total): optimize download total
+ * fix sqlite raw sql return datetime is string format
+ * fix(download_total): change column date to DateTime
+ * fix(services/download_total): fix download_total slow sql on `date >= $start and date <= $end`
+ * fix(markdown): replace marked use remarkable
+
+2.0.0-rc.3 / 2014-12-14
+==================
+
+ * fix(services): need to detect instance isDirty or not before save()
+
+2.0.0-rc.2 / 2014-12-11
+==================
+
+ * add download API, closes [#529](https://github.com/cnpm/cnpmjs.org/issues/529)
+ * fix missing home page title (@rockdai)
+ * Fix typo in view/web/package.html (@LoicMahieu)
+
 2.0.0-rc.1 / 2014-12-09
 ==================
 
@@ -8,7 +135,7 @@
 2.0.0-beta5 / 2014-12-05
 ==================
 
- * hotfix package.html typo. Closes #521
+ * hotfix package.html typo. Closes [#521](https://github.com/cnpm/cnpmjs.org/issues/521)
  * Add editorconfig
  * fix(web/package): package name to long cause style problem fix
  * fix(css): use github-markdown-css for markdown body
@@ -20,7 +147,7 @@
  * fix(registry): add missing /-/short api
  * zoom sync link
  * new design for package page
- * image max width, fixed #505
+ * image max width, fixed [#505](https://github.com/cnpm/cnpmjs.org/issues/505)
  * feat(middleware): block Ruby user-agent
 
 2.0.0-beta3 / 2014-11-12
@@ -50,7 +177,7 @@
 2.0.0-beta0 / 2014-11-02
 ==================
 
- * ungrade koa-markdown to use remarkable, close #482
+ * ungrade koa-markdown to use remarkable, close [#482](https://github.com/cnpm/cnpmjs.org/issues/482)
  * fix(module_log): limit module sync log size to 1MB
  * refactor(config): remove adaptScope config key
  * chore(Makefile): $ make install-production

Makefile

@@ -2,12 +2,18 @@ TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
 REPORTER = spec
 TIMEOUT = 30000
 MOCHA_OPTS =
-REGISTRY = --registry=https://registry.npm.taobao.org
 DB = sqlite
+DISTURL = http://npm.taobao.org/mirrors/iojs
+BIN = iojs
+
+ifeq ($(findstring io.js, $(shell which node)),)
+	BIN = node
+	DISTURL = http://npm.taobao.org/mirrors/node
+endif
 
 install:
-	@npm install --build-from-source $(REGISTRY) \
-		--disturl=https://npm.taobao.org/dist
+	@npm install --build-from-source --registry=http://registry.npm.taobao.org \
+		--disturl=$(DISTURL)
 
 install-production production:
 	@NODE_ENV=production $(MAKE) install
@@ -16,12 +22,16 @@ jshint: install
 	@-node_modules/.bin/jshint ./
 
 init-database:
-	@node --harmony test/init_db.js
+	@$(BIN) --harmony test/init_db.js
 
 init-mysql:
 	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
 	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
 
+init-pg:
+	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@psql -c 'CREATE DATABASE cnpmjs_test;'
+
 test: install init-database
 	@NODE_ENV=test DB=${DB} node_modules/.bin/mocha \
 		--harmony \
@@ -40,10 +50,13 @@ test-sqlite:
 test-mysql: init-mysql
 	@$(MAKE) test DB=mysql
 
+test-pg: init-pg
+	@DB_PORT=5432 DB_USER=$(USER) $(MAKE) test DB=postgres
+
 test-all: test-sqlite test-mysql
 
 test-cov cov: install init-database
-	@NODE_ENV=test DB=${DB} node --harmony \
+	@NODE_ENV=test DB=${DB} $(BIN) --harmony \
 		node_modules/.bin/istanbul cover --preserve-comments \
 		node_modules/.bin/_mocha \
 		-- -u exports \
@@ -63,12 +76,12 @@ test-cov-mysql: init-mysql
 	@$(MAKE) test-cov DB=mysql
 
 test-travis: install init-database
-	@NODE_ENV=test DB=${DB} CNPM_SOURCE_NPM=https://registry.npmjs.org CNPM_SOURCE_NPM_ISCNPM=false \
-		node --harmony \
+	@NODE_ENV=test DB=${DB} CNPM_SOURCE_NPM=http://registry.npmjs.com CNPM_SOURCE_NPM_ISCNPM=false \
+		$(BIN) --harmony \
 		node_modules/.bin/istanbul cover --preserve-comments \
 		node_modules/.bin/_mocha \
 		--report lcovonly \
-		-- \
+		-- -u exports \
 		--reporter dot \
 		--timeout $(TIMEOUT) \
 		--require should \
@@ -84,10 +97,15 @@ test-travis-sqlite:
 test-travis-mysql: init-mysql
 	@$(MAKE) test-travis DB=mysql
 
-test-travis-all: test-travis-sqlite test-travis-mysql
+test-travis-pg:
+	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;' -U postgres
+	@psql -c 'CREATE DATABASE cnpmjs_test;' -U postgres
+	@DB_PORT=5432 DB_USER=postgres $(MAKE) test-travis DB=postgres
+
+test-travis-all: test-travis-sqlite test-travis-mysql test-travis-pg
 
 dev:
-	@NODE_ENV=development node_modules/.bin/node-dev --harmony dispatch.js
+	@NODE_ENV=development $(BIN) node_modules/.bin/node-dev --harmony dispatch.js
 
 contributors: install
 	@node_modules/.bin/contributors -f plain -o AUTHORS

README.md

@@ -42,9 +42,10 @@ Our goal is to provide a low cost maintenance and easy to use solution for priva
 * Build a mirror NPM. (we use it to build a mirror in China: [cnpmjs.org](http://cnpmjs.org/))
 * Build a completely independent NPM registry to store whatever you like.
 
-### Features
+## Features
 
 * **Support "scoped" packages**: [npm/npm#5239](https://github.com/npm/npm/issues/5239)
+* **Support [CORS](http://en.wikipedia.org/wiki/Cross-origin_resource_sharing)**
 * **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 You can get the source code through `npm` or `git`.
 * **Low cost and easy maintenance**: `package.json` info store in MySQL, MariaDB, SQLite or PostgreSQL databases,
@@ -59,17 +60,20 @@ And it easy to wrap for your own registry which build with `cnpmjs.org`.
 * **Compatible with NPM client**: you can use the origin NPM client with `cnpmjs.org`,
 only need to change the registry in config. Even include manual synchronization (through `install` command).
 * **Version badge**: base on [shields.io](http://shields.io/) ![cnpm-badge](http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square)
+* **Support http_proxy**: if you're behind firewall, need to request through http proxy
 
 **PROTIP** Be sure to read [Migrating from 1.x to 2.x](https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x)
 as well as [New features in 2.x](https://github.com/cnpm/cnpmjs.org/wiki/New-features-in-2.x).
 
 ## Getting Start
 
+* [Deploy a private npm registry in 5 minutes](https://github.com/cnpm/cnpmjs.org/wiki/Deploy-a-private-npm-registry-in-5-minutes)
 * @[dead-horse](https://github.com/dead-horse): [What is cnpm?](http://deadhorse.me/slides/cnpmjs.html)
 * install and deploy cnpmjs.org through npm: [examples](https://github.com/cnpm/custom-cnpm-example)
 * Mirror NPM in China: [cnpmjs.org](http://cnpmjs.org)
 * cnpm client: [cnpm](https://github.com/cnpm/cnpm), `npm install -g cnpm`
 * [How to deploy cnpmjs.org](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
+* [Sync packages through `http_proxy`](https://github.com/cnpm/cnpmjs.org/wiki/Sync-packages-through-http_proxy)
 * [wiki](https://github.com/cnpm/cnpmjs.org/wiki)
 
 ## Develop on your local machine
@@ -98,7 +102,7 @@ $ make test
 # coverage
 $ make test-cov
 
-# udpate dependencies
+# update dependencies
 $ make autod
 
 # start server with development mode
@@ -114,27 +118,10 @@ $ make dev
 
 Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
 
+## Sponsors
+
+- [![UCloud云计算](https://www.ucloud.cn/www/img/www/logo.png)](http://www.ucloud.cn/sdk?sem=sdk-CNPMJS)
+
 ## License
 
-(The MIT License)
-
-Copyright(c) cnpmjs.org and other contributors.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+MIT

backup/dump.js

@@ -1,142 +0,0 @@
-/**!
- * cnpmjs.org - backup/dump.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-/**
- * 1. dump module
- * 2. dump tag
- * 3. dump user
- * 4. total
- * 5. download_total
- */
-
-var path = require('path');
-var fs = require('fs');
-var moment = require('moment');
-var eventproxy = require('eventproxy');
-var util = require('util');
-var zlib = require('zlib');
-var mysql = require('../common/mysql');
-var nfs = require('../common/nfs');
-var config = require('../config');
-
-function dumpTable(name, lastRow, callback) {
-  var sql = 'SELECT * from ' + name + ' WHERE gmt_modified >=? ORDER BY gmt_modified ASC LIMIT 10000;';
-  mysql.query(sql, [lastRow.gmt_modified], function (err, rows) {
-    if (err || rows.length === 0) {
-      return callback(err, rows);
-    }
-    if (rows[0].id === lastRow.id) {
-      rows = rows.slice(1);
-    }
-    callback(null, rows);
-  });
-}
-
-function log() {
-  var str = '[' + moment().format('YYYY-MM-DD HH:mm:ss') + '] ' + util.format.apply(util, arguments);
-  console.log(str);
-}
-
-function syncTable(name, callback) {
-  var datadir = __dirname;
-  var dataFile = path.join(datadir, moment().format('YYYY-MM-DD-HH') + '_' + name + '.json');
-  var lastRowFile = path.join(datadir, name + '_lastdate.json');
-  var lastRow = null;
-  if (fs.existsSync(lastRowFile)) {
-    lastRow = require(lastRowFile);
-    lastRow.gmt_modified = new Date(Date.parse(lastRow.gmt_modified));
-  } else {
-    lastRow = {
-      gmt_modified: new Date('2011-11-11'),
-    };
-  }
-
-  log('getting "%s" since %j', name, lastRow);
-  dumpTable(name, lastRow, function (err, rows) {
-    console.log('[%s] got %d rows', Date(), rows && rows.length || 0);
-    if (err) {
-      return callback(err);
-    }
-
-    if (!rows || rows.length === 0) {
-      return callback();
-    }
-
-    var writeStream = fs.createWriteStream(dataFile, {flags: 'a'});
-    writeStream.once('error', callback);
-    for (var i = 0; i < rows.length; i++) {
-      writeStream.write(JSON.stringify(rows[i]) + '\n');
-    }
-    writeStream.end();
-    writeStream.on('finish', function () {
-      log('append %d rows to %s', rows.length, dataFile);
-      var gzfile = dataFile + '.gz';
-      var gzip = zlib.createGzip();
-      var inp = fs.createReadStream(dataFile);
-      var out = fs.createWriteStream(gzfile);
-      inp.pipe(gzip).pipe(out);
-      out.once('error', callback);
-      out.on('finish', function () {
-        var key = path.join(config.backupFilePrefix, path.basename(gzfile));
-        log('saving %s to %s ...', gzfile, key);
-        nfs.upload(gzfile, {key: key}, function (err, result) {
-          if (err) {
-            return callback(err);
-          }
-
-          lastRow = rows[rows.length - 1];
-          lastRow = {gmt_modified: lastRow.gmt_modified, id: lastRow.id};
-          fs.writeFileSync(lastRowFile, JSON.stringify(lastRow));
-          log('save %s data file to %j, lastrow: %j', name, result, lastRow);
-          callback();
-        });
-      });
-    });
-  });
-}
-
-var ep = eventproxy.create();
-ep.fail(function (err) {
-  log('error: %s', err.stack);
-  process.exit(1);
-});
-
-syncTable('module', ep.done('module'));
-
-ep.on('module', function () {
-  syncTable('tag', ep.done('tag'));
-});
-
-ep.on('tag', function () {
-  syncTable('user', ep.done('user'));
-});
-
-ep.on('user', function () {
-  syncTable('total', ep.done('total'));
-});
-
-ep.on('total', function () {
-  syncTable('download_total', ep.done('download_total'));
-});
-
-ep.on('download_total', function () {
-  ep.emit('finish');
-});
-
-ep.on('finish', function () {
-  log('finish, %d process exit', process.pid);
-  process.exit(0);
-});

bin/cli.js

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+
+/**!
+ * cnpmjs.org - bin/cli.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+require('gnode');
+var debug = require('debug')('cnpmjs.org:cli');
+var program = require('commander');
+var path = require('path');
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+var treekill = require('treekill');
+var version = require('../package.json').version;
+
+function list(val) {
+  return val.split(',');
+}
+
+program
+  .version(version);
+
+program
+  .command('start')
+  .description('start cnpmjs.org server')
+  .option('--admins <admins>', 'set admins', list)
+  .option('--scopes <scopes>', 'set scopes', list)
+  // .option('--cluster', 'enable cluster mode')
+  .option('--dataDir <dataDir>', 'cnpmjs.org data dir, default is `$HOME/.cnpmjs.org`')
+  .action(start);
+
+program
+  .command('stop')
+  .description('stop cnpmjs.org server')
+  .option('--dataDir <dataDir>', 'cnpmjs.org data dir, default is `$HOME/.cnpmjs.org`')
+  .action(stop);
+
+program.parse(process.argv);
+
+
+function start(options) {
+  stop(options);
+  var dataDir = options.dataDir || path.join(process.env.HOME, '.cnpmjs.org');
+  mkdirp.sync(dataDir);
+
+  var configfile = path.join(dataDir, 'config.json');
+  var config = {};
+  if (fs.existsSync(configfile)) {
+    try {
+      config = require(configfile);
+    } catch (err) {
+      console.warn('load old %s error: %s', configfile, err);
+    }
+  }
+  // config.enableCluster = !!options.cluster;
+  if (options.admins) {
+    config.admins = {};
+    for (var i = 0; i < options.admins.length; i++) {
+      config.admins[options.admins[i]] = options.admins[i] + '@localhost.com';
+    }
+  }
+  if (options.scopes) {
+    config.scopes = options.scopes.map(function (name) {
+      if (name[0] !== '@') {
+        name = '@' + name;
+      }
+      return name;
+    });
+  }
+
+  var configJSON = JSON.stringify(config, null, 2);
+  fs.writeFileSync(configfile, configJSON);
+
+  debug('save config %s to %s', configJSON, configfile);
+
+  // if sqlite db file not exists, init first
+  initDatabase();
+
+  require('../dispatch');
+  fs.writeFileSync(path.join(dataDir, 'pid'), process.pid + '');
+}
+
+function stop(options) {
+  var dataDir = options.dataDir || path.join(process.env.HOME, '.cnpmjs.org');
+  var pidfile = path.join(dataDir, 'pid');
+  if (fs.existsSync(pidfile)) {
+    var pid = Number(fs.readFileSync(pidfile, 'utf8'));
+    treekill(pid, function (err) {
+      if (err) {
+        console.log(err);
+        throw err;
+      }
+      console.log('cnpmjs.org server:%d stop', pid);
+      fs.unlinkSync(pidfile);
+    });
+  }
+}
+
+function initDatabase() {
+  var models = require('../models');
+
+  models.sequelize.sync({ force: false })
+    .then(function () {
+      models.Total.init(function (err) {
+        if (err) {
+          console.error('[models/init_script.js] sequelize init fail');
+          console.error(err);
+          throw err;
+        } else {
+          console.log('[models/init_script.js] `sqlite` sequelize sync and init success');
+        }
+      });
+    })
+    .catch(function (err) {
+      console.error('[models/init_script.js] sequelize sync fail');
+      console.error(err);
+      throw err;
+    });
+}

bin/nodejsctl

@@ -52,9 +52,9 @@ stop()
     kill -15 $PID
     sleep 2
 
-    node_num=`ps -ef |grep node|grep ${PROJECT_NAME}|grep -v grep|wc -l`
+    node_num=`ps -ef | grep ${PROJECT_NAME} | grep -v grep | wc -l`
     if [ $node_num != 0 ]; then
-      ps -ef |grep node | grep ${PROJECT_NAME} |grep -v grep|awk '{print $2}'|xargs kill -9
+      ps -ef | grep ${PROJECT_NAME} |grep -v grep|awk '{print $2}'|xargs kill -9
       ipcs -s | grep 0x | awk '{print $2}' | xargs -n1 ipcrm -s  > /dev/null 2>&1
       ipcs -m | grep 0x | awk '{print $2}' | xargs -n1 ipcrm -m  > /dev/null 2>&1
     fi

bin/restore_module_deps.js

@@ -1,74 +0,0 @@
-/**!
- * cnpmjs.org - bin/restore_module_deps.js
- *
- * Copyright(c) 2014
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
-
-var mysql = require('../common/mysql');
-var Module = require('../proxy/module');
-var ModuleDeps = require('../proxy/module_deps');
-
-var addCount = 0;
-
-function restore(id, callback) {
-  var sql = 'SELECT id, name, package FROM module WHERE id > ? ORDER BY id ASC LIMIT 1000';
-  mysql.query(sql, [id], function (err, rows) {
-    if (err) {
-      return callback(err);
-    }
-    if (rows.length === 0) {
-      return callback(null, []);
-    }
-
-    console.log('[%s] got %d rows', id, rows.length);
-
-    rows.forEach(function (r) {
-      Module.parseRow(r);
-      if (!r.package) {
-        return;
-      }
-      var deps = Object.keys(r.package.dependencies || {});
-      if (!Array.isArray(deps) || !deps.length) {
-        return;
-      }
-      deps.forEach(function (dep) {
-        ModuleDeps.add(dep, r.name, function (err) {
-          // console.log('[%s] add %s <= %s, error: %s', id, dep, r.name, err);
-        });
-      });
-      addCount += deps.length;
-    });
-    setTimeout(function () {
-      console.log('[%s] add %d relations', id, addCount);
-      callback(null, rows);
-    }, 1000);
-  });
-}
-
-var id = 0;
-function run() {
-  restore(id, function (err, rows) {
-    if (err) {
-      throw err;
-    }
-    if (rows.length === 0) {
-      console.log('finished, last id: %s, exit.', id);
-      process.exit(0);
-    }
-
-    id = rows[rows.length - 1].id;
-    run();
-  });
-}
-
-run();

common/mail.js

@@ -14,10 +14,10 @@
  * Module dependencies.
  */
 
-var mailConfig = require('../config').mail;
 var nodemailer = require('nodemailer');
 var utility = require('utility');
 var os = require('os');
+var mailConfig = require('../config').mail;
 
 var smtpConfig;
 if (mailConfig.auth) {
@@ -25,6 +25,7 @@ if (mailConfig.auth) {
   smtpConfig = mailConfig;
 } else {
   smtpConfig = {
+    enable: mailConfig.enable,
     // backward compat
     host: mailConfig.host,
     port: mailConfig.port,
@@ -37,7 +38,7 @@ if (mailConfig.auth) {
   };
 }
 
-var transport = nodemailer.createTransport(smtpConfig);
+var transport;
 
 /**
  * Send notice email with mail level and appname.
@@ -71,6 +72,15 @@ LEVELS.forEach(function (level) {
 exports.send = function (to, subject, html, callback) {
   callback = callback || utility.noop;
 
+  if (mailConfig.enable === false) {
+    console.log('[send mail debug] [%s] to: %s, subject: %s\n%s', Date(), to, subject, html);
+    return callback();
+  }
+
+  if (!transport) {
+    transport = nodemailer.createTransport(smtpConfig);
+  }
+
   var message = {
     from: mailConfig.from || mailConfig.sender,
     to: to,

common/markdown.js

@@ -0,0 +1,34 @@
+/**!
+ * cnpmjs.org - common/markdown.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var xss = require('xss');
+var MarkdownIt = require('markdown-it');
+
+// allow class attr on code
+xss.whiteList.code = ['class'];
+
+var md = new MarkdownIt({
+  html: true,
+  linkify: true,
+});
+
+exports.render = function (content, filterXss) {
+  var html = md.render(content);
+  if (filterXss !== false) {
+    html = xss(html);
+  }
+  return html;
+};

common/redis.js

@@ -1,37 +0,0 @@
-/**!
- * cnpmjs.org - common/redis.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var config = require('../config');
-
-// close redis by set config.redis to `null` or `{}`
-if (config.redis && config.redis.host && config.redis.port) {
-
-  var redis = require('redis');
-  var wrapper = require('co-redis');
-  var logger = require('./logger');
-  var _client = redis.createClient(config.redis.port, config.redis.host);
-
-  _client.on('error', function (err) {
-    logger.error(err);
-  });
-
-  module.exports = wrapper(_client);
-
-} else {
-  console.warn('[%s] [worker:%s:common/redis.js] Redis config can not found',
-    Date(), process.pid);
-  module.exports = null;
-}

common/urllib.js

@@ -14,18 +14,58 @@
  * Module dependencies.
  */
 
+var urlparse = require('url').parse;
 var urllib = require('urllib');
 var HttpAgent = require('agentkeepalive');
 var HttpsAgent = require('agentkeepalive').HttpsAgent;
+var config = require('../config');
+
+var httpAgent;
+var httpsAgent;
+
+if (config.httpProxy) {
+  var tunnel = require('tunnel-agent');
+  var urlinfo = urlparse(config.httpProxy);
+  if (urlinfo.protocol === 'http:') {
+    httpAgent = tunnel.httpOverHttp({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+    httpsAgent = tunnel.httpsOverHttp({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+  } else if (urlinfo.protocol === 'https:') {
+    httpAgent = tunnel.httpOverHttps({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+    httpsAgent = tunnel.httpsOverHttps({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+  } else {
+    throw new TypeError('httpProxy format error: ' + config.httpProxy);
+  }
+} else {
+  httpAgent = new HttpAgent({
+    timeout: 0,
+    keepAliveTimeout: 15000
+  });
+  httpsAgent = new HttpsAgent({
+    timeout: 0,
+    keepAliveTimeout: 15000
+  });
+}
 
-var httpAgent = new HttpAgent({
-  timeout: 0,
-  keepAliveTimeout: 15000
-});
-var httpsAgent = new HttpsAgent({
-  timeout: 0,
-  keepAliveTimeout: 15000
-});
 var client = urllib.create({
   agent: httpAgent,
   httpsAgent: httpsAgent

config/index.js

@@ -6,7 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -24,6 +24,7 @@ var os = require('os');
 var version = require('../package.json').version;
 
 var root = path.dirname(__dirname);
+var dataDir = path.join(process.env.HOME || root, '.cnpmjs.org');
 
 var config = {
   version: version,
@@ -37,6 +38,7 @@ var config = {
   /*
    * server configure
    */
+
   registryPort: 7001,
   webPort: 7002,
   bindingHost: '127.0.0.1', // only binding on 127.0.0.1 for local access
@@ -44,15 +46,17 @@ var config = {
   // debug mode
   // if in debug mode, some middleware like limit wont load
   // logger module will print to stdout
-  debug: true,
+  debug: process.env.NODE_ENV === 'development',
+  // page mode, enable on development env
+  pagemock: process.env.NODE_ENV === 'development',
   // session secret
   sessionSecret: 'cnpmjs.org test session secret',
   // max request json body size
   jsonLimit: '10mb',
   // log dir name
-  logdir: path.join(root, '.tmp/logs'),
+  logdir: path.join(dataDir, 'logs'),
   // update file template dir
-  uploadDir: path.join(root, '.dist'),
+  uploadDir: path.join(dataDir, 'downloads'),
   // web page viewCache
   viewCache: false,
 
@@ -81,6 +85,7 @@ var config = {
   // email notification for errors
   // check https://github.com/andris9/Nodemailer for more informations
   mail: {
+    enable: false,
     appname: 'cnpmjs.org',
     from: 'cnpmjs.org mail sender <adderss@gmail.com>',
     service: 'gmail',
@@ -89,20 +94,9 @@ var config = {
       pass: 'your password'
     }
   },
-  // forward Compat with old style
-  // mail: {
-  //   appname: 'cnpmjs.org',
-  //   sender: 'cnpmjs.org mail sender <adderss@gmail.com>',
-  //   host: 'smtp.gmail.com',
-  //   port: 465,
-  //   user: 'address@gmail.com',
-  //   pass: 'your password',
-  //   ssl: true,
-  //   debug: false
-  // },
-
 
   logoURL: '//ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg', // cnpm logo image url
+  adBanner: '',
   customReadmeFile: '', // you can use your custom readme file instead the cnpm one
   customFooter: '', // you can add copyright and site total script html here
   npmClientName: 'cnpm', // use `${name} install package`
@@ -141,20 +135,18 @@ var config = {
     },
 
     // the storage engine for 'sqlite'
-    // default store into ~/cnpmjs.org.sqlite
-    storage: path.join(process.env.HOME || root, 'cnpmjs.org.sqlite'),
+    // default store into ~/.cnpmjs.org/data.sqlite
+    storage: path.join(dataDir, 'data.sqlite'),
 
     logging: !!process.env.SQL_DEBUG,
   },
 
-  // redis config
-  // use for koa-limit module as storage
-  redis: null,
-
   // package tarball store in local filesystem by default
   nfs: require('fs-cnpm')({
-    dir: path.join(root, '.tmp', 'dist')
+    dir: path.join(dataDir, 'nfs')
   }),
+  // if set true, will 302 redirect to `nfs.url(dist.key)`
+  downloadRedirectToNFS: false,
 
   // registry url name
   registryHost: 'r.cnpmjs.org',
@@ -163,18 +155,13 @@ var config = {
    * registry mode config
    */
 
-  // enable private mode, only admin can publish, other use just can sync package from source npm
-  enablePrivate: true,
+  // enable private mode or not
+  // private mode: only admins can publish, other users just can sync package from source npm
+  // public mode: all users can publish
+  enablePrivate: false,
 
   // registry scopes, if don't set, means do not support scopes
-  scopes: [
-    '@cnpm',
-    '@cnpmtest'
-  ],
-
-  // force user publish with scope
-  // but admins still can publish without scope
-  forcePublishWithScope: true,
+  scopes: [ '@cnpm', '@cnpmtest' ],
 
   // some registry already have some private packages in global scope
   // but we want to treat them as scoped private packages,
@@ -185,15 +172,6 @@ var config = {
    * sync configs
    */
 
-  // sync dist config
-  // sync node.js dist from nodejs.org
-  noticeSyncDistError: true,
-  disturl: 'http://nodejs.org/dist',
-  syncDist: false,
-
-  pythonDisturl: 'https://www.python.org/ftp',
-  syncPythonDist: false,
-
   // the official npm registry
   // cnpm wont directly sync from this one
   // but sometimes will request it for some package infomations
@@ -213,7 +191,7 @@ var config = {
   syncByInstall: true,
 
   // sync mode select
-  // none: do not sync any module
+  // none: do not sync any module, proxy all public modules from sourceNpmRegistry
   // exist: only sync exist modules
   // all: sync all modules
   syncModel: 'none', // 'none', 'all', 'exist'
@@ -235,12 +213,34 @@ var config = {
 
   // badge subject on http://shields.io/
   badgeSubject: 'cnpm',
+
+  // custom user service, @see https://github.com/cnpm/cnpmjs.org/wiki/Use-Your-Own-User-Authorization
+  userService: null,
+
+  // always-auth https://docs.npmjs.com/misc/config#always-auth
+  // Force npm to always require authentication when accessing the registry, even for GET requests.
+  alwaysAuth: false,
+
+  // if you're behind firewall, need to request through http proxy, please set this
+  // e.g.: `httpProxy: 'http://proxy.mycompany.com:8080'`
+  httpProxy: null,
 };
 
-// load config/config.js, everything in config.js will cover the same key in index.js
-var customConfig = path.join(root, 'config/config.js');
-if (fs.existsSync(customConfig)) {
-  copy(require(customConfig)).override(config);
+if (process.env.NODE_ENV !== 'test') {
+  var customConfig;
+  if (process.env.NODE_ENV === 'development') {
+    customConfig = path.join(root, 'config', 'config.js');
+  } else {
+    // 1. try to load `$dataDir/config.json` first, not exists then goto 2.
+    // 2. load config/config.js, everything in config.js will cover the same key in index.js
+    customConfig = path.join(dataDir, 'config.json');
+    if (!fs.existsSync(customConfig)) {
+      customConfig = path.join(root, 'config', 'config.js');
+    }
+  }
+  if (fs.existsSync(customConfig)) {
+    copy(require(customConfig)).override(config);
+  }
 }
 
 mkdirp.sync(config.logdir);

controllers/registry/package/dist_tag.js

@@ -0,0 +1,84 @@
+/**!
+ * cnpmjs.org - controllers/registry/package/dist_tag.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../../services/package');
+
+function ok() {
+  return {
+    ok: "dist-tags updated"
+  };
+}
+
+// GET /-/package/:pkg/dist-tags -- returns the package's dist-tags
+exports.index = function* () {
+  var name = this.params.name || this.params[0];
+  var rows = yield* packageService.listModuleTags(name);
+  var tags = {};
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    tags[row.tag] = row.version;
+  }
+  this.body = tags;
+};
+
+// PUT /-/package/:pkg/dist-tags -- Set package's dist-tags to provided object body (removing missing)
+exports.save = function* () {
+  var name = this.params.name || this.params[0];
+  yield packageService.removeModuleTags(name);
+  yield* exports.update.call(this);
+};
+
+// POST /-/package/:pkg/dist-tags -- Add/modify dist-tags from provided object body (merge)
+exports.update = function* () {
+  var name = this.params.name || this.params[0];
+  var tags = this.request.body;
+  for (var tag in tags) {
+    var version = tags[tag];
+    yield packageService.addModuleTag(name, tag, version);
+  }
+  this.status = 201;
+  this.body = ok();
+};
+
+// PUT /-/package/:pkg/dist-tags/:tag -- Set package's dist-tags[tag] to provided string body
+// POST /-/package/:pkg/dist-tags/:tag -- Same as PUT /-/package/:pkg/dist-tags/:tag
+exports.set = function* () {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  var version = this.request.body;
+  // make sure version exists
+  var pkg = yield packageService.getModule(name, version);
+  if (!pkg) {
+    this.status = 400;
+    this.body = {
+      error: 'version_error',
+      reason: name + '@' + version + ' not exists'
+    };
+    return;
+  }
+
+  yield packageService.addModuleTag(name, tag, version);
+  this.status = 201;
+  this.body = ok();
+};
+
+// DELETE /-/package/:pkg/dist-tags/:tag -- Remove tag from dist-tags
+exports.destroy = function* () {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  yield packageService.removeModuleTagsByNames(name, tag);
+  this.body = ok();
+};

controllers/registry/package/download.js

@@ -24,6 +24,7 @@ var common = require('../../../lib/common');
 var downloadAsReadStream = require('../../utils').downloadAsReadStream;
 var packageService = require('../../../services/package');
 var downloadTotalService = require('../../../services/download_total');
+var config = require('../../../config');
 
 var _downloads = {};
 
@@ -51,22 +52,25 @@ module.exports = function* download(next) {
     return;
   }
 
+  _downloads[name] = (_downloads[name] || 0) + 1;
+
+  if (config.downloadRedirectToNFS && url) {
+    this.status = 302;
+    this.set('Location', url);
+    return;
+  }
+
   var dist = row.package.dist;
   if (!dist.key) {
-    debug('get tarball by 302, url: %s', dist.tarball || url);
+    // try to use nsf.url() first
+    url = url || dist.tarball;
+    debug('get tarball by 302, url: %s', url);
     this.status = 302;
-    this.set('Location', dist.tarball || url);
-    _downloads[name] = (_downloads[name] || 0) + 1;
+    this.set('Location', url);
     return;
   }
 
   // else use `dist.key` to get tarball from nfs
-  if (!nfs.download) {
-    return yield* next;
-  }
-
-  _downloads[name] = (_downloads[name] || 0) + 1;
-
   if (typeof dist.size === 'number' && dist.size > 0) {
     this.length = dist.size;
   }

controllers/registry/package/download_total.js

@@ -0,0 +1,80 @@
+/*!
+ * cnpmjs.org - controllers/registry/package/download_total.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var DownloadTotal = require('../../../services/download_total');
+var DATE_REG = /^\d{4}-\d{2}-\d{2}$/;
+
+module.exports = function* downloadTotal () {
+  var range = this.params.range;
+  var name = this.params.name;
+
+  range = range.split(':');
+  if (range.length !== 2
+      || !range[0].match(DATE_REG)
+      || !range[1].match(DATE_REG)) {
+    this.status = 400;
+    this.body = {
+      error: 'range_error',
+      reason: 'range must be YYYY-MM-DD:YYYY-MM-DD style'
+    };
+    return;
+  }
+
+  this.body = name
+    ? yield getPackageTotal(name, range[0], range[1])
+    : yield getTotal(range[0], range[1]);
+};
+
+function* getPackageTotal(name, start, end) {
+  var res = yield DownloadTotal.getModuleTotal(name, start, end);
+  var downloads = res.map(function (row) {
+    return {
+      day: row.date,
+      downloads: row.count
+    };
+  });
+
+  downloads.sort(function (a, b) {
+    return a.day > b.day ? 1 : -1;
+  });
+
+  return {
+    downloads: downloads,
+    package: name,
+    start: start,
+    end: end
+  };
+}
+
+function* getTotal(start, end) {
+  var res = yield DownloadTotal.getTotal(start, end);
+  var downloads = res.map(function (row) {
+    return {
+      day: row.date,
+      downloads: row.count
+    };
+  });
+
+  downloads.sort(function (a, b) {
+    return a.day > b.day ? 1 : -1;
+  });
+
+  return {
+    downloads: downloads,
+    start: start,
+    end: end
+  };
+}

controllers/registry/package/list.js

@@ -16,10 +16,9 @@
 
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
 var packageService = require('../../../services/package');
-var npmService = require('../../../services/npm');
-var config = require('../../../config');
-var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var common = require('../../../lib/common');
 var SyncModuleWorker = require('../../sync_module_worker');
+var config = require('../../../config');
 
 /**
  * list all version of a module
@@ -72,7 +71,10 @@ module.exports = function* list() {
 
   var starUserMap = {};
   for (var i = 0; i < starUsers.length; i++) {
-    starUserMap[starUsers[i]] = true;
+    var starUser = starUsers[i];
+    if (starUser[0] !== '"' && starUser[0] !== "'") {
+      starUserMap[starUser] = true;
+    }
   }
   starUsers = starUserMap;
 
@@ -111,23 +113,7 @@ module.exports = function* list() {
     var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
     debug('start sync %s, get log id %s', name, logId);
 
-    // try to get package from official registry
-    var r = yield* npmService.request('/' + name, {
-      registry: config.officialNpmRegistry
-    });
-
-    debug('requet from officialNpmRegistry response %s', r.status);
-    if (r.status !== 200) {
-      this.status = 404;
-      this.body = {
-        error: 'not_found',
-        reason: 'document not found'
-      };
-      return;
-    }
-
-    this.body = r.data;
-    return;
+    return this.redirect(config.officialNpmRegistry + this.url);
   }
 
   var latestMod = null;
@@ -147,7 +133,7 @@ module.exports = function* list() {
   for (var i = 0; i < rows.length; i++) {
     var row = rows[i];
     var pkg = row.package;
-    setDownloadURL(pkg, this);
+    common.setDownloadURL(pkg, this);
     pkg._cnpm_publish_time = row.publish_time;
     versions[pkg.version] = pkg;
 

controllers/registry/package/show.js

@@ -17,7 +17,6 @@
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:show');
 var semver = require('semver');
 var packageService = require('../../../services/package');
-var npmService = require('../../../services/npm');
 var setDownloadURL = require('../../../lib/common').setDownloadURL;
 var SyncModuleWorker = require('../../sync_module_worker');
 var config = require('../../../config');
@@ -65,20 +64,5 @@ module.exports = function* show() {
   var logId = yield* SyncModuleWorker.sync(name, 'sync-by-install');
   debug('start sync %s, get log id %s', name, logId);
 
-  // rty to get package from official registry
-  var r = yield npmService.request('/' + name + '/' + tag, {
-    registry: config.officialNpmRegistry
-  });
-
-  if (r.status !== 200) {
-    debug('requet from officialNpmRegistry response %s', r.status);
-    this.status = 404;
-    this.body = {
-      error: 'not exist',
-      reason: 'tag or version not found: ' + tag
-    };
-    return;
-  }
-
-  this.body = r.data;
+  this.redirect(config.officialNpmRegistry + this.url);
 };

controllers/sync.js

@@ -22,10 +22,18 @@ var config = require('../config');
 exports.sync = function* () {
   var username = this.user.name || 'anonymous';
   var name = this.params.name;
+  var type = 'package';
+  if (name.indexOf(':') > 0) {
+    // user:fengmk2
+    // package:pedding
+    var splits = name.split(':');
+    type = splits[0];
+    name = splits[1];
+  }
   var publish = this.query.publish === 'true';
   var noDep = this.query.nodeps === 'true';
   debug('sync %s with query: %j', name, this.query);
-  if (publish && !this.user.isAdmin) {
+  if (type === 'package' && publish && !this.user.isAdmin) {
     this.status = 403;
     this.body = {
       error: 'no_perms',
@@ -35,6 +43,7 @@ exports.sync = function* () {
   }
 
   var options = {
+    type: type,
     publish: publish,
     noDep: noDep,
     syncUpstreamFirst: config.sourceNpmRegistryIsCNpm,

controllers/sync_module_worker.js

@@ -37,8 +37,10 @@ var npmSerivce = require('../services/npm');
 var packageService = require('../services/package');
 var logService = require('../services/module_log');
 var User = require('../models').User;
+var os = require('os');
 
-var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+var USER_AGENT = 'sync.cnpmjs.org/' + config.version +
+  ' hostname/' + os.hostname() + ' ' + urllib.USER_AGENT;
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
@@ -49,6 +51,7 @@ function SyncModuleWorker(options) {
     options.name = [options.name];
   }
 
+  this.type = options.type || 'package';
   this.names = options.name;
   this.startName = this.names[0];
 
@@ -79,8 +82,9 @@ SyncModuleWorker.prototype.finish = function () {
     return;
   }
   this._finished = true;
-  this.log('[done] Sync %s module finished, %d success, %d fail\nSuccess: [ %s ]\nFail: [ %s ]',
+  this.log('[done] Sync %s %s finished, %d success, %d fail\nSuccess: [ %s ]\nFail: [ %s ]',
     this.startName,
+    this.type,
     this.successes.length, this.fails.length,
     this.successes.join(', '), this.fails.join(', '));
   this.emit('end');
@@ -112,10 +116,13 @@ SyncModuleWorker.prototype._saveLog = function () {
   that._log = '';
   co(function* () {
     yield* logService.append(that._logId, logstr);
-  })(function (err) {
-    if (err) {
-      logger.error(err);
+  }).then(function () {
+    that._loging = false;
+    if (that._log) {
+      that._saveLog();
     }
+  }).catch(function (err) {
+    logger.error(err);
     that._loging = false;
     if (that._log) {
       that._saveLog();
@@ -127,7 +134,7 @@ SyncModuleWorker.prototype.start = function () {
   this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
     this.username, this.names[0], this.concurrency, this.noDep, this._publish);
   var that = this;
-  co(function *() {
+  co(function* () {
     // sync upstream
     if (that.syncUpstreamFirst) {
       try {
@@ -137,12 +144,19 @@ SyncModuleWorker.prototype.start = function () {
       }
     }
 
+    if (that.type === 'user') {
+      yield that.syncUser();
+      return;
+    }
+
     var arr = [];
     for (var i = 0; i < that.concurrency; i++) {
       arr.push(that.next(i));
     }
     yield arr;
-  })();
+  }).catch(function (err) {
+    logger.error(err);
+  });
 };
 
 SyncModuleWorker.prototype.pushSuccess = function (name) {
@@ -182,7 +196,11 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
 };
 
 SyncModuleWorker.prototype.syncUpstream = function* (name) {
-  var url = config.sourceNpmRegistry + '/' + name + '/sync';
+  var syncname = name;
+  if (this.type === 'user') {
+    syncname = this.type + ':' + syncname;
+  }
+  var url = config.sourceNpmRegistry + '/' + syncname + '/sync';
   if (this.noDep) {
     url += '?nodeps=true';
   }
@@ -192,7 +210,8 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
     headers: {
       'content-length': 0
     },
-    dataType: 'json'
+    dataType: 'json',
+    gzip: true,
   });
 
   if (r.status !== 201 || !r.data.ok) {
@@ -210,7 +229,8 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
     var synclogURL = logURL + '?offset=' + offset;
     var rs = yield urllib.request(synclogURL, {
       timeout: 20000,
-      dataType: 'json'
+      dataType: 'json',
+      gzip: true,
     });
 
     if (rs.status !== 200 || !rs.data.ok) {
@@ -248,7 +268,28 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
   this.log('----------------- Synced upstream %s -------------------', logURL);
 };
 
+SyncModuleWorker.prototype.syncUser = function* () {
+  for (var i = 0; i < this.names.length; i++) {
+    var username = this.names[i];
+    try {
+      var user = yield _saveNpmUser(username);
+      this.pushSuccess(username);
+      this.log('[c#%s] [%s] sync success: %j', 0, username, user);
+    } catch (err) {
+      this.pushFail(username);
+      this.log('[c#%s] [error] [%s] sync error: %s', 0, username, err.stack);
+    }
+  }
+  this.finish();
+};
+
 SyncModuleWorker.prototype.next = function* (concurrencyId) {
+  if (config.syncModel === 'none') {
+    this.log('[c#%d] [%s] syncModel is none, ignore',
+      concurrencyId, name);
+    return this.finish();
+  }
+
   var name = this.names.shift();
   if (!name) {
     return setImmediate(this.finish.bind(this));
@@ -344,6 +385,7 @@ function* _saveNpmUser(username) {
     return;
   }
   yield* User.saveNpmUser(user);
+  return user;
 }
 
 function* _saveMaintainer(modName, username, action) {
@@ -358,7 +400,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   var mods = yield* packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
-  if (this._isLocalModule(mods)) {
+  if (common.isLocalModule(mods)) {
     // publish on cnpm, dont sync this version package
     this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -396,16 +438,6 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   this.log('    [%s] delete nfs files: %j success', name, keys);
 };
 
-SyncModuleWorker.prototype._isLocalModule = function (mods) {
-  for (var i = 0; i < mods.length; i++) {
-    var r = mods[i];
-    if (r.package && r.package._publish_on_cnpm) {
-      return true;
-    }
-  }
-  return false;
-};
-
 SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var that = this;
   var hasModules = false;
@@ -420,7 +452,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var existsStarUsers = result[2];
   var existsNpmMaintainers = result[3];
 
-  if (that._isLocalModule(moduleRows)) {
+  if (common.isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
     that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -760,7 +792,9 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     names.forEach(function (username) {
       var r = map[username];
       if (!r || !r.json) {
-        missingUsers.push(username);
+        if (username[0] !== '"' && username[0] !== "'") {
+          missingUsers.push(username);
+        }
       }
     });
 
@@ -848,7 +882,8 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     timeout: 600000, // 10 minutes download
     headers: {
       'user-agent': USER_AGENT
-    }
+    },
+    gzip: true,
   };
 
   var dependencies = Object.keys(sourcePackage.dependencies || {});
@@ -939,6 +974,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       shasum: shasum
     };
     // upload to NFS
+    logger.syncInfo('[sync_module_worker] uploading %j to nfs', options);
     var result = yield nfs.upload(filepath, options);
     return yield *afterUpload(result);
   } finally {
@@ -1002,6 +1038,7 @@ SyncModuleWorker.sync = function* (name, username, options) {
   var result = yield* logService.create({name: name, username: username});
   var worker = new SyncModuleWorker({
     logId: result.id,
+    type: options.type,
     name: name,
     username: username,
     noDep: options.noDep,

controllers/utils.js

@@ -27,6 +27,11 @@ var config = require('../config');
 var DOWNLOAD_TIMEOUT = ms('10m');
 
 exports.downloadAsReadStream = function* (key) {
+  var options = { timeout: DOWNLOAD_TIMEOUT };
+  if (nfs.createDownloadStream) {
+    return yield nfs.createDownloadStream(key, options);
+  }
+
   var tmpPath = path.join(config.uploadDir,
     utility.randomString() + key.replace(/\//g, '-'));
   function cleanup() {
@@ -35,7 +40,7 @@ exports.downloadAsReadStream = function* (key) {
   }
   debug('downloadAsReadStream() %s to %s', key, tmpPath);
   try {
-    yield nfs.download(key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
+    yield nfs.download(key, tmpPath, options);
   } catch (err) {
     debug('downloadAsReadStream() %s to %s error: %s', key, tmpPath, err.stack);
     cleanup();

controllers/web/dist.js

@@ -1,100 +0,0 @@
-/**!
- * cnpmjs.org - controllers/web/dist.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:controllers:web:dist');
-var mime = require('mime');
-var urlparse = require('url').parse;
-var distService = require('../../services/dist');
-var config = require('../../config');
-var downloadAsReadStream = require('../utils').downloadAsReadStream;
-
-function padding(max, current, pad) {
-  pad = pad || ' ';
-  var left = max - current;
-  var str = '';
-  for (var i = 0; i < left; i++) {
-    str += pad;
-  }
-  return str;
-}
-
-exports.list = function* (next) {
-  var params = this.params;
-  var url = params[0];
-  if (!url) {
-    // GET /dist => /dist/
-    return this.redirect('/dist/');
-  }
-
-  var isDir = url[url.length - 1] === '/';
-  if (!isDir) {
-    return yield* download.call(this, next);
-  }
-
-  var items = yield* distService.listdir(url);
-  if (url === '/') {
-    // phantomjs/
-    items.push({
-      name: 'phantomjs/',
-      date: '',
-    });
-  }
-
-  yield this.render('dist', {
-    title: 'Mirror index of ' + config.disturl + url,
-    disturl: config.disturl,
-    dirname: url,
-    items: items,
-    padding: padding
-  });
-};
-
-function* download(next) {
-  var fullname = this.params[0];
-  var info = yield* distService.getfile(fullname);
-  debug('download %s got %j', fullname, info);
-  if (!info || !info.url) {
-    return yield* next;
-  }
-
-  if (/\.(html|js|css|json|txt)$/.test(fullname)) {
-    if (info.url.indexOf('http') === 0) {
-      info.url = urlparse(info.url).path;
-    }
-    return yield* pipe.call(this, info, false);
-  }
-
-  if (info.url.indexOf('http') === 0) {
-    return this.redirect(info.url);
-  }
-  yield* pipe.call(this, info, true);
-}
-
-function* pipe(info, attachment) {
-  debug('pipe %j, attachment: %s', info, attachment);
-  // download it from nfs
-  if (typeof info.size === 'number' && info.size > 0) {
-    this.length = info.size;
-  }
-  this.type = mime.lookup(info.url);
-  if (attachment) {
-    this.attachment(info.name);
-  }
-  if (info.sha1) {
-    this.etag = info.sha1;
-  }
-  this.body = yield* downloadAsReadStream(info.url);
-}

controllers/web/package/search.js

@@ -6,7 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -35,13 +35,12 @@ module.exports = function* search() {
 
   // return a json result
   if (this.query && this.query.type === 'json') {
-    this.body = {
+    this.jsonp = {
       keyword: word,
       match: match,
       packages: result.searchMatchs,
       keywords: result.keywordMatchs,
     };
-    this.type = 'application/json; charset=utf-8';
     return;
   }
   yield this.render('search', {

controllers/web/package/show.js

@@ -20,12 +20,12 @@ var bytes = require('bytes');
 var giturl = require('giturl');
 var moment = require('moment');
 var semver = require('semver');
-var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
 var config = require('../../../config');
 var utils = require('../../utils');
 var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var renderMarkdown = require('../../../common/markdown').render;
 var packageService = require('../../../services/package');
 
 module.exports = function* show(next) {
@@ -90,7 +90,12 @@ module.exports = function* show(next) {
   pkg.package.fromNow = moment(pkg.publish_time).fromNow();
   pkg = pkg.package;
   pkg.users = users;
-  pkg.readme = marked(pkg.readme || '');
+  if (pkg.readme && typeof pkg.readme !== 'string') {
+    pkg.readme = 'readme is not string: ' + JSON.stringify(pkg.readme);
+  } else {
+    pkg.readme = renderMarkdown(pkg.readme || '');
+  }
+
   if (!pkg.readme) {
     pkg.readme = pkg.description || '';
   }
@@ -115,22 +120,6 @@ module.exports = function* show(next) {
     }
   }
 
-  if (pkg.contributors) {
-    // registry.cnpmjs.org/compressible
-    if (!Array.isArray(pkg.contributors)) {
-      pkg.contributors = [pkg.contributors];
-    }
-    for (var i = 0; i < pkg.contributors.length; i++) {
-      var contributor = pkg.contributors[i];
-      if (contributor.email) {
-        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, true);
-      }
-      if (config.packagePageContributorSearch || !contributor.url) {
-        contributor.url = '/~' + encodeURIComponent(contributor.name);
-      }
-    }
-  }
-
   if (pkg.repository === 'undefined') {
     pkg.repository = null;
   }

controllers/web/show_sync.js

@@ -1,5 +1,5 @@
 /**!
- * cnpmjs.org - controllers/web/package/show_sync.js
+ * cnpmjs.org - controllers/web/show_sync.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -20,8 +20,15 @@ module.exports = function* showSync() {
   if (!name) {
     return this.redirect('/');
   }
+  var type = 'package';
+  if (name.indexOf(':') > 0) {
+    var splits = name.split(':');
+    name = splits[1];
+    type = splits[0];
+  }
   yield this.render('sync', {
+    type: type,
     name: name,
-    title: 'Sync - ' + name,
+    title: 'Sync ' + type + ' - ' + name,
   });
 };

dispatch.js

@@ -23,6 +23,9 @@ var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
 var syncPath = path.join(__dirname, 'sync');
 
+console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s',
+  config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel);
+
 if (config.enableCluster) {
   forkWorker();
   if (config.syncModel !== 'none') {

docs/Migrating-from-1.x-to-2.x.md

@@ -0,0 +1,143 @@
+# Migrating from 1.x to 2.x
+
+2.x using [Sequelize] ORM to supports MySQL, MariaDB, SQLite or PostgreSQL databases.
+
+## New download total table structure
+
+### Create `downloads` table SQL
+
+You should create `downloads` table first:
+
+```sql
+CREATE TABLE IF NOT EXISTS `downloads` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+  `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
+  `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
+  `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
+  `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
+  `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
+  `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
+  `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
+  `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
+  `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
+  `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
+  `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
+  `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
+  `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
+  `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
+  `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
+  `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
+  `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
+  `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
+  `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
+  `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
+  `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
+  `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
+  `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
+  `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
+  `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
+  `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
+  `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
+  `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
+  `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
+  `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
+  `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
+  `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_date` (`name`, `date`),
+  KEY `date` (`date`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+```
+
+### Sync `download_total` to `downloads`
+
+Then use [sync_download_total.js](../tools/sync_download_total.js) scrpt to sync datas from `download_total`:
+
+```bash
+$ node --harmony tools/sync_download_total.js
+```
+
+# `config.js` changes in 2.x
+
+## New database config
+
+```js
+/**
+* database config
+*/
+
+database: {
+  db: 'cnpmjs_test',
+  username: 'root',
+  password: '',
+
+  // the sql dialect of the database
+  // - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
+  dialect: 'sqlite',
+
+  // custom host; default: 127.0.0.1
+  host: '127.0.0.1',
+
+  // custom port; default: 3306
+  port: 3306,
+
+  // use pooling in order to reduce db connection overload and to increase speed
+  // currently only for mysql and postgresql (since v1.5.0)
+  pool: {
+    maxConnections: 10,
+    minConnections: 0,
+    maxIdleTime: 30000
+  },
+
+  // the storage engine for 'sqlite'
+  // default store into ~/cnpmjs.org.sqlite
+  storage: path.join(process.env.HOME || root, 'cnpmjs.org.sqlite'),
+
+  logging: !!process.env.SQL_DEBUG,
+},
+```
+
+If you're still using MySQL and old config.js `mysqlServers: []` from 1.x:
+
+```js
+mysqlServers: [
+{
+  host: '127.0.0.1',
+  port: 3306,
+  user: 'root',
+  password: ''
+}
+],
+mysqlDatabase: 'cnpmjs_test',
+mysqlMaxConnections: 4,
+mysqlQueryTimeout: 5000,
+```
+
+We will do forward compat, and auto change old style config.js to:
+
+```js
+database: {
+  db: 'cnpmjs_test',
+  username: 'root',
+  password: '',
+  dialect: 'mysql',
+  host: '127.0.0.1',
+  port: 3306,
+  pool: {
+    maxConnections: 10,
+    minConnections: 0,
+    maxIdleTime: 30000
+    },
+  logging: !!process.env.SQL_DEBUG,
+},
+```
+
+## remove `adaptScope`
+
+`adaptScope: true | false` feature was removed.
+
+
+[Sequelize]: http://sequelizejs.com/

docs/db.sql

@@ -151,17 +151,60 @@ INSERT INTO total(name, gmt_modified) VALUES('total', now())
 -- ALTER TABLE `total` ADD `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync'
 -- ALTER TABLE `total` ADD `last_sync_module` varchar(100) NOT NULL COMMENT 'last sync success module name';
 
-CREATE TABLE IF NOT EXISTS `download_total` (
- `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
- `gmt_create` datetime NOT NULL COMMENT 'create time',
- `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
- `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
- `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
- PRIMARY KEY (`id`),
- UNIQUE KEY `date_name` (`date`, `name`)
+-- CREATE TABLE IF NOT EXISTS `download_total` (
+--  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+--  `gmt_create` datetime NOT NULL COMMENT 'create time',
+--  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+--  `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format',
+--  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
+--  PRIMARY KEY (`id`),
+--  UNIQUE KEY `date_name` (`date`, `name`)
+-- ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE  `download_total` CHANGE `date` `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format';
+
+CREATE TABLE IF NOT EXISTS `downloads` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+  `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
+  `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
+  `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
+  `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
+  `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
+  `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
+  `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
+  `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
+  `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
+  `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
+  `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
+  `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
+  `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
+  `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
+  `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
+  `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
+  `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
+  `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
+  `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
+  `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
+  `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
+  `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
+  `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
+  `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
+  `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
+  `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
+  `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
+  `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
+  `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
+  `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
+  `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
+  `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_date` (`name`, `date`),
+  KEY `date` (`date`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
--- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE IF NOT EXISTS `module_deps` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',

docs/registry-api.md

@@ -19,22 +19,20 @@ All data is sent and received as JSON.
 $ curl -i https://registry.npmjs.org
 
 HTTP/1.1 200 OK
-Date: Tue, 05 Aug 2014 10:53:24 GMT
-Server: CouchDB/1.5.0 (Erlang OTP/R16B03)
-Content-Type: text/plain; charset=utf-8
-Cache-Control: max-age=60
-Content-Length: 258
-Accept-Ranges: bytes
-Via: 1.1 varnish
-Age: 11
-X-Served-By: cache-ty67-TYO
-X-Cache: HIT
-X-Cache-Hits: 1
-X-Timer: S1407236004.867906,VS0,VE0
 
-{"db_name":"registry","doc_count":90789,"doc_del_count":381,"update_seq":137250,"purge_seq":0,
-"compact_running":false,"disk_size":436228219,"data_size":332875061,
-"instance_start_time":"1405721973718703","disk_format_version":6,"committed_update_seq":137250}
+{
+  "db_name": "registry",
+  "doc_count": 123772,
+  "doc_del_count": 377,
+  "update_seq": 685591,
+  "purge_seq": 0,
+  "compact_running": false,
+  "disk_size": 634187899,
+  "data_size": 445454185,
+  "instance_start_time": "1420670152481614",
+  "disk_format_version": 6,
+  "committed_update_seq": 685591
+}
 ```
 
 ## Client Errors
@@ -61,22 +59,10 @@ $ curl -u "username:password" https://registry.npmjs.org
 ## Failed login limit
 
 ```bash
-$ curl -i -X PUT -u foo:pwd \
-  -d '{"name":"foo","email":"foo@bar.com","type":"user","roles":[]}' \
-  https://registry.npmjs.org/-/user/org.couchdb.user:foo/-rev/11-d226c6afa9286ab5b9eb858c429bdabf
+$ curl -i -X "GET" -u "foo:pwd" \
+    "https://registry.npmjs.com/-/user/org.couchdb.user:npm-user-service-testuser?write=true"
 
 HTTP/1.1 401 Unauthorized
-Date: Tue, 05 Aug 2014 15:33:25 GMT
-Server: CouchDB/1.5.0 (Erlang OTP/R14B04)
-Content-Type: text/plain; charset=utf-8
-Cache-Control: max-age=60
-Content-Length: 67
-Accept-Ranges: bytes
-Via: 1.1 varnish
-X-Served-By: cache-ty66-TYO
-X-Cache: MISS
-X-Cache-Hits: 0
-X-Timer: S1407252805.261390,VS0,VE434
 
 {"error":"unauthorized","reason":"Name or password is incorrect."}
 ```
@@ -101,14 +87,11 @@ X-Timer: S1407252805.261390,VS0,VE434
 GET /:package
 ```
 
-#### Response
+#### Response 200
 
 ```json
 HTTP/1.1 200 OK
 Etag: "8UDCP753LFXOG42NMX88JAN40"
-Content-Type: application/json
-Cache-Control: max-age=60
-Content-Length: 2243
 
 {
   "_id": "pedding",
@@ -246,6 +229,17 @@ Content-Length: 2243
 }
 ```
 
+#### Response 404
+
+```json
+HTTP/1.1 404 Object Not Found
+
+{
+  "error": "not_found",
+  "reason": "document not found"
+}
+```
+
 ### ~~Get a special version or tag package~~
 
 __deprecated__
@@ -254,7 +248,7 @@ __deprecated__
 GET /:package/:tag_or_version
 ```
 
-#### Reponse
+#### Reponse 200
 
 ```json
 HTTP/1.1 200 OK
@@ -745,9 +739,52 @@ HTTP/1.1 200 OK
 
 ## User
 
-* [Get a single user](/docs/registry-api.md#get-a-single-user)
-* [Add a new user](/docs/registry-api.md#add-a-new-user)
-* [Update a exists user](/docs/registry-api.md#update-a-exists-user)
+- [Auth user](/docs/registry-api.md#auth-user)
+- [Get a single user](/docs/registry-api.md#get-a-single-user)
+- [Add a new user](/docs/registry-api.md#add-a-new-user)
+- [Update a exists user](/docs/registry-api.md#update-a-exists-user)
+
+### Auth user
+
+* Authentication required.
+
+```
+GET /-/user/org.couchdb.user::username?write=true
+```
+
+#### Response 200
+
+```json
+HTTP/1.1 200 OK
+ETag: "5-a31b61ba3c50b50f7fcaf185e079e17a"
+
+{
+  "_id": "org.couchdb.user:npm-user-service-testuser",
+  "_rev": "5-a31b61ba3c50b50f7fcaf185e079e17a",
+  "password_scheme": "pbkdf2",
+  "iterations": 10,
+  "name": "npm-user-service-testuser",
+  "email": "fengmk2@gmail.com",
+  "type": "user",
+  "roles": [],
+  "date": "2015-01-04T08:28:51.378Z",
+  "password_scheme": "pbkdf2",
+  "iterations": 10,
+  "derived_key": "644157c126b93356e6eba2c59fdf1b7ec644ebf2",
+  "salt": "5d13874c0aa10751e35743bacd6eedd5"
+}
+```
+
+#### Response 401
+
+```json
+HTTP/1.1 401 Unauthorized
+
+{
+  "error": "unauthorized",
+  "reason": "Name or password is incorrect."
+}
+```
 
 ### Get a single user
 
@@ -755,7 +792,7 @@ HTTP/1.1 200 OK
 GET /-/user/org.couchdb.user::username
 ```
 
-#### Response
+#### Response 200
 
 ```json
 HTTP/1.1 200 OK
@@ -825,6 +862,17 @@ ETag: "32-984ee97e01aea166dcab6d1517c730e3"
 }
 ```
 
+#### Response 404
+
+```json
+HTTP/1.1 404 Object Not Found
+
+{
+  "error": "not_found",
+  "reason": "missing"
+}
+```
+
 ### Add a new user
 
 ```
@@ -845,7 +893,7 @@ PUT /-/user/org.couchdb.user::username
 }
 ```
 
-#### Response
+#### Response 201
 
 ```json
 Status: 201 Created
@@ -857,6 +905,19 @@ Status: 201 Created
 }
 ```
 
+#### Response 409
+
+User already exists
+
+```json
+HTTP/1.1 409 Conflict
+
+{
+  "error": "conflict",
+  "reason": "Document update conflict."
+}
+```
+
 ### Update a exists user
 
 * Authentication required.

docs/web/readme.md

@@ -4,11 +4,10 @@ So `cnpm` is meaning: **Company npm**.
 
 ## Registry
 
-* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
-* Current [cnpmjs.org](/) version: <span id="app-version"></span>
-* Mirror of [Node.js Manual & Documentation](/dist/latest/docs/api/index.html)
-* Mirror of [nodejs.org/dist](http://nodejs.org/dist): [/dist mirror](/dist)
-* Mirror of [phantomjs downloads](https://bitbucket.org/ariya/phantomjs/downloads): [phantomjs mirror](/dist/phantomjs/)
+- Our public registry: [r.cnpmjs.org](//r.cnpmjs.org), syncing from [registry.npmjs.org](//registry.npmjs.org)
+- [cnpmjs.org](/) version: <span id="app-version"></span>
+- [Node.js](https://nodejs.org) version: <span id="node-version"></span>
+- For developers behind the GFW, please visit [the Chinese mirror](https://npm.taobao.org). 中国用户请访问[国内镜像站点](https://npm.taobao.org/)。
 
 <table class="downloads">
   <tbody>
@@ -51,15 +50,13 @@ So `cnpm` is meaning: **Company npm**.
   </table>
 </div>
 
-Running on [Node.js](http://nodejs.org), version <span id="node-version"></span>.
-
 <script src="/js/readme.js"></script>
 
 ## Version Badge
 
 Default style is `flat-square`.
 
-Badge URL: `http://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-badge](http://cnpmjs.org/badge/v/cnpmjs.org.svg)
+Badge URL: `https://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-badge](//cnpmjs.org/badge/v/cnpmjs.org.svg)
 
 * `<0.1.0 & >=0.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.0.1-red.svg?style=flat-square)
 * `<1.0.0 & >=0.1.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.1.0-green.svg?style=flat-square)
@@ -69,30 +66,30 @@ Badge URL: `http://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-badge](http:/
 
 use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
 
-```
-npm install -g cnpm --registry=http://r.cnpmjs.org
+```bash
+$ npm install -g cnpm --registry=https://r.cnpmjs.org
 ```
 
 Or you can alias NPM to use it:
 
 ```bash
-alias cnpm="npm --registry=http://r.cnpmjs.org \
+alias cnpm="npm --registry=https://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=http://cnpmjs.org/dist \
+--disturl=https://cnpmjs.org/mirrors/node \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=http://cnpmjs.org/dist \
+  --disturl=https://cnpmjs.org/mirrors/node \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
 ### install
 
-Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](//r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](//registry.npmjs.org)), and sync this package to cnpm in the backend.
 
-```
+```bash
 $ cnpm install [name]
 ```
 
@@ -104,10 +101,10 @@ Only `cnpm` cli has this command. Meaning sync package from source npm.
 $ cnpm sync connect
 ```
 
-sync package on web: [cnpmjs.org/sync/connect](http://cnpmjs.org/sync/connect)
+sync package on web: [sync/connect](/sync/connect)
 
 ```bash
-$ open http://cnpmjs.org/sync/connect
+$ open https://cnpmjs.org/sync/connect
 ```
 
 ### publish / unpublish
@@ -137,4 +134,8 @@ Release [History](/history).
 
 ## npm and cnpm relation
 
-![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
+![npm&cnpm](https://dn-cnpm.qbox.me/cnpm-npm-relation.png)
+
+## Sponsors
+
+- [![UCloud云计算](https://www.ucloud.cn/www/img/www/logo.png)](http://www.ucloud.cn/sdk?sem=sdk-CNPMJS)

lib/common.js

@@ -56,3 +56,13 @@ exports.isMaintainer = function (user, maintainers) {
 
   return match.length > 0;
 };
+
+exports.isLocalModule = function (mods) {
+  for (var i = 0; i < mods.length; i++) {
+    var r = mods[i];
+    if (r.package && r.package._publish_on_cnpm) {
+      return true;
+    }
+  }
+  return false;
+};

middleware/auth.js

@@ -16,6 +16,7 @@
 
 var debug = require('debug')('cnpmjs.org:middleware:auth');
 var UserService = require('../services/user');
+var config = require('../config');
 
 /**
  * Parse the request authorization
@@ -30,12 +31,12 @@ module.exports = function () {
     authorization = authorization.trim();
     debug('%s %s with %j', this.method, this.url, authorization);
     if (!authorization) {
-      return yield* next;
+      return yield* unauthorized.call(this, next);
     }
 
     authorization = new Buffer(authorization, 'base64').toString().split(':');
     if (authorization.length !== 2) {
-      return yield* next;
+      return yield* unauthorized.call(this, next);
     }
 
     var username = authorization[0];
@@ -52,7 +53,7 @@ module.exports = function () {
 
     if (!row) {
       debug('auth fail user: %j, headers: %j', row, this.header);
-      return yield* next;
+      return yield* unauthorized.call(this, next);
     }
 
     this.user.name = row.login;
@@ -62,3 +63,19 @@ module.exports = function () {
     yield* next;
   };
 };
+
+function* unauthorized(next) {
+  if (!config.alwaysAuth || this.method !== 'GET') {
+    return yield* next;
+  }
+  this.status = 401;
+  this.set('WWW-Authenticate', 'Basic realm="sample"');
+  if (this.accepts(['html', 'json']) === 'json') {
+    this.body = {
+      error: 'unauthorized',
+      reason: 'login first'
+    };
+  } else {
+    this.body = 'login first';
+  }
+}

middleware/exists_package.js

@@ -0,0 +1,30 @@
+/**!
+ * cnpmjs.org - middleware/exists_package.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../services/package');
+
+module.exports = function* (next) {
+  var name = this.params.name || this.params[0];
+  var pkg = yield packageService.getLatestModule(name);
+  if (pkg) {
+    return yield* next;
+  }
+  this.status = 404;
+  this.body = {
+    error: 'not_found',
+    reason: 'document not found'
+  };
+};

middleware/limit.js

@@ -16,7 +16,6 @@
 
 var config = require('../config');
 var limit = require('koa-limit');
-var store = require('../common/redis');
 
 var limitConfig = config.limit;
 
@@ -25,10 +24,5 @@ if (!limitConfig.enable) {
     yield *next;
   };
 } else {
-
-  if (!config.debug) {
-    limitConfig.store = store;
-  }
-
   module.exports = limit(limitConfig);
 }

middleware/login.js

@@ -34,7 +34,7 @@ module.exports = function *login(next) {
     this.status = 401;
     this.body = {
       error: 'unauthorized',
-      reason: 'Login first.'
+      reason: 'Login first'
     };
     return;
   }

middleware/opensearch.js

@@ -22,7 +22,7 @@ var template = '<?xml version="1.0" encoding="UTF-8"?>\
     <Url method="get" type="text/html" template="http://${host}/browse/keyword/{searchTerms}"/>\
  </OpenSearchDescription>';
 
-module.exports = function *publishable(next) {
+module.exports = function *opensearch(next) {
   if (this.path === '/opensearch.xml') {
     this.type = 'text/xml';
     this.charset = 'utf-8';

middleware/proxy_to_npm.js

@@ -0,0 +1,59 @@
+/**!
+ * cnpmjs.org - middleware/proxy_to_npm.js
+ *
+ * Copyright(c) Alibaba Group Holding Limited.
+ *
+ * Authors:
+ *   苏千 <suqian.yf@alipay.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:middleware:proxy_to_npm');
+var config = require('../config');
+
+module.exports = function (options) {
+  var redirectUrl = config.sourceNpmRegistry;
+  var proxyUrls = [
+    // /:pkg, dont contains scoped package
+    /^\/[\w\-\.]+$/,
+    // /-/package/:pkg/dist-tags
+    /^\/\-\/package\/[\w\-\.]+\/dist-tags/,
+  ];
+  if (options && options.isWeb) {
+    redirectUrl = redirectUrl.replace('//registry.', '//');
+    proxyUrls = [
+      // /package/:pkg
+      /^\/package\/[\w\-\.]+$/,
+    ];
+  }
+  return function* proxyToNpm(next) {
+    if (config.syncModel !== 'none') {
+      return yield* next;
+    }
+    // only proxy read requests
+    if (this.method !== 'GET' && this.method !== 'HEAD') {
+      return yield* next;
+    }
+
+    var pathname = this.path;
+    var match;
+    for (var i = 0; i < proxyUrls.length; i++) {
+      match = proxyUrls[i].test(pathname);
+      if (match) {
+        break;
+      }
+    }
+    if (!match) {
+      return yield* next;
+    }
+
+    var url = redirectUrl + this.url;
+    debug('proxy to %s', url);
+    this.redirect(url);
+  };
+};

middleware/publishable.js

@@ -29,9 +29,8 @@ module.exports = function *publishable(next) {
     return;
   }
 
-  // public mode, all user have permission to publish
-  // but if `config.scopes` exist, only can publish with scopes in `config.scope`
-  // if `config.forcePublishWithScope` set to true, only admins can publish without scope
+  // public mode, all user have permission to publish `scoped package`
+  // and only can publish with scopes in `ctx.user.scopes`, default is `config.scopes`
 
   var name = this.params.name || this.params[0];
 
@@ -49,7 +48,7 @@ module.exports = function *publishable(next) {
   }
 
   // none-scope
-  if (checkNoneScope(this)) {
+  if (checkNoneScope(name, this)) {
     return yield* next;
   }
 };
@@ -82,19 +81,21 @@ function checkScope(name, ctx) {
  * check if user have permission to publish without scope
  */
 
-function checkNoneScope(ctx) {
-  if (!config.scopes
-    || !config.scopes.length
-    || !config.forcePublishWithScope) {
-    return true;
-  }
-
-  // only admins can publish or unpublish non-scope modules
-  if (ctx.user.isAdmin) {
+function checkNoneScope(name, ctx) {
+  // admins unpublished everything
+  if (ctx.user.isAdmin && ctx.method === 'DELETE') {
     return true;
   }
 
   ctx.status = 403;
+  if (ctx.user.scopes.length === 0) {
+    ctx.body = {
+      error: 'no_perms',
+      reason: 'can\'t publish non-scoped package, please set `config.scopes`'
+    };
+    return;
+  }
+
   ctx.body = {
     error: 'no_perms',
     reason: 'only allow publish with ' + ctx.user.scopes.join(', ') + ' scope(s)'

middleware/sync_by_install.js

@@ -22,17 +22,17 @@ var config = require('../config');
 
 module.exports = function* syncByInstall(next) {
   this.allowSync = false;
-  if (!config.syncByInstall || !config.enablePrivate) {
+  if (!config.syncByInstall) {
     // only config.enablePrivate should enable sync on install
     return yield* next;
   }
-  // request not by node, consider it request from web
+  // request not by node, consider it request from web, dont sync
   var ua = this.get('user-agent');
   if (!ua || ua.indexOf('node') < 0) {
     return yield* next;
   }
 
-  // if request with `/xxx?write=true`, meaning the read request using for write
+  // if request with `/xxx?write=true`, meaning the read request using for write, dont sync
   if (this.query.write) {
     return yield* next;
   }

models/dist_dir.js

@@ -1,64 +0,0 @@
-/**!
- * cnpmjs.org - models/dist_dir.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-/*
-CREATE TABLE IF NOT EXISTS `dist_dir` (
- `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
- `gmt_create` datetime NOT NULL COMMENT 'create time',
- `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(200) NOT NULL COMMENT 'dir name',
- `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
- `date` varchar(20) COMMENT '02-May-2014 01:06',
- PRIMARY KEY (`id`),
- UNIQUE KEY `dist_dir_parent_name` (`parent`, `name`),
- KEY `dist_dir_gmt_modified` (`gmt_modified`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
- */
-
-module.exports = function (sequelize, DataTypes) {
-  return sequelize.define('DistDir', {
-    name: {
-      type: DataTypes.STRING(200),
-      allowNull: false,
-      comment: 'dir name',
-    },
-    parent: {
-      type: DataTypes.STRING(200),
-      allowNull: false,
-      defaultValue: '/',
-      comment: 'parent dir',
-    },
-    date: {
-      type: DataTypes.STRING(20),
-      allowNull: false,
-      comment: '02-May-2014 01:06'
-    }
-  }, {
-    tableName: 'dist_dir',
-    comment: 'dist dir info',
-    indexes: [
-      {
-        unique: true,
-        fields: ['parent', 'name']
-      },
-      {
-        fields: ['gmt_modified']
-      }
-    ],
-    classMethods: {
-    }
-  });
-};

models/dist_file.js

@@ -1,82 +0,0 @@
-/**!
- * cnpmjs.org - models/dist_file.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-/*
-CREATE TABLE IF NOT EXISTS `dist_file` (
- `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
- `gmt_create` datetime NOT NULL COMMENT 'create time',
- `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'file name',
- `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
- `date` varchar(20) COMMENT '02-May-2014 01:06',
- `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
- `sha1` varchar(40) COMMENT 'sha1 hex value',
- `url` varchar(2048),
- PRIMARY KEY (`id`),
- UNIQUE KEY `dist_file_parent_name` (`parent`, `name`),
- KEY `dist_file_gmt_modified` (`gmt_modified`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';
- */
-
-module.exports = function (sequelize, DataTypes) {
-  return sequelize.define('DistFile', {
-    name: {
-      type: DataTypes.STRING(200),
-      allowNull: false,
-      comment: 'dir name',
-    },
-    parent: {
-      type: DataTypes.STRING(200),
-      allowNull: false,
-      defaultValue: '/',
-      comment: 'parent dir',
-    },
-    date: {
-      type: DataTypes.STRING(20),
-      allowNull: false,
-      comment: '02-May-2014 01:06'
-    },
-    size: {
-      type: DataTypes.INTEGER(10).UNSIGNED,
-      allowNull: false,
-      defaultValue: 0,
-      comment: 'file size'
-    },
-    sha1: {
-      type: DataTypes.STRING(40),
-      allowNull: false,
-      comment: 'sha1 hex value'
-    },
-    url: {
-      type: DataTypes.STRING(2048),
-      allowNull: false
-    }
-  }, {
-    tableName: 'dist_file',
-    comment: 'dist file info',
-    indexes: [
-      {
-        unique: true,
-        fields: ['parent', 'name']
-      },
-      {
-        fields: ['gmt_modified']
-      }
-    ],
-    classMethods: {
-    }
-  });
-};

models/download_total.js

@@ -14,42 +14,256 @@
  * Module dependencies.
  */
 
-// CREATE TABLE IF NOT EXISTS `download_total` (
+// CREATE TABLE IF NOT EXISTS `downloads` (
 //  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
 //  `gmt_create` datetime NOT NULL COMMENT 'create time',
 //  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
-//  `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
 //  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
-//  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
+//  `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
+//  `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
+//  `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
+//  `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
+//  `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
+//  `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
+//  `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
+//  `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
+//  `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
+//  `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
+//  `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
+//  `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
+//  `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
+//  `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
+//  `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
+//  `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
+//  `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
+//  `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
+//  `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
+//  `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
+//  `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
+//  `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
+//  `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
+//  `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
+//  `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
+//  `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
+//  `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
+//  `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
+//  `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
+//  `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
+//  `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
+//  `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
 //  PRIMARY KEY (`id`),
-//  UNIQUE KEY `download_total_date_name` (`date`, `name`)
+//  UNIQUE KEY `name_date` (`name`, `date`)
+//  KEY `date` (`date`)
 // ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
 
 module.exports = function (sequelize, DataTypes) {
   return sequelize.define('DownloadTotal', {
-    date: {
-      type: DataTypes.STRING(10),
-      allowNull: false,
-      comment: 'YYYY-MM-DD format',
-    },
     name: {
       type: DataTypes.STRING(100),
       allowNull: false,
       comment: 'module name',
     },
-    count: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+    date: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      comment: 'YYYYMM format',
+    },
+    d01: {
+      type: DataTypes.BIGINT(20),
       allowNull: false,
       defaultValue: 0,
-      comment: 'download count',
-    }
+      comment: '01 download count',
+    },
+    d02: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '02 download count',
+    },
+    d03: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '03 download count',
+    },
+    d04: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '04 download count',
+    },
+    d05: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '05 download count',
+    },
+    d06: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '06 download count',
+    },
+    d07: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '07 download count',
+    },
+    d08: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '08 download count',
+    },
+    d09: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '09 download count',
+    },
+    d10: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '10 download count',
+    },
+    d11: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '11 download count',
+    },
+    d12: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '12 download count',
+    },
+    d13: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '13 download count',
+    },
+    d14: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '14 download count',
+    },
+    d15: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '15 download count',
+    },
+    d16: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '16 download count',
+    },
+    d17: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '17 download count',
+    },
+    d18: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '18 download count',
+    },
+    d19: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '19 download count',
+    },
+    d20: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '20 download count',
+    },
+    d21: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '21 download count',
+    },
+    d22: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '22 download count',
+    },
+    d23: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '23 download count',
+    },
+    d24: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '24 download count',
+    },
+    d25: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '25 download count',
+    },
+    d26: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '26 download count',
+    },
+    d27: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '27 download count',
+    },
+    d28: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '28 download count',
+    },
+    d29: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '29 download count',
+    },
+    d30: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '30 download count',
+    },
+    d31: {
+      type: DataTypes.BIGINT(20),
+      allowNull: false,
+      defaultValue: 0,
+      comment: '31 download count',
+    },
   }, {
-    tableName: 'download_total',
+    tableName: 'downloads',
     comment: 'module download total info',
     indexes: [
       {
         unique: true,
-        fields: ['date', 'name']
+        fields: ['name', 'date']
+      },
+      {
+        fields: ['date']
       }
     ],
     classMethods: {

models/index.js

@@ -15,6 +15,7 @@
  */
 
 var path = require('path');
+var Sequelize = require('sequelize');
 var sequelize = require('../common/sequelize');
 
 function load(name) {
@@ -36,11 +37,14 @@ module.exports = {
   User: load('user'),
   Total: load('total'),
   DownloadTotal: load('download_total'),
-  DistFile: load('dist_file'),
-  DistDir: load('dist_dir'),
 
   query: function* (sql, args) {
-    return yield this.sequelize.query(sql, null, {raw: true}, args);
+    var options = { replacements: args };
+    var data = yield this.sequelize.query(sql, options).spread();
+    if (/select /i.test(sql)) {
+      return data[0];
+    }
+    return data[1];
   },
   queryOne: function* (sql, args) {
     var rows = yield* this.query(sql, args);

models/init_script.js

@@ -18,12 +18,20 @@ var config = require('../config');
 
 config.database.logging = console.log;
 
-// $ node --harmony models/init_script.js <force> <dialect>
+// $ node --harmony models/init_script.js <force> <dialect> <port> <username>
 var force = process.argv[2] === 'true';
 var dialect = process.argv[3];
 if (dialect) {
   config.database.dialect = dialect;
 }
+var port = process.argv[4];
+if (port) {
+  config.database.port = parseInt(port);
+}
+var username = process.argv[5];
+if (username) {
+  config.database.username = username;
+}
 
 var models = require('./');
 
@@ -44,5 +52,5 @@ models.sequelize.sync({ force: force })
   .catch(function (err) {
     console.error('[models/init_script.js] sequelize sync fail');
     console.error(err);
-    throw err;
+    process.exit(1);
   });

models/module.js

@@ -14,8 +14,6 @@
  * Module dependencies.
  */
 
-var utils = require('./utils');
-
 /*
 CREATE TABLE IF NOT EXISTS `module` (
   `id` INTEGER NOT NULL auto_increment ,
@@ -73,12 +71,12 @@ module.exports = function (sequelize, DataTypes) {
       allowNull: true,
     },
     dist_size: {
-      type: DataTypes.INTEGER.UNSIGNED,
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
     },
     publish_time: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+      type: DataTypes.BIGINT(20),
       allowNull: true,
     }
   }, {

models/module_unpublished.js

@@ -70,7 +70,10 @@ module.exports = function (sequelize, DataTypes) {
         });
         if (row) {
           row.package = pkg;
-          return yield row.save(['package']);
+          if (row.isDirty) {
+            row = yield row.save(['package']);
+          }
+          return row;
         }
 
         row = this.build({

models/tag.js

@@ -47,7 +47,7 @@ module.exports = function (sequelize, DataTypes) {
       comment: 'module version',
     },
     module_id: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+      type: DataTypes.BIGINT(20),
       allowNull: false,
       comment: 'module id'
     }

models/total.js

@@ -40,49 +40,49 @@ module.exports = function (sequelize, DataTypes) {
       comment: 'total name'
     },
     module_delete: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+      type: DataTypes.BIGINT(20),
       allowNull: false,
       defaultValue: 0,
       comment: 'module delete count',
     },
     last_sync_time: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+      type: DataTypes.BIGINT(20),
       allowNull: false,
       defaultValue: 0,
       comment: 'last timestamp sync from official registry',
     },
     last_exist_sync_time: {
-      type: DataTypes.BIGINT(20).UNSIGNED,
+      type: DataTypes.BIGINT(20),
       allowNull: false,
       defaultValue: 0,
       comment: 'last timestamp sync exist packages from official registry',
     },
     sync_status: {
-      type: 'TINYINT',
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
       comment: 'system sync from official registry status',
     },
     need_sync_num: {
-      type: DataTypes.INTEGER.UNSIGNED,
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
       comment: 'how many packages need to be sync',
     },
     success_sync_num: {
-      type: DataTypes.INTEGER.UNSIGNED,
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
       comment: 'how many packages sync success at this time',
     },
     fail_sync_num: {
-      type: DataTypes.INTEGER.UNSIGNED,
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
       comment: 'how many packages sync fail at this time',
     },
     left_sync_num: {
-      type: DataTypes.INTEGER.UNSIGNED,
+      type: DataTypes.INTEGER,
       allowNull: false,
       defaultValue: 0,
       comment: 'how many packages left to be sync',

models/user.js

@@ -117,6 +117,9 @@ module.exports = function (sequelize, DataTypes) {
         return yield this.find({ where: { name: name } });
       },
       listByNames: function* (names) {
+        if (!names || names.length === 0) {
+          return [];
+        }
         return yield this.findAll({
           where: {
             name: {
@@ -152,7 +155,10 @@ module.exports = function (sequelize, DataTypes) {
         user.json = data;
         user.email = data.email || '';
         user.rev = data._rev || '';
-        return yield user.save();
+        if (user.isDirty) {
+          user = yield user.save();
+        }
+        return user;
       },
       saveCustomUser: function* (data) {
         var name = data.user.login;
@@ -176,7 +182,10 @@ module.exports = function (sequelize, DataTypes) {
         user.rev = rev;
         user.salt = salt;
         user.password_sha = passwordSha;
-        return yield user.save();
+        if (user.isDirty) {
+          user = yield user.save();
+        }
+        return user;
       },
 
       // add cnpm user

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.0.0-rc.1",
+  "version": "2.1.0",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -9,43 +9,51 @@
     "status": "./bin/nodejsctl status",
     "stop": "./bin/nodejsctl stop"
   },
+  "bin": {
+    "cnpmjs.org": "bin/cli.js"
+  },
   "dependencies": {
     "agentkeepalive": "~1.2.0",
-    "bluebird": "~2.3.10",
+    "bluebird": "~2.9.7",
     "bytes": "~1.0.0",
-    "cfork": "~1.2.1",
-    "cheerio": "~0.17.0",
-    "co": "~3.1.0",
-    "co-defer": "~0.1.1",
+    "cfork": "~1.2.2",
+    "co": "~4.3.1",
+    "co-defer": "~1.0.0",
     "co-gather": "~0.0.1",
-    "co-redis": "~1.1.0",
     "co-sleep": "~0.0.1",
+    "commander": "~2.6.0",
     "copy-to": "~2.0.1",
-    "debug": "~2.1.0",
+    "debug": "~2.1.1",
     "error-formater": "~1.0.3",
-    "fs-cnpm": "~1.1.0",
+    "fs-cnpm": "~1.2.0",
     "giturl": "~0.0.3",
-    "graceful": "~0.1.0",
+    "gnode": "~0.1.1",
+    "graceful": "~1.0.0",
     "gravatar": "~1.1.0",
-    "humanize-ms": "~1.0.0",
+    "humanize-ms": "~1.0.1",
     "humanize-number": "~0.0.2",
-    "koa": "~0.13.0",
+    "kcors": "~1.0.1",
+    "koa": "~0.17.0",
     "koa-limit": "~1.0.2",
-    "koa-markdown": "~1.0.0",
-    "koa-middlewares": "~1.4.1",
-    "marked": "~0.3.2",
-    "mime": "~1.2.11",
+    "koa-markdown": "~2.0.1",
+    "koa-middlewares": "~2.1.0",
+    "koa-mock": "~1.1.4",
+    "koa-safe-jsonp": "~0.3.0",
+    "markdown-it": "~3.0.6",
+    "mime": "~1.3.4",
     "mini-logger": "~1.0.0",
     "mkdirp": "~0.5.0",
-    "moment": "~2.8.3",
-    "mysql": "~2.5.2",
+    "moment": "~2.9.0",
+    "mysql": "~2.5.4",
     "nodemailer": "~1.3.0",
-    "redis": "~0.12.1",
-    "semver": "~4.1.0",
-    "sequelize": "~2.0.0-rc2",
-    "thunkify-wrap": "~1.0.3",
-    "urllib": "~2.0.2",
-    "utility": "~1.2.0"
+    "semver": "~4.2.0",
+    "sequelize": "~2.0.1",
+    "thunkify-wrap": "~1.0.4",
+    "treekill": "~1.0.0",
+    "tunnel-agent": "^0.4.0",
+    "urllib": "~2.2.2",
+    "utility": "~1.3.0",
+    "xss": "~0.1.20"
   },
   "devDependencies": {
     "autod": "*",
@@ -54,11 +62,12 @@
     "contributors": "*",
     "istanbul-harmony": "*",
     "jshint": "*",
-    "koa-mock": "^1.0.2",
     "mm": "*",
     "mocha": "*",
     "node-dev": "*",
     "pedding": "*",
+    "pg": "~4.2.0",
+    "pg-hstore": "~2.3.1",
     "should": "~4.0.4",
     "should-http": "*",
     "sqlite3": "*",

public/css/atom-dark.css

@@ -0,0 +1,253 @@
+.editor-colors {
+  background-color: #1d1f21;
+  color: #c5c8c6;
+}
+.editor .invisible-character {
+  color: rgba(197, 200, 198, 0.2);
+}
+.editor .indent-guide {
+  color: rgba(197, 200, 198, 0.2);
+}
+.editor .wrap-guide {
+  background-color: rgba(197, 200, 198, 0.1);
+}
+.editor .gutter {
+  background-color: #292c2f;
+}
+.editor .gutter .cursor-line {
+  background-color: rgba(255, 255, 255, 0.14);
+}
+.editor .line-number.cursor-line-no-selection {
+  background-color: rgba(255, 255, 255, 0.14);
+}
+.editor .gutter .line-number.folded,
+.editor .gutter .line-number:after,
+.editor .fold-marker:after {
+  color: #fba0e3;
+}
+.editor .invisible {
+  color: #c5c8c6;
+}
+.editor .cursor {
+  border-color: #ffffff;
+}
+.editor .selection .region {
+  background-color: #444444;
+}
+.editor .source.gfm {
+  color: #999;
+}
+.editor .gfm .markup.heading {
+  color: #eee;
+}
+.editor .gfm .link {
+  color: #555;
+}
+.editor .gfm .variable.list,
+.editor .gfm .support.quote {
+  color: #555;
+}
+.editor .gfm .link .entity {
+  color: #ddd;
+}
+.editor .gfm .raw {
+  color: #aaa;
+}
+.editor .markdown .paragraph {
+  color: #999;
+}
+.editor .markdown .heading {
+  color: #eee;
+}
+.editor .markdown .raw {
+  color: #aaa;
+}
+.editor .markdown .link {
+  color: #555;
+}
+.editor .markdown .link .string {
+  color: #555;
+}
+.editor .markdown .link .string.title {
+  color: #ddd;
+}
+.editor .search-results .marker .region {
+  background-color: transparent;
+  border: 1px solid #888888;
+}
+.editor .search-results .marker.current-result .region {
+  border: 1px solid #ffffff;
+}
+.bracket-matcher {
+  border-bottom: 1px solid #f8de7e;
+  margin-top: -1px;
+  opacity: .7;
+}
+.comment {
+  color: #7C7C7C;
+}
+.entity {
+  color: #FFD2A7;
+}
+.entity.name.type {
+  text-decoration: underline;
+  color: #FFFFB6;
+}
+.entity.other.inherited-class {
+  color: #9B5C2E;
+}
+.keyword {
+  color: #96CBFE;
+}
+.keyword.control {
+  color: #96CBFE;
+}
+.keyword.operator {
+  color: #EDEDED;
+}
+.storage {
+  color: #CFCB90;
+}
+.storage.modifier {
+  color: #96CBFE;
+}
+.constant {
+  color: #99CC99;
+}
+.constant.numeric {
+  color: #FF73FD;
+}
+.variable {
+  color: #C6C5FE;
+}
+.invalid.deprecated {
+  text-decoration: underline;
+  color: #FD5FF1;
+}
+.invalid.illegal {
+  color: #FD5FF1;
+  background-color: rgba(86, 45, 86, 0.75);
+}
+.source .string .source,
+.source .string .meta.embedded.line {
+  color: #EDEDED;
+}
+.source .string .punctuation.section.embedded {
+  color: #00A0A0;
+}
+.source .string .punctuation.section.embedded .source {
+  color: #00A0A0;
+}
+.string {
+  color: #A8FF60;
+}
+.string .constant {
+  color: #00A0A0;
+}
+.string.regexp {
+  color: #E9C062;
+}
+.string.regexp .constant.character.escape,
+.string.regexp .source.ruby.embedded,
+.string.regexp .string.regexp.arbitrary-repitition {
+  color: #FF8000;
+}
+.string.regexp.group {
+  color: #C6A24F;
+  background-color: rgba(255, 255, 255, 0.06);
+}
+.string.regexp.character-class {
+  color: #B18A3D;
+}
+.string .variable {
+  color: #8A9A95;
+}
+.support {
+  color: #FFFFB6;
+}
+.support.function {
+  color: #DAD085;
+}
+.support.constant {
+  color: #FFD2A7;
+}
+.support.type.property-name.css {
+  color: #EDEDED;
+}
+.source .entity.name.tag,
+.source .entity.other.attribute-name,
+.meta.tag.inline,
+.meta.tag.inline .entity {
+  color: #96CBFE;
+}
+.entity.other.attribute-name {
+  color: #FFD7B1;
+}
+.entity.name.tag.namespace,
+.entity.other.attribute-name.namespace {
+  color: #E18964;
+}
+.meta.preprocessor.c {
+  color: #8996A8;
+}
+.meta.preprocessor.c .keyword {
+  color: #AFC4DB;
+}
+.meta.cast {
+  color: #676767;
+}
+.meta.sgml.html .meta.doctype,
+.meta.sgml.html .meta.doctype .entity,
+.meta.sgml.html .meta.doctype .string,
+.meta.xml-processing,
+.meta.xml-processing .entity,
+.meta.xml-processing .string {
+  color: #494949;
+}
+.meta.tag,
+.meta.tag .entity {
+  color: #96CBFE;
+}
+.meta.selector.css .entity.name.tag {
+  text-decoration: underline;
+  color: #96CBFE;
+}
+.meta.selector.css .entity.other.attribute-name.tag.pseudo-class {
+  color: #8F9D6A;
+}
+.meta.selector.css .entity.other.attribute-name.id {
+  color: #8B98AB;
+}
+.meta.selector.css .entity.other.attribute-name.class {
+  color: #62B1FE;
+}
+.meta.property-group .support.constant.property-value.css,
+.meta.property-value .support.constant.property-value.css {
+  color: #F9EE98;
+}
+.meta.preprocessor.at-rule .keyword.control.at-rule {
+  color: #8693A5;
+}
+.meta.property-value .support.constant.named-color.css,
+.meta.property-value .constant {
+  color: #87C38A;
+}
+.meta.constructor.argument.css {
+  color: #8F9D6A;
+}
+.meta.diff,
+.meta.diff.header {
+  color: #F8F8F8;
+  background-color: #0E2231;
+}
+.meta.separator {
+  color: #60A633;
+  background-color: #242424;
+}
+.meta.line.entry.logfile,
+.meta.line.exit.logfile {
+  background-color: rgba(238, 238, 238, 0.16);
+}
+.meta.line.error.logfile {
+  background-color: #751012;
+}

public/css/atom-light.css

@@ -0,0 +1,185 @@
+.editor,
+.editor .gutter {
+  background-color: #ffffff;
+  color: #555555;
+}
+.editor .invisible-character {
+  color: rgba(85, 85, 85, 0.2);
+}
+.editor .indent-guide {
+  color: rgba(85, 85, 85, 0.2);
+}
+.editor .wrap-guide {
+  background-color: rgba(85, 85, 85, 0.2);
+}
+.editor .gutter {
+  color: #555555;
+  background: #ffffff;
+}
+.editor .gutter .line-number.folded,
+.editor .gutter .line-number:after,
+.editor .fold-marker:after {
+  color: #e87b00;
+}
+.editor .invisible {
+  color: #555;
+}
+.editor .selection .region {
+  background-color: #e1e1e1;
+}
+.editor.is-focused .cursor {
+  border-color: #000000;
+}
+.editor.is-focused .selection .region {
+  background-color: #afc4da;
+}
+.editor.is-focused .line-number.cursor-line-no-selection,
+.editor.is-focused .line.cursor-line {
+  background-color: rgba(255, 255, 134, 0.34);
+}
+.editor .comment {
+  color: #999988;
+  font-style: italic;
+}
+.editor .string {
+  color: #D14;
+}
+.editor .source .string .source,
+.editor .source .string .meta.embedded.line {
+  color: #5A5A5A;
+}
+.editor .source .string .punctuation.section.embedded {
+  color: #920B2D;
+}
+.editor .source .string .punctuation.section.embedded .source {
+  color: #920B2D;
+}
+.editor .constant.numeric {
+  color: #D14;
+}
+.editor .constant.language {
+  color: #606aa1;
+}
+.editor .constant.character,
+.editor .constant.other {
+  color: #606aa1;
+}
+.editor .constant.symbol {
+  color: #990073;
+}
+.editor .constant.numeric.line-number.find-in-files .match {
+  color: rgba(143, 190, 0, 0.63);
+}
+.editor .variable {
+  color: #008080;
+}
+.editor .variable.parameter {
+  color: #606aa1;
+}
+.editor .keyword {
+  color: #222;
+  font-weight: bold;
+}
+.editor .keyword.unit {
+  color: #445588;
+}
+.editor .keyword.special-method {
+  color: #0086B3;
+}
+.editor .storage {
+  color: #222;
+}
+.editor .storage.type {
+  color: #222;
+}
+.editor .entity.name.class {
+  text-decoration: underline;
+  color: #606aa1;
+}
+.editor .entity.other.inherited-class {
+  text-decoration: underline;
+  color: #606aa1;
+}
+.editor .entity.name.function {
+  color: #900;
+}
+.editor .entity.name.tag {
+  color: #008080;
+}
+.editor .entity.other.attribute-name {
+  color: #458;
+  font-weight: bold;
+}
+.editor .entity.name.filename.find-in-files {
+  color: #E6DB74;
+}
+.editor .support.constant,
+.editor .support.function,
+.editor .support.type {
+  color: #458;
+}
+.editor .support.class {
+  color: #008080;
+}
+.editor .invalid {
+  color: #F8F8F0;
+  background-color: #00A8C6;
+}
+.editor .invalid.deprecated {
+  color: #F8F8F0;
+  background-color: #8FBE00;
+}
+.editor .meta.structure.dictionary.json > .string.quoted.double.json,
+.editor .meta.structure.dictionary.json > .string.quoted.double.json .punctuation.string {
+  color: #000080;
+}
+.editor .meta.structure.dictionary.value.json > .string.quoted.double.json {
+  color: #d14;
+}
+.editor .meta.diff,
+.editor .meta.diff.header {
+  color: #75715E;
+}
+.editor .css.support.property-name {
+  font-weight: bold;
+  color: #333;
+}
+.editor .css.constant {
+  color: #099;
+}
+.editor .source.gfm {
+  color: #444;
+}
+.editor .gfm .markup.heading {
+  color: #111;
+}
+.editor .gfm .link {
+  color: #888;
+}
+.editor .gfm .variable.list {
+  color: #888;
+}
+.editor .markdown .paragraph {
+  color: #444;
+}
+.editor .markdown .heading {
+  color: #111;
+}
+.editor .markdown .link {
+  color: #888;
+}
+.editor .markdown .link .string {
+  color: #888;
+}
+.editor .search-results .marker .region {
+  background-color: transparent;
+  border: 1px solid #999999;
+}
+.editor .search-results .marker.current-result .region {
+  border: 1px solid #000000;
+}
+.bracket-matcher {
+  background-color: #C9C9C9;
+  opacity: .7;
+  border-bottom: 0 none;
+}

public/css/main.css

@@ -26,7 +26,7 @@ header #search-input {
 }
 .pack-overview-background {
   height: 300px;
-  background: #428bca url(http://rockdai.u.qiniudn.com/bs-docs-masthead-pattern.png) repeat center center;
+  background: #428bca url(https://dn-cnpm.qbox.me/bs-docs-masthead-pattern.png) repeat center center;
   position: absolute;
   width: 100%;
   z-index: -1;
@@ -101,9 +101,7 @@ header #search-input {
   color: #333333;
 }
 .cols-box {
-  display: -moz-box;
-  display: -webkit-box;
-  display: box;
+  display: flex;
   -webkit-box-pack: center;
 }
 .cols-box ul {
@@ -112,9 +110,7 @@ header #search-input {
   padding: 0;
 }
 .cols-box-item {
-  -moz-box-flex: 1;
-  -webkit-box-flex: 1;
-  box-flex: 1;
+  flex: 1;
   height: 250px;
   margin-right: 30px;
 }

routes/registry.js

@@ -20,6 +20,7 @@ var login = require('../middleware/login');
 var publishable = require('../middleware/publishable');
 var syncByInstall = require('../middleware/sync_by_install');
 var editable = require('../middleware/editable');
+var existsPackage = require('../middleware/exists_package');
 
 var showTotal = require('../controllers/total');
 
@@ -34,6 +35,7 @@ var removePackage = require('../controllers/registry/package/remove');
 var removeOneVersion = require('../controllers/registry/package/remove_version');
 var updatePackage = require('../controllers/registry/package/update');
 var downloadPackage = require('../controllers/registry/package/download');
+var downloadTotal = require('../controllers/registry/package/download_total');
 
 var addUser = require('../controllers/registry/user/add');
 var showUser = require('../controllers/registry/user/show');
@@ -41,6 +43,7 @@ var updateUser = require('../controllers/registry/user/update');
 
 var sync = require('../controllers/sync');
 var userPackage = require('../controllers/registry/user_package');
+var tags = require('../controllers/registry/package/dist_tag');
 
 function routes(app) {
 
@@ -108,6 +111,32 @@ function routes(app) {
 
   // list all packages of user
   app.get('/-/by-user/:user', userPackage.list);
+
+  // download times
+  app.get('/downloads/range/:range/:name', downloadTotal);
+  app.get('/downloads/range/:range', downloadTotal);
+
+  // GET /-/package/:pkg/dist-tags -- returns the package's dist-tags
+  app.get('/-/package/:name/dist-tags', existsPackage, tags.index);
+  app.get(/^\/\-\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/dist\-tags$/, existsPackage, tags.index);
+
+  // PUT /-/package/:pkg/dist-tags -- Set package's dist-tags to provided object body (removing missing)
+  app.put('/-/package/:name/dist-tags', login, existsPackage, editable, tags.save);
+  app.put(/^\/\-\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/dist\-tags$/, login, existsPackage, editable, tags.save);
+
+  // POST /-/package/:pkg/dist-tags -- Add/modify dist-tags from provided object body (merge)
+  app.post('/-/package/:name/dist-tags', login, existsPackage, editable, tags.update);
+  app.post(/^\/\-\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/dist\-tags$/, login, existsPackage, editable, tags.update);
+
+  // PUT /-/package/:pkg/dist-tags/:tag -- Set package's dist-tags[tag] to provided string body
+  app.put('/-/package/:name/dist-tags/:tag', login, existsPackage, editable, tags.set);
+  app.put(/^\/\-\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/dist\-tags\/([\w\-\.]+)$/, login, existsPackage, editable, tags.set);
+  // POST /-/package/:pkg/dist-tags/:tag -- Same as PUT /-/package/:pkg/dist-tags/:tag
+  app.post('/-/package/:name/dist-tags/:tag', login, existsPackage, editable, tags.set);
+
+  // DELETE /-/package/:pkg/dist-tags/:tag -- Remove tag from dist-tags
+  app.delete('/-/package/:name/dist-tags/:tag', login, existsPackage, editable, tags.destroy);
+  app.delete(/^\/\-\/package\/(@[\w\-\.]+\/[\w\-\.]+)\/dist\-tags\/([\w\-\.]+)$/, login, existsPackage, editable, tags.destroy);
 }
 
 module.exports = routes;

routes/web.js

@@ -6,7 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 "use strict";
@@ -19,12 +19,10 @@ var showPackage = require('../controllers/web/package/show');
 var searchPackage = require('../controllers/web/package/search');
 var searchRange = require('../controllers/web/package/search_range');
 var listPrivates = require('../controllers/web/package/list_privates');
-var showSync = require('../controllers/web/package/show_sync');
+var showSync = require('../controllers/web/show_sync');
 var showUser = require('../controllers/web/user/show');
-
 var sync = require('../controllers/sync');
 var showTotal = require('../controllers/total');
-var dist = require('../controllers/web/dist');
 var badge = require('../controllers/web/badge');
 
 function routes(app) {
@@ -52,8 +50,6 @@ function routes(app) {
 
   app.get('/_list/search/search', searchRange);
 
-  app.get(/^\/dist(\/.*)?/, dist.list);
-
   app.get(/^\/badge\/v\/([@\w\-\.\/]+)\.svg$/, badge.version);
 }
 

servers/registry.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+global.Promise = require('bluebird');
 var koa = require('koa');
 var app = module.exports = koa();
 var http = require('http');
@@ -26,6 +27,8 @@ var block = require('../middleware/block');
 var auth = require('../middleware/auth');
 var staticCache = require('../middleware/static');
 var notFound = require('../middleware/registry_not_found');
+var cors = require('kcors');
+var proxyToNpm = require('../middleware/proxy_to_npm');
 
 app.use(block());
 middlewares.jsonp(app);
@@ -36,7 +39,11 @@ app.use(staticCache);
 app.keys = ['todokey', config.sessionSecret];
 app.proxy = true;
 app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
+app.use(cors({
+  allowMethods: 'GET,HEAD'
+}));
 app.use(auth());
+app.use(proxyToNpm());
 app.use(notFound);
 
 if (config.enableCompress) {

servers/web.js

@@ -6,7 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 "use strict";
@@ -15,16 +15,20 @@
  * Module dependencies.
  */
 
+global.Promise = require('bluebird');
 var opensearch = require('../middleware/opensearch');
 var notFound = require('../middleware/web_not_found');
 var staticCache = require('../middleware/static');
 var middlewares = require('koa-middlewares');
+var markdownMiddleware = require('koa-markdown');
 var block = require('../middleware/block');
 var logger = require('../common/logger');
+var renderMarkdown = require('../common/markdown').render;
 var auth = require('../middleware/auth');
-var markdown = require('koa-markdown');
+var proxyToNpm = require('../middleware/proxy_to_npm');
 var routes = require('../routes/web');
 var config = require('../config');
+var jsonp = require('koa-safe-jsonp');
 var path = require('path');
 var http = require('http');
 var koa = require('koa');
@@ -32,6 +36,8 @@ var fs = require('fs');
 
 var app = koa();
 
+jsonp(app);
+
 var rootdir = path.dirname(__dirname);
 
 app.use(block());
@@ -39,7 +45,7 @@ app.use(middlewares.rt({headerName: 'X-ReadTime'}));
 app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
 app.use(staticCache);
 
-if (process.env.NODE_ENV === 'development') {
+if (config.pagemock) {
   app.use(require('koa-mock')({
     datadir: path.join(rootdir, 'test', 'mocks')
   }));
@@ -48,6 +54,9 @@ if (process.env.NODE_ENV === 'development') {
 app.use(opensearch);
 app.keys = ['todokey', config.sessionSecret];
 app.proxy = true;
+app.use(proxyToNpm({
+  isWeb: true
+}));
 app.use(middlewares.bodyParser());
 app.use(auth());
 app.use(notFound);
@@ -66,7 +75,8 @@ var layoutFile = path.join(viewDir, '_layout.html');
 var footer = config.customFooter || fs.readFileSync(path.join(viewDir, 'footer.html'), 'utf8');
 var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
   .replace('{{footer}}', footer)
-  .replace('{{logoURL}}', config.logoURL);
+  .replace('{{logoURL}}', config.logoURL)
+  .replace('{{adBanner}}', config.adBanner || '');
 fs.writeFileSync(layoutFile, layout);
 
 // custom web readme home page support
@@ -79,16 +89,17 @@ if (config.customReadmeFile) {
 }
 fs.writeFileSync(readmeFile, readmeContent);
 
-app.use(markdown({
+app.use(markdownMiddleware({
   baseUrl: '/',
   root: docDir,
   layout: layoutFile,
-  titleHolder: '<%- locals.title %>',
+  titleHolder: '<%= locals.title %>',
   bodyHolder: '<%- locals.body %>',
   indexName: '_readme',
-  remarkableOptions: {
-    html: true
-  }
+  cache: true,
+  render: function (content) {
+    return renderMarkdown(content, false);
+  },
 }));
 
 var locals = {
@@ -116,6 +127,7 @@ routes(app);
 
 app.on('error', function (err, ctx) {
   err.url = err.url || ctx.request.url;
+  console.log(err);
   console.log(err.stack);
   logger.error(err);
 });

services/dist.js

@@ -1,91 +0,0 @@
-/**!
- * cnpmjs.org - services/dist.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-/* jshint -W032 */
-
-/**
- * Module dependencies.
- */
-
-var path = require('path');
-var models = require('../models');
-var File = models.DistFile;
-var Dir = models.DistDir;
-
-exports.savefile = function* (info) {
-  var row = yield File.find({
-    where: {
-      parent: info.parent,
-      name: info.name
-    }
-  });
-  if (!row) {
-    row = File.build({
-      parent: info.parent,
-      name: info.name
-    });
-  }
-  row.date = info.date;
-  row.size = info.size;
-  row.url = info.url;
-  row.sha1 = info.sha1;
-  return yield row.save();
-};
-
-exports.savedir = function* (info) {
-  var row = yield Dir.find({
-    where: {
-      parent: info.parent,
-      name: info.name
-    }
-  });
-  if (!row) {
-    row = Dir.build({
-      parent: info.parent,
-      name: info.name
-    });
-  }
-  row.date = info.date;
-  return yield row.save();
-};
-
-exports.listdir = function* (name) {
-  var rs = yield [
-    File.findAll({
-      attributrs: ['name', 'parent', 'date'],
-      where: {
-        parent: name
-      }
-    }),
-    Dir.findAll({
-      attributrs: ['name', 'parent', 'date', 'size', 'url', 'sha1'],
-      where: {
-        parent: name
-      }
-    })
-  ];
-  return rs[0].concat(rs[1]);
-};
-
-exports.getfile = function* (fullname) {
-  var name = path.basename(fullname);
-  var parent = path.dirname(fullname);
-  if (parent !== '/') {
-    parent += '/';
-  }
-  return yield File.find({
-    attributrs: ['name', 'parent', 'date', 'size', 'url', 'sha1'],
-    where: {
-      parent: parent,
-      name: name
-    }
-  });
-};

services/download_total.js

@@ -14,41 +14,104 @@
  * Module dependencies.
  */
 
-var models = require('../models');
-var DownloadTotal = models.DownloadTotal;
+var utility = require('utility');
+var DownloadTotal = require('../models').DownloadTotal;
 
 exports.getModuleTotal = function* (name, start, end) {
-  return yield DownloadTotal.findAll({
+  var startMonth = parseYearMonth(start);
+  var endMonth = parseYearMonth(end);
+  var rows = yield DownloadTotal.findAll({
     where: {
       date: {
-        gte: start,
-        lte: end
+        gte: startMonth,
+        lte: endMonth
       },
       name: name
     }
   });
+  return formatRows(rows, start, end);
 };
 
 exports.plusModuleTotal = function* (data) {
+  var yearMonth = parseYearMonth(data.date);
+  // all module download total
   var row = yield DownloadTotal.find({
     where: {
-      date: data.date,
-      name: data.name
+      name: '__all__',
+      date: yearMonth
     }
   });
   if (!row) {
     row = DownloadTotal.build({
-      date: data.date,
-      name: data.name
+      name: '__all__',
+      date: yearMonth,
     });
   }
-  row.count += data.count;
-  return yield row.save();
-};
+  var field = 'd' + data.date.substring(8, 10);
+  if (typeof row[field] === 'string') {
+    // pg bigint is string...
+    row[field] = utility.toSafeNumber(row[field]);
+  }
+  row[field] += data.count;
+  if (row.isDirty) {
+    yield row.save();
+  }
 
+  row = yield DownloadTotal.find({
+    where: {
+      name: data.name,
+      date: yearMonth,
+    }
+  });
+  if (!row) {
+    row = DownloadTotal.build({
+      name: data.name,
+      date: yearMonth,
+    });
+  }
+  var field = 'd' + data.date.substring(8, 10);
+  if (typeof row[field] === 'string') {
+    // pg bigint is string...
+    row[field] = utility.toSafeNumber(row[field]);
+  }
+  row[field] += data.count;
+  if (row.isDirty) {
+    return yield row.save();
+  }
+  return row;
+};
 
 exports.getTotal = function* (start, end) {
-  var sql = 'SELECT date, sum(count) AS count FROM download_total \
-    WHERE date>=? AND date<=? GROUP BY date;';
-  return yield models.query(sql, [start, end]);
+  return yield* exports.getModuleTotal('__all__', start, end);
 };
+
+function parseYearMonth(date) {
+  return Number(date.substring(0, 7).replace('-', ''));
+}
+
+function formatRows(rows, startDate, endDate) {
+  var dates = [];
+  rows.forEach(function (row) {
+    var date = String(row.date);
+    var month = date.substring(4, 6);
+    var year = date.substring(0, 4);
+    var yearMonth = year + '-' + month;
+    for (var i = 1; i <= 31; i++) {
+      var day = i < 10 ? '0' + i : String(i);
+      var field = 'd' + day;
+      var d = yearMonth + '-' + day;
+      var count = row[field];
+      if (typeof count === 'string') {
+        count = utility.toSafeNumber(count);
+      }
+      if (count > 0 && d >= startDate && d <= endDate) {
+        dates.push({
+          name: row.name,
+          count: count,
+          date: d
+        });
+      }
+    }
+  });
+  return dates;
+}

services/npm.js

@@ -18,7 +18,7 @@
 var urllib = require('../common/urllib');
 var config = require('../config');
 
-var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+var USER_AGENT = 'npm_service.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function* request(url, options) {
   options = options || {};
@@ -27,6 +27,8 @@ function* request(url, options) {
   options.headers = {
     'user-agent': USER_AGENT
   };
+  options.gzip = true;
+  options.followRedirect = true;
   var registry = options.registry || config.sourceNpmRegistry;
   url = registry + url;
   var r;
@@ -51,6 +53,7 @@ exports.getUser = function* (name) {
   var r = yield* request(url);
   var data = r.data;
   if (data && !data.name) {
+    // 404
     data = null;
   }
   return data;
@@ -84,20 +87,25 @@ exports.getShort = function* (timeout) {
 exports.getPopular = function* (top, timeout) {
   var r = yield* request('/-/_view/dependedUpon?group_level=1', {
     registry: config.officialNpmRegistry,
-    timeout: timeout || 60000
+    timeout: timeout || 120000
   });
   if (!r.data || !r.data.rows || !r.data.rows.length) {
     return [];
   }
 
-  return r.data.rows.sort(function (a, b) {
+  // deps number must >= 100
+  var rows = r.data.rows.filter(function (a) {
+    return a.value >= 100;
+  });
+
+  return rows.sort(function (a, b) {
     return b.value - a.value;
   })
   .slice(0, top)
   .map(function (r) {
-    return r.key && r.key[0];
+    return [r.key && r.key[0] && r.key[0].trim(), r.value];
   })
   .filter(function (r) {
-    return r;
+    return r[0];
   });
 };

services/package.js

@@ -37,6 +37,10 @@ function parseRow(row) {
         row.package = decodeURIComponent(row.package);
       }
       row.package = JSON.parse(row.package);
+      if (typeof row.publish_time === 'string') {
+        // pg bigint is string
+        row.publish_time = Number(row.publish_time);
+      }
     } catch (e) {
       console.warn('parse package error: %s, id: %s version: %s, error: %s', row.name, row.id, row.version, e);
     }
@@ -276,10 +280,12 @@ exports.saveModule = function* (mod) {
   item.dist_size = dist.size;
   item.description = description;
 
-  var newItem = yield item.save();
+  if (item.isDirty) {
+    item = yield item.save();
+  }
   var result = {
-    id: newItem.id,
-    gmt_modified: newItem.gmt_modified
+    id: item.id,
+    gmt_modified: item.gmt_modified
   };
 
   if (!Array.isArray(keywords)) {
@@ -397,7 +403,10 @@ exports.addModuleTag = function* (name, tag, version) {
   }
   row.module_id = mod.id;
   row.version = version;
-  return yield row.save();
+  if (row.isDirty) {
+    return yield row.save();
+  }
+  return row;
 };
 
 exports.getModuleTag = function* (name, tag) {
@@ -571,7 +580,12 @@ exports.addKeyword = function* (data) {
     item = ModuleKeyword.build(data);
   }
   item.description = data.description;
-  return yield item.save();
+  if (item.isDirty) {
+    // make sure object will change, otherwise will cause empty sql error
+    // @see https://github.com/cnpm/cnpmjs.org/issues/533
+    return yield item.save();
+  }
+  return item;
 };
 
 exports.addKeywords = function* (name, description, keywords) {
@@ -599,7 +613,7 @@ exports.search = function* (word, options) {
   // 3. keyword equal search
   var ids = {};
 
-  var sql = 'SELECT module_id FROM tag WHERE LOWER(name) LIKE LOWER(?) AND tag="latest" \
+  var sql = 'SELECT module_id FROM tag WHERE LOWER(name) LIKE LOWER(?) AND tag=\'latest\' \
     ORDER BY name LIMIT ?;';
   var rows = yield* models.query(sql, [word + '%', limit ]);
   for (var i = 0; i < rows.length; i++) {

services/total.js

@@ -18,15 +18,20 @@ var config = require('../config');
 var models = require('../models');
 var Total = models.Total;
 
+var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM module;';
+var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM module;';
+var TOTAL_USER_SQL = 'SELECT count(name) AS count FROM user;';
+if (config.database.dialect === 'postgres') {
+  // pg not allow table name as 'user'
+  TOTAL_USER_SQL = 'SELECT count(name) AS count FROM public.user;';
+}
+
 exports.get = function* () {
   // var DB_SIZE_SQL = 'SELECT TABLE_NAME AS name, data_length, index_length \
   //   FROM information_schema.tables WHERE TABLE_SCHEMA = ? \
   //   GROUP BY TABLE_NAME \
   //   ORDER BY data_length DESC \
   //   LIMIT 0, 200';
-  var TOTAL_MODULE_SQL = 'SELECT count(distinct(name)) AS count FROM `module`;';
-  var TOTAL_VERSION_SQL = 'SELECT count(name) AS count FROM `module`;';
-  var TOTAL_USER_SQL = 'SELECT count(name) AS count FROM `user`;';
   var rs = yield [
     // models.query(DB_SIZE_SQL, [config.db]),
     models.queryOne(TOTAL_MODULE_SQL),
@@ -41,6 +46,10 @@ exports.get = function* () {
   var uc = rs[2];
   var info = rs[3] || {};
 
+  if (typeof info.module_delete === 'string') {
+    info.module_delete = Number(info.module_delete);
+  }
+
   var total = {
     data_tables: {},
     disk_size: 0,
@@ -82,42 +91,49 @@ exports.get = function* () {
 };
 
 exports.getTotalInfo = function* () {
-  return yield Total.find({
+  var row = yield Total.find({
     where: {
       name: 'total'
     }
   });
+  if (row && typeof row.module_delete === 'string') {
+    row.module_delete = Number(row.module_delete);
+  }
+  return row;
 };
 
 exports.plusDeleteModule = function* () {
-  var sql = 'UPDATE total SET module_delete=module_delete+1 WHERE name="total"';
+  var sql = 'UPDATE total SET module_delete=module_delete+1 WHERE name=\'total\'';
   return yield* models.query(sql);
 };
 
 exports.setLastSyncTime = function* (time) {
-  var sql = 'UPDATE total SET last_sync_time=? WHERE name="total"';
+  var sql = 'UPDATE total SET last_sync_time=? WHERE name=\'total\'';
   return yield* models.query(sql, [Number(time)]);
 };
 
 exports.setLastExistSyncTime = function* (time) {
-  var sql = 'UPDATE total SET last_exist_sync_time=? WHERE name="total"';
+  var sql = 'UPDATE total SET last_exist_sync_time=? WHERE name=\'total\'';
   return yield* models.query(sql, [Number(time)]);
 };
 
 exports.updateSyncStatus = function* (status) {
-  var sql = 'UPDATE total SET sync_status=? WHERE name="total"';
+  var sql = 'UPDATE total SET sync_status=? WHERE name=\'total\'';
   return yield* models.query(sql, [status]);
 };
 
 exports.updateSyncNum = function* (params) {
-  var arg = {
-    sync_status: params.syncStatus,
-    need_sync_num: params.need || 0,
-    success_sync_num: params.success || 0,
-    fail_sync_num: params.fail || 0,
-    left_sync_num: params.left || 0,
-    last_sync_module: params.lastSyncModule
-  };
-  var sql = 'UPDATE total SET ? WHERE name="total"';
-  return yield* models.query(sql, [arg]);
+  var args = [
+    params.syncStatus,
+    params.need || 0,
+    params.success || 0,
+    params.fail || 0,
+    params.left || 0,
+    params.lastSyncModule
+  ];
+  var sql = 'UPDATE total SET \
+    sync_status=?, need_sync_num=?, success_sync_num=?, \
+    fail_sync_num=?, left_sync_num=?, last_sync_module=? \
+    WHERE name=\'total\'';
+  return yield* models.query(sql, args);
 };

sync/index.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+global.Promise = require('bluebird');
 var debug = require('debug')('cnpmjs.org:sync:index');
 var co = require('co');
 var ms = require('humanize-ms');
@@ -22,7 +23,6 @@ var config = require('../config');
 var mail = require('../common/mail');
 var logger = require('../common/logger');
 var totalService = require('../services/total');
-var DistSyncer = require('./sync_dist');
 
 var sync = null;
 
@@ -43,14 +43,14 @@ if (!sync && config.enableCluster) {
 console.log('[%s] [sync_worker:%s] syncing with %s mode',
   Date(), process.pid, config.syncModel);
 
+function onerror(err) {
+  logger.error(err);
+}
+
 //set sync_status = 0 at first
 co(function* () {
   yield* totalService.updateSyncStatus(0);
-})(function (err) {
-  if (err) {
-    logger.error(err);
-  }
-});
+}).catch(onerror);
 
 var syncInterval = ms(config.syncInterval);
 var minSyncInterval = ms('5m');
@@ -60,7 +60,7 @@ if (!syncInterval || syncInterval < minSyncInterval) {
 
 // the same time only sync once
 var syncing = false;
-var handleSync = co(function* () {
+var syncFn = co.wrap(function* () {
   debug('mode: %s, syncing: %s', config.syncModel, syncing);
   if (!syncing) {
     syncing = true;
@@ -83,83 +83,19 @@ var handleSync = co(function* () {
 });
 
 if (sync) {
-  handleSync();
-  setInterval(handleSync, syncInterval);
-}
-
-/**
- * sync dist(node.js and phantomjs)
- */
-
-var syncingDist = false;
-var syncDist = co(function* () {
-  if (syncingDist) {
-    return;
-  }
-  syncingDist = true;
-  logger.syncInfo('Start syncing dist...');
-  var distSyncer = new DistSyncer({
-    disturl: config.disturl
-  });
-  try {
-    yield* distSyncer.start();
-    yield* distSyncer.syncPhantomjsDir();
-  } catch (err) {
-    err.message += ' (sync dist error)';
-    logger.syncError(err);
-    if (config.noticeSyncDistError) {
-      sendMailToAdmin(err, null, new Date());
-    }
-  }
-  syncingDist = false;
-});
-
-if (config.syncDist) {
-  syncDist();
-  setInterval(syncDist, syncInterval);
-} else {
-  logger.syncInfo('sync dist disable');
-}
-
-/**
- * sync python dist
- */
-
-var syncingPythonDist = false;
-var syncPythonDist = co(function* () {
-  if (syncingPythonDist) {
-    return;
-  }
-  syncingPythonDist = true;
-  logger.syncInfo('Start syncing python dist...');
-  var distSyncer = new DistSyncer({
-    disturl: config.pythonDisturl
-  });
-  try {
-    yield* distSyncer.start();
-  } catch (err) {
-    err.message += ' (sync python dist error)';
-    logger.syncError(err);
-    if (config.noticeSyncDistError) {
-      sendMailToAdmin(err, null, new Date());
-    }
-  }
-  syncingPythonDist = false;
-});
-
-if (config.syncPythonDist) {
-  syncPythonDist();
-  setInterval(syncPythonDist, syncInterval);
-} else {
-  logger.syncInfo('sync python dist disable');
+  syncFn().catch(onerror);
+  setInterval(function () {
+    syncFn().catch(onerror);
+  }, syncInterval);
 }
 
 /**
  * sync popular modules
  */
 
+var startSyncPopular = require('./sync_popular');
 var syncingPopular = false;
-var syncPopular = co(function* syncPopular() {
+var syncPopularFn = co.wrap(function* syncPopular() {
   if (syncingPopular) {
     return;
   }
@@ -168,7 +104,7 @@ var syncPopular = co(function* syncPopular() {
   var data;
   var error;
   try {
-    data = yield* require('./sync_popular');
+    data = yield* startSyncPopular();
   } catch (err) {
     error = err;
     error.message += ' (sync package error)';
@@ -186,8 +122,10 @@ var syncPopular = co(function* syncPopular() {
 });
 
 if (config.syncPopular) {
-  syncPopular();
-  setInterval(syncPopular, ms(config.syncPopularInterval));
+  syncPopularFn().catch(onerror);
+  setInterval(function () {
+    syncPopularFn().catch(onerror);
+  }, ms(config.syncPopularInterval));
 } else {
   logger.syncInfo('sync popular module disable');
 }

sync/status.js

@@ -37,7 +37,7 @@ Status.prototype.log = function (syncDone) {
   };
   co(function* () {
     yield* Total.updateSyncNum(params);
-  })();
+  }).catch(function () {});
 };
 
 Status.prototype.start = function () {

sync/sync_dist.js

@@ -1,454 +0,0 @@
-/**!
- * cnpmjs.org - sync/sync_dist.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:sync:sync_dist');
-var fs = require('fs');
-var bytes = require('bytes');
-var crypto = require('crypto');
-var utility = require('utility');
-var thunkify = require('thunkify-wrap');
-var cheerio = require('cheerio');
-var urlResolve = require('url').resolve;
-var common = require('../lib/common');
-var distService = require('../services/dist');
-var config = require('../config');
-var nfs = require('../common/nfs');
-var logger = require('../common/logger');
-var urllib = require('../common/urllib');
-
-var USER_AGENT = 'distsync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
-
-module.exports = DistSyncer;
-
-function DistSyncer(options) {
-  var disturl = options.disturl;
-  if (disturl[disturl.length - 1] === '/') {
-    disturl = disturl.replace(/(\/+)$/, '');
-  }
-  this._disturl = disturl;
-}
-
-var proto = DistSyncer.prototype;
-
-proto.start = function* (name) {
-  name = name || '/';
-  if (name[name.length - 1] !== '/') {
-    name += '/';
-  }
-  yield* this.syncDir(name);
-};
-
-proto.syncDir = function* (fullname, info) {
-  var news = yield* this.listdiff(fullname);
-  var files = [];
-  var dirs = [];
-
-  for (var i = 0; i < news.length; i++) {
-    var item = news[i];
-    if (item.type === 'dir') {
-      dirs.push(item);
-    } else {
-      files.push(item);
-    }
-  }
-
-  logger.syncInfo('sync %s:%s got %d new items, %d dirs, %d files to sync',
-    this._disturl, fullname, news.length, dirs.length, files.length);
-
-  for (var i = 0; i < files.length; i++) {
-    yield* this.syncFile(files[i]);
-  }
-
-  for (var i = 0; i < dirs.length; i++) {
-    var dir = dirs[i];
-    yield* this.syncDir(dir.parent + dir.name, dir);
-  }
-
-  if (info) {
-    logger.syncInfo('Save dir:%s %j to database', fullname, info);
-    yield* distService.savedir(info);
-  }
-
-  logger.syncInfo('Sync %s finished, %d dirs, %d files',
-    fullname, dirs.length, files.length);
-};
-
-proto.syncFile = function* (info) {
-  var name = info.parent + info.name;
-  name = process.pid + name.replace(/\//g, '_'); // make sure no parent dir
-  var isPhantomjsURL = false;
-  var downurl = this._disturl + info.parent + info.name;
-  if (info.downloadURL) {
-    downurl = info.downloadURL;
-    isPhantomjsURL = true;
-  }
-  var filepath = common.getTarballFilepath(name);
-  var ws = fs.createWriteStream(filepath);
-
-  var options = {
-    writeStream: ws,
-    followRedirect: true,
-    timeout: 6000000, // 100 minutes download
-    headers: {
-      'user-agent': USER_AGENT
-    }
-  };
-
-  try {
-    logger.syncInfo('downloading %s %s to %s, isPhantomjsURL: %s',
-      bytes(info.size), downurl, filepath, isPhantomjsURL);
-    // get tarball
-    var r = yield urllib.requestThunk(downurl, options);
-    var statusCode = r.status || -1;
-    logger.syncInfo('download %s got status %s, headers: %j',
-      downurl, statusCode, r.headers);
-    if (statusCode !== 200) {
-      var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
-      err.name = 'DownloadDistFileError';
-      throw err;
-    }
-
-    var shasum = crypto.createHash('sha1');
-    var dataSize = 0;
-    var rs = fs.createReadStream(filepath);
-    rs.on('data', function (data) {
-      shasum.update(data);
-      dataSize += data.length;
-    });
-    var end = thunkify.event(rs);
-    yield end(); // after end event emit
-
-    if (dataSize === 0) {
-      var err = new Error('Download ' + downurl + ' file size is zero');
-      err.name = 'DownloadDistFileZeroSizeError';
-      throw err;
-    }
-
-    if (isPhantomjsURL) {
-      debug('real size: %s, expect size: %s', dataSize, info.size);
-      if (dataSize < info.size) {
-        // phantomjs download page only show `6.7 MB`
-        var err = new Error('Download ' + downurl + ' file size is '
-          + dataSize + ' not match ' + info.size);
-        err.name = 'DownloadDistFileSizeError';
-        throw err;
-      }
-      info.size = dataSize;
-    } else if (info.size > 0 && dataSize !== info.size) {
-      var err = new Error('Download ' + downurl + ' file size is '
-        + dataSize + ' not match ' + info.size);
-      err.name = 'DownloadDistFileSizeError';
-      throw err;
-    }
-
-    shasum = shasum.digest('hex');
-    var args = {
-      key: '/dist' + info.parent + info.name,
-      size: info.size,
-      shasum: shasum,
-    };
-
-    // upload to NFS
-    logger.syncInfo('uploading %s to nfs:%s', filepath, args.key);
-    var result = yield nfs.upload(filepath, args);
-    info.url = result.url || result.key;
-    info.sha1 = shasum;
-
-    logger.syncInfo('upload %s to nfs:%s with size:%d, sha1:%s',
-      args.key, info.url, info.size, info.sha1);
-  } finally {
-    // remove tmp file whatever
-    fs.unlink(filepath, utility.noop);
-  }
-
-  logger.syncInfo('Sync dist file: %j done', info);
-  yield* distService.savefile(info);
-};
-
-// <a href="latest/">latest/</a>                                            02-May-2014 14:45                   -
-// <a href="node-v0.4.10.tar.gz">node-v0.4.10.tar.gz</a>                                26-Aug-2011 16:22            12410018
-var FILE_RE = /^<a[^>]+>([^<]+)<\/a>\s+(\d+\-\w+\-\d+ \d+\:\d+)\s+([\-\d]+)/;
-
-// */docs/api/
-var DOC_API_RE = /\/docs\/api\/$/;
-
-// <li><a href="documentation.html">About these Docs</a></li>
-// <li><a href="synopsis.html">Synopsis</a></li>
-// <li><a href="assert.html">Assertion Testing</a></li>
-// <li><a href="buffer.html">Buffer</a></li>
-// <li><a href="addons.html">C/C++ Addons</a></li>
-// <li><a href="child_process.html">Child Processes</a></li>
-// <div id="gtoc">
-//   <p>
-//     <a href="index.html" name="toc">Index</a> |
-//     <a href="all.html">View on single page</a> |
-//     <a href="index.json">View as JSON</a>
-//   </p>
-// </div>
-var DOC_API_FILE_ALL_RE = /<a[^"]+\"(\w+\.(?:html|json))\"[^>]*>[^<]+<\/a>/gm;
-var DOC_API_FILE_RE = /<a[^"]+\"(\w+\.(?:html|json))\"[^>]*>[^<]+<\/a>/;
-
-proto.listdir = function* (fullname) {
-  var url = this._disturl + fullname;
-  var isDocPath = false;
-  if (DOC_API_RE.test(fullname)) {
-    isDocPath = true;
-    url += 'index.html';
-  }
-  var result = yield urllib.requestThunk(url, {
-    timeout: 60000,
-  });
-  debug('listdir %s got %s, %j', url, result.status, result.headers);
-  var html = result.data && result.data.toString() || '';
-  var items = [];
-  // "last-modified":"Tue, 11 Mar 2014 22:44:36 GMT"
-  var date = result.headers['last-modified'] || result.headers.date || '';
-
-  if (isDocPath) {
-    // add assets/
-    items.push({
-      name: 'assets/',
-      date: date,
-      size: '-',
-      type: 'dir',
-      parent: fullname,
-    });
-
-    var needJSON = false;
-    var htmlfileNames = [];
-    var lines = html.match(DOC_API_FILE_ALL_RE) || [];
-    for (var i = 0; i < lines.length; i++) {
-      var m = DOC_API_FILE_RE.exec(lines[i].trim());
-      if (!m) {
-        continue;
-      }
-      var itemName = m[1];
-      items.push({
-        name: itemName,
-        date: date,
-        size: 0,
-        type: 'file',
-        parent: fullname,
-      });
-      if (itemName.indexOf('.json') > 0) {
-        needJSON = true;
-      }
-      if (itemName.indexOf('.html') > 0 && itemName !== 'index.html') {
-        htmlfileNames.push(itemName);
-      }
-    }
-    debug('listdir %s got %j', fullname, htmlfileNames);
-    if (needJSON) {
-      // node >= 0.8.0
-      htmlfileNames.forEach(function (itemName) {
-        items.push({
-          name: itemName.replace('.html', '.json'), // download *.json format
-          date: date,
-          size: 0,
-          type: 'file',
-          parent: fullname,
-        });
-      });
-    }
-  } else {
-    var lines = html.split('\n');
-    for (var i = 0; i < lines.length; i++) {
-      var m = FILE_RE.exec(lines[i].trim());
-      if (!m) {
-        continue;
-      }
-      var itemName = m[1].replace(/^\/+/, '');
-      if (!itemName) {
-        continue;
-      }
-
-      // filter /nightlies/*
-      if (itemName.indexOf('nightlies/') === 0) {
-        continue;
-      }
-
-      items.push({
-        name: itemName, // 'SHASUMS.txt', 'x64/'
-        date: m[2],
-        size: m[3] === '-' ? '-' : parseInt(m[3]),
-        type: m[3] === '-' ? 'dir' : 'file',
-        parent: fullname, // '/', '/v0.10.28/'
-      });
-    }
-  }
-
-  // node <= v0.11.11, /docs/ is not list, has a index.html
-  if (items.length === 0 && /\/docs\/$/.test(fullname)) {
-    items.push({
-      name: 'api/',
-      date: date,
-      size: '-',
-      type: 'dir',
-      parent: fullname,
-    });
-
-    // sh_main.js
-    // sh_javascript.min.js
-    items.push({
-      name: 'sh_main.js',
-      date: date,
-      size: 0,
-      type: 'file',
-      parent: fullname,
-    });
-
-    items.push({
-      name: 'sh_javascript.min.js',
-      date: date,
-      size: 0,
-      type: 'file',
-      parent: fullname,
-    });
-  }
-
-  return items;
-};
-
-proto.listdiff = function* (fullname) {
-  var items = yield* this.listdir(fullname);
-  if (items.length === 0) {
-    return items;
-  }
-  var exists = yield* distService.listdir(fullname);
-  debug('listdiff %s got %s exists items', fullname, exists.length);
-  var map = {};
-  for (var i = 0; i < exists.length; i++) {
-    var item = exists[i];
-    map[item.name] = item;
-  }
-  var news = [];
-  for (var i = 0; i < items.length; i++) {
-    var item = items[i];
-    var exist = map[item.name];
-    if (!exist || exist.date !== item.date) {
-      news.push(item);
-      continue;
-    }
-
-    if (item.size !== '-' && item.size !== exist.size) {
-      news.push(item);
-      continue;
-    }
-
-    debug('skip %s', item.name);
-  }
-  return news;
-};
-
-proto.syncPhantomjsDir = function* () {
-  var fullname = '/phantomjs/';
-  var files = yield* this.listPhantomjsDiff(fullname);
-
-  logger.syncInfo('sync remote:%s got %d files to sync',
-    fullname, files.length);
-
-  for (var i = 0; i < files.length; i++) {
-    yield* this.syncFile(files[i]);
-  }
-
-  logger.syncInfo('SyncPhantomjsDir %s finished, %d files',
-    fullname, files.length);
-};
-
-// <tr class="iterable-item" id="download-301626">
-//   <td class="name"><a class="execute" href="/ariya/phantomjs/downloads/phantomjs-1.9.7-windows.zip">phantomjs-1.9.7-windows.zip</a></td>
-//   <td class="size">6.7 MB</td>
-//   <td class="uploaded-by"><a href="/Vitallium">Vitallium</a></td>
-//   <td class="count">122956</td>
-//   <td class="date">
-//     <div>
-//       <time datetime="2014-01-27T18:29:53.706942" data-title="true">2014-01-27</time>
-//     </div>
-//   </td>
-//   <td class="delete">
-//
-//   </td>
-// </tr>
-
-proto.listPhantomjsDir = function* (fullname) {
-  var url = 'https://bitbucket.org/ariya/phantomjs/downloads';
-  var result = yield urllib.request(url, {
-    timeout: 60000,
-  });
-  debug('listPhantomjsDir %s got %s, %j', url, result.status, result.headers);
-  var html = result.data && result.data.toString() || '';
-  var $ = cheerio.load(html);
-  var items = [];
-  $('tr.iterable-item').each(function (_, el) {
-    var $el = $(this);
-    var $link = $el.find('.name a');
-    var name = $link.text();
-    var downloadURL = $link.attr('href');
-    if (!name || !downloadURL || !/\.(zip|bz2|gz)$/.test(downloadURL)) {
-      return;
-    }
-    downloadURL = urlResolve(url, downloadURL);
-    var size = parseInt(bytes($el.find('.size').text().toLowerCase().replace(/\s/g, '')));
-    if (size > 1024 * 1024) {
-      size -= 1024 * 1024;
-    } else if (size > 1024) {
-      size -= 1024;
-    } else {
-      size -= 10;
-    }
-    var date = $el.find('.date time').text();
-    items.push({
-      name: name, // 'SHASUMS.txt', 'x64/'
-      date: date,
-      size: size,
-      type: 'file',
-      parent: fullname,
-      downloadURL: downloadURL,
-    });
-  });
-  return items;
-};
-
-proto.listPhantomjsDiff = function* (fullname) {
-  var items = yield* this.listPhantomjsDir(fullname);
-  if (items.length === 0) {
-    return items;
-  }
-  var exists = yield* distService.listdir(fullname);
-  debug('listdiff %s got %s exists items', fullname, exists.length);
-  var map = {};
-  for (var i = 0; i < exists.length; i++) {
-    var item = exists[i];
-    map[item.name] = item;
-  }
-  var news = [];
-  for (var i = 0; i < items.length; i++) {
-    var item = items[i];
-    var exist = map[item.name];
-    if (!exist || exist.date !== item.date) {
-      news.push(item);
-      continue;
-    }
-
-    // if (item.size !== exist.size) {
-    //   news.push(item);
-    //   continue;
-    // }
-
-    debug('skip %s', item.name);
-  }
-  return news;
-};

sync/sync_popular.js

@@ -20,9 +20,15 @@ var config = require('../config');
 var npmService = require('../services/npm');
 var Status = require('./status');
 var SyncModuleWorker = require('../controllers/sync_module_worker');
+var logger = require('../common/logger');
 
 module.exports = function* syncPopular() {
   var packages = yield* npmService.getPopular(config.topPopular);
+  packages = packages.map(function (r) {
+    return r[0];
+  });
+
+  logger.syncInfo('Syncing %d popular packages, top 10: %j', packages.length, packages.slice(0, 10));
 
   var worker = new SyncModuleWorker({
     username: 'admin',

test/common/markdown.test.js

@@ -0,0 +1,43 @@
+/**!
+ * cnpmjs.org - test/common/markdown.test.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var utils = require('../utils');
+var markdown = require('../../common/markdown');
+
+describe('common/markdown.test.js', function () {
+  it('should render sonido readme', function () {
+    var readme = utils.getFileContent('sonido.md');
+    var md = markdown.render(readme);
+    md.should.equal('<p>Configuration Wizard: &lt;!--- This is what the user will see during the configuration -&gt;</p>\n');
+  });
+
+  it('should filter xss', function () {
+    var html = markdown.render('foo<script>alert(1)</script>/xss\n[foo](/foo) <a onclick="alert(1)">bar</a>\n"\'');
+    html.should.equal('<p>foo&lt;script&gt;alert(1)&lt;/script&gt;/xss\n<a href="/foo">foo</a> <a>bar</a>\n&quot;\'</p>\n');
+    markdown.render('[xss link](javascript:alert(2))').should.equal('<p>[xss link](javascript:alert(2))</p>\n');
+  });
+
+  it('should handle eat cpu markdown', function () {
+    var readme = utils.getFileContent('eat-cpu.md');
+    markdown.render(readme);
+  });
+
+  it('should not filter < and > on code', function () {
+    var content = utils.getFileContent('code.md');
+    var html = markdown.render(content);
+    html.should.containEql('<pre><code class="language-html">&lt;body&gt;hi&lt;/body&gt;');
+  });
+});

test/controllers/registry/module/public_mode.test.js

@@ -1,571 +0,0 @@
-/*!
- * cnpmjs.org - test/controllers/registry/module/public_mode.test.js
- * Copyright(c) 2014 dead_horse <dead_horse@qq.com>
- * MIT Licensed
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var should = require('should');
-var request = require('supertest');
-var mm = require('mm');
-var pedding = require('pedding');
-var config = require('../../../../config');
-var app = require('../../../../servers/registry');
-var utils = require('../../../utils');
-
-describe('controllers/registry/module/public_module.test.js', function () {
-  beforeEach(function () {
-    mm(config, 'enablePrivate', false);
-  });
-  before(function (done) {
-    mm(config, 'enablePrivate', false);
-    mm(config, 'forcePublishWithScope', false);
-    app = app.listen(0, function () {
-      done = pedding(2, done);
-      // name: publictestmodule
-      var pkg = utils.getPackage('publictestmodule', '0.0.1', utils.otherUser);
-
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .expect(201, function (err) {
-        should.not.exist(err);
-        pkg = utils.getPackage('publictestmodule', '0.0.2', utils.otherUser);
-        // publish 0.0.2
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(201, done);
-      });
-
-      // publicputmodule@0.1.9
-      var testpkg = utils.getPackage('publicputmodule', '0.1.9', utils.otherUser);
-
-      request(app)
-      .put('/' + testpkg.name)
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
-  });
-  afterEach(mm.restore);
-
-  describe('PUT /:name publish new flow addPackageAndDist()', function () {
-    beforeEach(function () {
-      mm(config, 'enablePrivate', false);
-      mm(config, 'forcePublishWithScope', false);
-    });
-
-    it('should publish with tgz base64, addPackageAndDist()', function (done) {
-      var pkg = utils.getPackage('publicpublishmodule', '0.0.2', utils.otherUser);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .expect(201, function (err, res) {
-        should.not.exist(err);
-        res.body.should.have.keys('ok', 'rev');
-        res.body.ok.should.equal(true);
-        pkg = utils.getPackage('publicpublishmodule', '0.0.2', utils.otherUser);
-        // upload again should 403
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(403, function (err, res) {
-          should.not.exist(err);
-          res.body.should.eql({
-            error: 'forbidden',
-            reason: 'cannot modify pre-existing version: 0.0.2'
-          });
-          done();
-        });
-      });
-    });
-
-    it('should other user pulbish 403', function (done) {
-      var pkg = utils.getPackage('publicpublishmodule', '0.0.3', utils.secondUser);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.secondUserAuth)
-      .send(pkg)
-      .expect(403, done);
-    });
-
-    it('should admin pulbish 403', function (done) {
-      var pkg = utils.getPackage('publicpublishmodule', '0.0.3', utils.admin);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(403, done);
-    });
-
-    it('should publish with scope, addPackageAndDist()', function (done) {
-      mm(config, 'forcePublishWithScope', false);
-      var pkg = utils.getPackage('@cnpm/publicpublishmodule', '0.0.2', utils.otherUser);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .expect(201, function (err, res) {
-        should.not.exist(err);
-        res.body.should.have.keys('ok', 'rev');
-        res.body.ok.should.equal(true);
-
-        // upload again should 403
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(403, function (err, res) {
-          should.not.exist(err);
-          res.body.should.eql({
-            error: 'forbidden',
-            reason: 'cannot modify pre-existing version: 0.0.2'
-          });
-          done();
-        });
-      });
-    });
-
-    describe('forcePublishWithScope = true', function () {
-      it('should publish without scope 403, addPackageAndDist()', function (done) {
-        mm(config, 'forcePublishWithScope', false);
-        var pkg = utils.getPackage('publicpublishmodule', '0.0.2');
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(403, done);
-      });
-
-      it('should admin publish without scope ok, addPackageAndDist()', function (done) {
-        mm(config, 'forcePublishWithScope', false);
-        var pkg = utils.getPackage('publicpublishmodule1', '0.0.4', utils.admin);
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.adminAuth)
-        .send(pkg)
-        .expect(201, done);
-      });
-    });
-  });
-
-  describe('PUT /:name/-rev/:rev removeWithVersions', function () {
-    var withoutScopeRev;
-    before(function (done) {
-      mm(config, 'enablePrivate', false);
-      mm(config, 'forcePublishWithScope', false);
-      var pkg = utils.getPackage('publicremovemodule', '0.0.1', utils.otherUser);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .expect(201, function (err, res) {
-        should.not.exist(err);
-        res.body.should.have.keys('ok', 'rev');
-        res.body.ok.should.equal(true);
-
-        pkg = utils.getPackage('publicremovemodule', '0.0.2', utils.otherUser);
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(201, function (err, res) {
-          should.not.exist(err);
-          withoutScopeRev = res.body.rev;
-          done();
-        });
-      });
-    });
-
-    it('should remove with version ok', function (done) {
-      request(app)
-      .put('/publicremovemodule/-rev/' + withoutScopeRev)
-      .set('authorization', utils.otherUserAuth)
-      .send({
-        versions: {
-          '0.0.1': {}
-        }
-      })
-      .expect(201, done);
-    });
-
-    it('should no auth user remove 403', function (done) {
-      request(app)
-      .put('/publicremovemodule/-rev/' + withoutScopeRev)
-      .set('authorization', utils.secondUserAuth)
-      .send({
-        versions: {
-          '0.0.1': {}
-        }
-      })
-      .expect(403, done);
-    });
-
-    it('should admin remove ok', function (done) {
-      request(app)
-      .put('/publicremovemodule/-rev/' + withoutScopeRev)
-      .set('authorization', utils.adminAuth)
-      .send({
-        versions: {
-          '0.0.1': {}
-        }
-      })
-      .expect(201, done);
-    });
-
-    describe('forcePublishWithScope = true', function () {
-      var withScopeRev;
-      before(function (done) {
-        mm(config, 'enablePrivate', false);
-        mm(config, 'forcePublishWithScope', true);
-        var pkg = utils.getPackage('@cnpm/publicremovemodule', '0.0.1', utils.otherUser);
-        request(app)
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(201, function (err, res) {
-          should.not.exist(err);
-          res.body.should.have.keys('ok', 'rev');
-          res.body.ok.should.equal(true);
-
-          pkg = utils.getPackage('@cnpm/publicremovemodule', '0.0.2', utils.otherUser);
-          request(app)
-          .put('/' + pkg.name)
-          .set('authorization', utils.otherUserAuth)
-          .send(pkg)
-          .expect(201, function (err, res) {
-            should.not.exist(err);
-            withScopeRev = res.body.rev;
-            done();
-          });
-        });
-      });
-
-      it('should remove without scope 403', function (done) {
-        mm(config, 'forcePublishWithScope', true);
-        request(app)
-        .put('/publicremovemodule/-rev/' + withoutScopeRev)
-        .set('authorization', utils.otherUserAuth)
-        .send({
-          versions: {
-            '0.0.1': {}
-          }
-        })
-        .expect(403, done);
-      });
-
-      it('should admin remove without scope ok', function (done) {
-        mm(config, 'forcePublishWithScope', true);
-        request(app)
-        .put('/publicremovemodule/-rev/' + withoutScopeRev)
-        .set('authorization', utils.adminAuth)
-        .send({
-          versions: {
-            '0.0.1': {}
-          }
-        })
-        .expect(201, done);
-      });
-
-      it('should remove with scope ok', function (done) {
-        mm(config, 'forcePublishWithScope', true);
-        request(app)
-        .put('/@cnpm/publicremovemodule/-rev/' + withScopeRev)
-        .set('authorization', utils.otherUserAuth)
-        .send({
-          versions: {
-            '0.0.1': {}
-          }
-        })
-        .expect(201, done);
-      });
-
-      it('should admin remove with scope ok', function (done) {
-        mm(config, 'forcePublishWithScope', true);
-        request(app)
-        .put('/@cnpm/publicremovemodule/-rev/' + withScopeRev)
-        .set('authorization', utils.adminAuth)
-        .send({
-          versions: {
-            '0.0.1': {}
-          }
-        })
-        .expect(201, done);
-      });
-    });
-  });
-
-  describe('DELETE /:name/download/:filename/-rev/:rev', function () {
-    var withoutScopeRev;
-    beforeEach(function () {
-      mm(config, 'enablePrivate', false);
-      mm(config, 'forcePublishWithScope', false);
-    });
-    beforeEach(function (done) {
-      mm(config, 'enablePrivate', false);
-      mm(config, 'forcePublishWithScope', false);
-      var pkg = utils.getPackage('public-test-delete-download-module', '0.1.9', utils.otherUser);
-      request(app)
-      .put('/' + pkg.name)
-      .set('content-type', 'application/json')
-      .set('authorization', utils.otherUserAuth)
-      .send(pkg)
-      .end(function (err, res) {
-        should.not.exist(err);
-        if (res.body.rev) {
-          withoutScopeRev = res.body.rev;
-        }
-        done();
-      });
-    });
-
-    it('should delete 403 when auth error', function (done) {
-      request(app)
-      .del('/public-test-delete-download-module/download/public-test-delete-download-module-0.1.9.tgz/-rev/' + withoutScopeRev)
-      .set('authorization', utils.secondUserAuth)
-      .expect(403, done);
-    });
-
-    it('should delete file ok', function (done) {
-      request(app)
-      .del('/public-test-delete-download-module/download/public-test-delete-download-module-0.1.9.tgz/-rev/' + withoutScopeRev)
-      .set('authorization', utils.otherUserAuth)
-      .expect(200, done);
-    });
-
-    it('should admin delete file ok', function (done) {
-      request(app)
-      .del('/public-test-delete-download-module/download/public-test-delete-download-module-0.1.9.tgz/-rev/' + withoutScopeRev)
-      .set('authorization', utils.adminAuth)
-      .expect(200, done);
-    });
-
-    describe('forcePublishWithScope = true', function () {
-      var withScopeRev;
-      beforeEach(function () {
-        mm(config, 'enablePrivate', false);
-        mm(config, 'forcePublishWithScope', true);
-      });
-      beforeEach(function (done) {
-        mm(config, 'enablePrivate', false);
-        mm(config, 'forcePublishWithScope', true);
-        var pkg = utils.getPackage('@cnpm/public-test-delete-download-module', '0.1.9', utils.otherUser);
-        request(app)
-        .put('/' + pkg.name)
-        .set('content-type', 'application/json')
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .end(function (err, res) {
-          should.not.exist(err);
-          if (res.body.rev) {
-            withScopeRev = res.body.rev;
-          }
-          done();
-        });
-      });
-
-      it('should delete file without scope 403', function (done) {
-        request(app)
-        .del('/public-test-delete-download-module/download/public-test-delete-download-module-0.1.9.tgz/-rev/' + withoutScopeRev)
-        .set('authorization', utils.otherUserAuth)
-        .expect(403, done);
-      });
-
-      it('should admin delete file without scope ok', function (done) {
-        request(app)
-        .del('/public-test-delete-download-module/download/public-test-delete-download-module-0.1.9.tgz/-rev/' + withoutScopeRev)
-        .set('authorization', utils.adminAuth)
-        .expect(200, done);
-      });
-      it('should delete file with scope ok', function (done) {
-        request(app)
-        .del('/@cnpm/public-test-delete-download-module/download/@cnpm/public-test-delete-download-module-0.1.9.tgz/-rev/' + withScopeRev)
-        .set('authorization', utils.otherUserAuth)
-        .expect(200, done);
-      });
-
-      it('should admin delete file with scope ok', function (done) {
-        request(app)
-        .del('/@cnpm/public-test-delete-download-module/download/@cnpm/public-test-delete-download-module-0.1.9.tgz/-rev/' + withScopeRev)
-        .set('authorization', utils.adminAuth)
-        .expect(200, done);
-      });
-    });
-  });
-
-  describe('PUT /:name/:tag updateTag()', function () {
-    it('should create new tag ok', function (done) {
-      request(app)
-      .put('/publictestmodule/newtag')
-      .set('content-type', 'application/json')
-      .set('authorization', utils.otherUserAuth)
-      .send('"0.0.1"')
-      .expect(201, done);
-    });
-
-    it('shold update tag not maintainer 403', function (done) {
-      request(app)
-      .put('/publictestmodule/newtag')
-      .set('content-type', 'application/json')
-      .set('authorization', utils.secondUserAuth)
-      .send('"0.0.1"')
-      .expect(403, done);
-    });
-
-    it('should admin update tag ok', function (done) {
-      request(app)
-      .put('/publictestmodule/newtag')
-      .set('content-type', 'application/json')
-      .set('authorization', utils.adminAuth)
-      .send('"0.0.1"')
-      .expect(201, done);
-    });
-  });
-
-  describe('DELETE /:name/-rev/:rev', function () {
-    describe('remove all modules by name', function () {
-      beforeEach(function () {
-        mm(config, 'enablePrivate', false);
-        mm(config, 'forcePublishWithScope', false);
-      });
-      before(function (done) {
-        mm(config, 'enablePrivate', false);
-        mm(config, 'forcePublishWithScope', false);
-        var pkg = utils.getPackage('public-remove-all-module', '0.0.1', utils.otherUser);
-        request(app)
-        .put('/public-remove-all-module')
-        .set('content-type', 'application/json')
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(201, function (err) {
-          should.not.exist(err);
-          var pkg = utils.getPackage('public-remove-all-module-admin', '0.0.1', utils.otherUser);
-          request(app)
-          .put('/public-remove-all-module-admin')
-          .set('content-type', 'application/json')
-          .set('authorization', utils.otherUserAuth)
-          .send(pkg)
-          .expect(201, done);
-        });
-      });
-
-      it('should fail when user not maintainer', function (done) {
-        request(app)
-        .del('/public-remove-all-module/-rev/1')
-        .set('authorization', utils.secondUserAuth)
-        .expect(403, function (err, res) {
-          should.not.exist(err);
-          res.body.should.eql({
-            error: 'forbidden user',
-            reason: 'cnpmjstest102 not authorized to modify public-remove-all-module'
-          });
-          done();
-        });
-      });
-
-      it('should maintainer remove ok', function (done) {
-        request(app)
-        .del('/public-remove-all-module/-rev/1')
-        .set('authorization', utils.otherUserAuth)
-        .expect(200, function (err, res) {
-          should.not.exist(err);
-          should.not.exist(res.headers['set-cookie']);
-          done();
-        });
-      });
-
-      it('should admin remove ok', function (done) {
-        request(app)
-        .del('/public-remove-all-module-admin/-rev/1')
-        .set('authorization', utils.adminAuth)
-        .expect(200, function (err, res) {
-          should.not.exist(err);
-          should.not.exist(res.headers['set-cookie']);
-          done();
-        });
-      });
-
-      describe('forcePublishWithScope = true', function () {
-        before(function (done) {
-          mm(config, 'enablePrivate', false);
-          mm(config, 'forcePublishWithScope', true);
-          var pkg = utils.getPackage('@cnpm/public-remove-all-module', '0.0.1', utils.otherUser);
-          request(app)
-          .put('/@cnpm/public-remove-all-module')
-          .set('content-type', 'application/json')
-          .set('authorization', utils.otherUserAuth)
-          .send(pkg)
-          .expect(201, function (err) {
-            should.not.exist(err);
-            var pkg = utils.getPackage('@cnpm/public-remove-all-module-admin', '0.0.1', utils.otherUser);
-            request(app)
-            .put('/@cnpm/public-remove-all-module-admin')
-            .set('content-type', 'application/json')
-            .set('authorization', utils.otherUserAuth)
-            .send(pkg)
-            .expect(201, function (err) {
-              should.not.exist(err);
-              var pkg = utils.getPackage('public-remove-all-module-admin', '0.1.1', utils.admin);
-              request(app)
-              .put('/public-remove-all-module-admin')
-              .set('content-type', 'application/json')
-              .set('authorization', utils.adminAuth)
-              .send(pkg)
-              .expect(201, done);
-            });
-          });
-        });
-
-        it('should fail when user remove module without scope', function (done) {
-          mm(config, 'forcePublishWithScope', true);
-          request(app)
-          .del('/public-remove-all-module/-rev/1')
-          .set('authorization', utils.otherUserAuth)
-          .expect(403, done);
-        });
-
-        it('should admin remove module without scope ok', function (done) {
-          mm(config, 'forcePublishWithScope', true);
-          request(app)
-          .del('/public-remove-all-module-admin/-rev/1')
-          .set('authorization', utils.adminAuth)
-          .expect(200, done);
-        });
-
-        it('should maintainer remove ok', function (done) {
-          mm(config, 'forcePublishWithScope', true);
-          request(app)
-          .del('/@cnpm/public-remove-all-module/-rev/1')
-          .set('authorization', utils.otherUserAuth)
-          .expect(200, function (err, res) {
-            should.not.exist(err);
-            should.not.exist(res.headers['set-cookie']);
-            done();
-          });
-        });
-
-        it('should admin remove ok', function (done) {
-          mm(config, 'forcePublishWithScope', true);
-          request(app)
-          .del('/@cnpm/public-remove-all-module-admin/-rev/1')
-          .set('authorization', utils.adminAuth)
-          .expect(200, function (err, res) {
-            should.not.exist(err);
-            should.not.exist(res.headers['set-cookie']);
-            done();
-          });
-        });
-      });
-    });
-  });
-});

test/controllers/registry/package/deprecate.test.js

@@ -22,9 +22,8 @@ var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
 describe('controllers/registry/package/deprecate.test.js', function () {
-  var pkgname = 'testmodule-deprecate';
+  var pkgname = '@cnpmtest/testmodule-deprecate';
   before(function (done) {
-    done = pedding(2, done);
     var pkg = utils.getPackage(pkgname, '0.0.1', utils.admin);
 
     request(app.listen())
@@ -49,36 +48,11 @@ describe('controllers/registry/package/deprecate.test.js', function () {
         .expect(201, done);
       });
     });
-
-    pkg = utils.getPackage('@cnpmtest/testmodule-deprecate', '1.0.0', utils.admin);
-    request(app.listen())
-    .put('/' + pkg.name)
-    .set('authorization', utils.adminAuth)
-    .send(pkg)
-    .expect(201, done);
   });
 
   afterEach(mm.restore);
 
   describe('PUT /:name', function () {
-    it('should deprecate scoped package version@1.0.0', function (done) {
-      request(app.listen())
-      .put('/@cnpmtest/testmodule-deprecate')
-      .set('authorization', utils.adminAuth)
-      .send({
-        name: pkgname,
-        versions: {
-          '1.0.0': {
-            deprecated: 'mock test deprecated message 1.0.0'
-          }
-        }
-      })
-      .expect({
-        ok: true
-      })
-      .expect(201, done);
-    });
-
     it('should deprecate version@1.0.0', function (done) {
       request(app.listen())
       .put('/' + pkgname)
@@ -205,7 +179,7 @@ describe('controllers/registry/package/deprecate.test.js', function () {
       .expect(400, done);
     });
 
-    it('should 403', function (done) {
+    it('should 403 can not modified when use is not maintainer', function (done) {
       request(app.listen())
       .put('/' + pkgname)
       .set('authorization', utils.otherUserAuth)
@@ -224,8 +198,8 @@ describe('controllers/registry/package/deprecate.test.js', function () {
         }
       })
       .expect({
-        error: 'no_perms',
-        reason: 'Private mode enable, only admin can publish this module'
+        error: 'forbidden user',
+        reason: 'cnpmjstest101 not authorized to modify @cnpmtest/testmodule-deprecate, please contact maintainers: cnpmjstest10'
       })
       .expect(403, done);
     });

test/controllers/registry/package/dist_tag.test.js

@@ -0,0 +1,316 @@
+/**!
+ * cnpmjs.org - test/controllers/registry/package/dist_tag.test.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var request = require('supertest');
+var mm = require('mm');
+var pedding = require('pedding');
+var should = require('should');
+var config = require('../../../../config');
+var app = require('../../../../servers/registry');
+var utils = require('../../../utils');
+
+describe('controllers/registry/package/dist_tag.test.js', function () {
+  afterEach(mm.restore);
+
+  describe('index()', function () {
+    before(function (done) {
+      done = pedding(2, done);
+      utils.sync('byte', done);
+
+      var pkg2 = utils.getPackage('@cnpmtest/dist_tag_test_module_index', '1.0.1', utils.otherUser);
+      request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/' + pkg2.name + '/dist-tags/next')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect(201, done);
+      });
+    });
+
+    it('should get normal pakcage tags', function (done) {
+      mm(config, 'syncModel', 'all');
+      request(app.listen())
+      .get('/-/package/byte/dist-tags')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.latest.should.be.a.String;
+        done();
+      });
+    });
+
+    it('should get scoped pakcage tags', function (done) {
+      request(app.listen())
+      .get('/-/package/@cnpmtest/dist_tag_test_module_index/dist-tags')
+      .expect(200)
+      .expect({
+        latest: '1.0.1',
+        next: '1.0.1'
+      }, done);
+    });
+
+    it('should 404 when package not exists', function (done) {
+      request(app.listen())
+      .get('/-/package/@cnpmtest/not-exists/dist-tags')
+      .expect(404)
+      .expect({
+        error: 'not_found',
+        reason: 'document not found'
+      }, done);
+    });
+  });
+
+  describe('set()', function () {
+    before(function (done) {
+      var pkg2 = utils.getPackage('@cnpmtest/dist_tag_test_module_set', '1.0.1', utils.otherUser);
+      request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/' + pkg2.name + '/dist-tags/next')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect(201, done);
+      });
+    });
+
+    it('should 400 when set not exists version', function (done) {
+      request(app.listen())
+      .put('/-/package/@cnpmtest/dist_tag_test_module_set/dist-tags/next')
+      .set('authorization', utils.otherUserAuth)
+      .set('content-type', 'application/json')
+      .send(JSON.stringify('2.0.1'))
+      .expect({
+        error: 'version_error',
+        reason: '@cnpmtest/dist_tag_test_module_set@2.0.1 not exists'
+      })
+      .expect(400, done);
+    });
+
+    it('should 201 set exists tag', function (done) {
+      request(app.listen())
+      .put('/-/package/@cnpmtest/dist_tag_test_module_set/dist-tags/exists')
+      .set('authorization', utils.otherUserAuth)
+      .set('content-type', 'application/json')
+      .send(JSON.stringify('1.0.1'))
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/@cnpmtest/dist_tag_test_module_set/dist-tags/exists')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect({
+          ok: 'dist-tags updated'
+        })
+        .expect(201, done);
+      });
+    });
+  });
+
+  describe('destroy()', function () {
+    before(function (done) {
+      var pkg2 = utils.getPackage('@cnpmtest/dist_tag_test_module_destroy', '1.0.1', utils.otherUser);
+      request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/' + pkg2.name + '/dist-tags/next')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect(201, done);
+      });
+    });
+
+    it('should destroy exists scoped tag', function (done) {
+      request(app.listen())
+      .delete('/-/package/@cnpmtest/dist_tag_test_module_destroy/dist-tags/next')
+      .set('authorization', utils.otherUserAuth)
+      .set('content-type', 'application/json')
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(200, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .get('/-/package/@cnpmtest/dist_tag_test_module_destroy/dist-tags')
+        .expect(200)
+        .expect({
+          latest: '1.0.1',
+        }, done);
+      });
+    });
+
+    it('should 404 destroy not exists tag', function (done) {
+      request(app.listen())
+      .delete('/-/package/@cnpmtest/dist_tag_test_module_destroy/dist-tags/not-exists')
+      .set('authorization', utils.otherUserAuth)
+      .set('content-type', 'application/json')
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(200, done);
+    });
+  });
+
+  describe('save()', function () {
+    before(function (done) {
+      var pkg2 = utils.getPackage('@cnpmtest/dist_tag_test_module_save', '1.0.1', utils.otherUser);
+      request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/' + pkg2.name + '/dist-tags/next')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect(201, done);
+      });
+    });
+
+    it('should overwrite exists tags', function (done) {
+      request(app.listen())
+      .put('/-/package/@cnpmtest/dist_tag_test_module_save/dist-tags')
+      .set('authorization', utils.otherUserAuth)
+      .send({
+        latest: '1.0.1',
+        new: '1.0.1'
+      })
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .get('/-/package/@cnpmtest/dist_tag_test_module_save/dist-tags')
+        .expect(200)
+        .expect({
+          latest: '1.0.1',
+          new: '1.0.1',
+        }, done);
+      });
+    });
+
+    it('should overwrite exists scoped tags', function (done) {
+      request(app.listen())
+      .put('/-/package/@cnpmtest/dist_tag_test_module_save/dist-tags')
+      .set('authorization', utils.otherUserAuth)
+      .send({
+        latest: '1.0.1',
+        new: '1.0.1'
+      })
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .get('/-/package/@cnpmtest/dist_tag_test_module_save/dist-tags')
+        .expect(200)
+        .expect({
+          latest: '1.0.1',
+          new: '1.0.1',
+        }, done);
+      });
+    });
+  });
+
+  describe('update()', function () {
+    before(function (done) {
+      var pkg2 = utils.getPackage('@cnpmtest/dist_tag_test_module_update', '1.0.1', utils.otherUser);
+      request(app.listen())
+      .put('/' + pkg2.name)
+      .set('authorization', utils.otherUserAuth)
+      .send(pkg2)
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .put('/-/package/' + pkg2.name + '/dist-tags/next')
+        .set('authorization', utils.otherUserAuth)
+        .set('content-type', 'application/json')
+        .send(JSON.stringify('1.0.1'))
+        .expect(201, done);
+      });
+    });
+
+    it('should merge exists tags', function (done) {
+      request(app.listen())
+      .post('/-/package/@cnpmtest/dist_tag_test_module_update/dist-tags')
+      .set('authorization', utils.otherUserAuth)
+      .send({
+        latest: '1.0.1',
+        new: '1.0.1'
+      })
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .get('/-/package/@cnpmtest/dist_tag_test_module_update/dist-tags')
+        .expect(200)
+        .expect({
+          latest: '1.0.1',
+          next: '1.0.1',
+          new: '1.0.1',
+        }, done);
+      });
+    });
+
+    it('should merge exists scoped tags', function (done) {
+      request(app.listen())
+      .post('/-/package/@cnpmtest/dist_tag_test_module_update/dist-tags')
+      .set('authorization', utils.otherUserAuth)
+      .send({
+        latest: '1.0.1',
+        new: '1.0.1'
+      })
+      .expect({
+        ok: 'dist-tags updated'
+      })
+      .expect(201, function (err) {
+        should.not.exist(err);
+        request(app.listen())
+        .get('/-/package/@cnpmtest/dist_tag_test_module_update/dist-tags')
+        .expect(200)
+        .expect({
+          latest: '1.0.1',
+          next: '1.0.1',
+          new: '1.0.1',
+        }, done);
+      });
+    });
+  });
+});

test/controllers/registry/package/download.test.js

@@ -20,7 +20,7 @@ var utils = require('../../../utils');
 
 describe('controllers/registry/package/download.test.js', function () {
   before(function (done) {
-    var pkg = utils.getPackage('download-test-module', '1.0.0', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/download-test-module', '1.0.0', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
@@ -31,19 +31,19 @@ describe('controllers/registry/package/download.test.js', function () {
   describe('GET /:name/download/:filename', function () {
     it('should download a file with 200', function (done) {
       request(app.listen())
-      .get('/download-test-module/download/download-test-module-1.0.0.tgz')
+      .get('/@cnpmtest/download-test-module/download/@cnpmtest/download-test-module-1.0.0.tgz')
       .expect(200, done);
     });
 
     it('should alias /:name/-/:filename to /:name/download/:filename', function (done) {
       request(app.listen())
-      .get('/download-test-module/-/download-test-module-1.0.0.tgz')
+      .get('/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
       .expect(200, done);
     });
 
     it('should 404 when package not exists', function (done) {
       request(app.listen())
-      .get('/download-test-module-not-exists/download/download-test-module-not-exists-1.0.0.tgz')
+      .get('/@cnpmtest/download-test-module-not-exists/download/@cnpmtest/download-test-module-not-exists-1.0.0.tgz')
       .expect(404, done);
     });
   });

test/controllers/registry/package/download_total.test.js

@@ -0,0 +1,103 @@
+/**!
+ * cnpmjs.org - test/controllers/registry/package/download_total.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var request = require('supertest');
+var mm = require('mm');
+var DownloadTotal = require('../../../../services/download_total');
+var app = require('../../../../servers/registry');
+
+describe('controllers/registry/package/download_total.test.js', function () {
+  afterEach(mm.restore);
+
+  it('should error when range error', function (done) {
+    request(app.listen())
+    .get('/downloads/range/2014-10-10:xxxx/koa')
+    .expect(400)
+    .expect({
+      error: 'range_error',
+      reason: 'range must be YYYY-MM-DD:YYYY-MM-DD style'
+    }, done);
+  });
+
+  it('should get package downloads ok', function (done) {
+    mm.data(DownloadTotal, 'getModuleTotal', [{
+      id: 1,
+      count: 10,
+      date: '2014-12-03',
+      name: 'koa'
+    }, {
+      id: 1,
+      count: 8,
+      date: '2014-12-01',
+      name: 'koa'
+    }, {
+      id: 1,
+      count: 5,
+      date: '2014-12-02',
+      name: 'koa'
+    }]);
+
+    request(app.listen())
+    .get('/downloads/range/2014-12-01:2014-12-03/koa')
+    .expect(200)
+    .expect({
+      start: '2014-12-01',
+      end: '2014-12-03',
+      package: 'koa',
+      downloads: [{
+        day: '2014-12-01',
+        downloads: 8
+      }, {
+        day: '2014-12-02',
+        downloads: 5
+      }, {
+        day: '2014-12-03',
+        downloads: 10
+      }]
+    }, done);
+  });
+
+  it('should get total downloads ok', function (done) {
+    mm.data(DownloadTotal, 'getTotal', [{
+      count: 20,
+      date: '2014-12-03',
+    }, {
+      count: 8,
+      date: '2014-12-01',
+    }, {
+      count: 5,
+      date: '2014-12-02',
+    }]);
+
+    request(app.listen())
+    .get('/downloads/range/2014-12-01:2014-12-03')
+    .expect(200)
+    .expect({
+      start: '2014-12-01',
+      end: '2014-12-03',
+      downloads: [{
+        day: '2014-12-01',
+        downloads: 8
+      }, {
+        day: '2014-12-02',
+        downloads: 5
+      }, {
+        day: '2014-12-03',
+        downloads: 20
+      }]
+    }, done);
+  });
+});

test/controllers/registry/package/list.test.js

@@ -17,35 +17,38 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var pedding = require('pedding');
 var packageService = require('../../../../services/package');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
-var SyncModuleWorker = require('../../../../controllers/sync_module_worker');
 var config = require('../../../../config');
 
 describe('controllers/registry/package/list.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-list-1', '0.0.1', utils.admin);
+    done = pedding(2, done);
+    utils.sync('baidu', done);
+
+    var pkg = utils.getPackage('@cnpmtest/testmodule-list-1', '0.0.1', utils.otherUser);
     pkg.versions['0.0.1'].dependencies = {
       bytetest: '~0.0.1',
       mocha: '~1.0.0'
     };
     request(app.listen())
     .put('/' + pkg.name)
-    .set('authorization', utils.adminAuth)
+    .set('authorization', utils.otherUserAuth)
     .send(pkg)
     .expect(201, function (err) {
       should.not.exist(err);
-      var pkg = utils.getPackage('testmodule-list-1', '1.0.0', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/testmodule-list-1', '1.0.0', utils.otherUser);
       pkg.versions['1.0.0'].dependencies = {
         bytetest: '~0.0.1',
         mocha: '~1.0.0'
       };
       request(app.listen())
       .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
+      .set('authorization', utils.otherUserAuth)
       .send(pkg)
       .expect(201, done);
     });
@@ -53,16 +56,16 @@ describe('controllers/registry/package/list.test.js', function () {
 
   it('should return all versions', function (done) {
     request(app.listen())
-    .get('/testmodule-list-1')
+    .get('/@cnpmtest/testmodule-list-1')
     .expect(200, function (err, res) {
       should.not.exist(err);
       var data = res.body;
-      data.name.should.equal('testmodule-list-1');
+      data.name.should.equal('@cnpmtest/testmodule-list-1');
       Object.keys(data.versions).should.eql(['1.0.0', '0.0.1']);
 
       // should 304
       request(app)
-      .get('/testmodule-list-1')
+      .get('/@cnpmtest/testmodule-list-1')
       .set('If-None-Match', res.headers.etag)
       .expect(304, done);
     });
@@ -73,11 +76,11 @@ describe('controllers/registry/package/list.test.js', function () {
       return ['fengmk2', 'foouser'];
     });
     request(app.listen())
-    .get('/testmodule-list-1')
+    .get('/@cnpmtest/testmodule-list-1')
     .expect(200, function (err, res) {
       should.not.exist(err);
       var data = res.body;
-      data.name.should.equal('testmodule-list-1');
+      data.name.should.equal('@cnpmtest/testmodule-list-1');
       data.users.should.eql({
         fengmk2: true,
         foouser: true
@@ -88,7 +91,7 @@ describe('controllers/registry/package/list.test.js', function () {
 
   it('should 404 when package not exists', function (done) {
     request(app.listen())
-    .get('/not-exists-package')
+    .get('/@cnpmtest/not-exists-package')
     .expect(404)
     .expect({
       error: 'not_found',
@@ -99,7 +102,7 @@ describe('controllers/registry/package/list.test.js', function () {
   it('should not sync not-exists package when config.syncByInstall = false', function (done) {
     mm(config, 'syncByInstall', false);
     request(app.listen())
-    .get('/not-exists-package')
+    .get('/@cnpmtest/not-exists-package')
     .expect(404)
     .expect({
       error: 'not_found',
@@ -110,8 +113,8 @@ describe('controllers/registry/package/list.test.js', function () {
   it('should sync not-exists package when config.syncByInstall = true', function (done) {
     mm(config, 'syncByInstall', true);
     request(app.listen())
-    .get('/pedding')
-    .expect(200, done);
+    .get('/should')
+    .expect(302, done);
   });
 
   it('should not sync not-exists scoped package', function (done) {
@@ -127,15 +130,11 @@ describe('controllers/registry/package/list.test.js', function () {
 
   describe('unpublished', function () {
     before(function (done) {
-      var worker = new SyncModuleWorker({
-        name: 'tfs',
-        noDep: true,
-      });
-      worker.start();
-      worker.on('end', done);
+      utils.sync('tfs', done);
     });
 
     it('should show unpublished info', function (done) {
+      mm(config, 'syncModel', 'all');
       request(app.listen())
       .get('/tfs')
       .expect(404, function (err, res) {
@@ -150,15 +149,11 @@ describe('controllers/registry/package/list.test.js', function () {
 
   describe('npm package', function () {
     before(function (done) {
-      var worker = new SyncModuleWorker({
-        name: 'tair',
-        noDep: true,
-      });
-      worker.start();
-      worker.on('end', done);
+      utils.sync('tair', done);
     });
 
     it('should show npm package after sync', function (done) {
+      mm(config, 'syncModel', 'all');
       request(app.listen())
       .get('/tair')
       .expect(200, function (err, res) {
@@ -180,7 +175,7 @@ describe('controllers/registry/package/list.test.js', function () {
     var tagModified;
     before(function (done) {
       request(app.listen())
-      .put('/testmodule-list-1/test-tag')
+      .put('/@cnpmtest/testmodule-list-1/test-tag')
       .set('content-type', 'application/json')
       .set('authorization', utils.adminAuth)
       .send(JSON.stringify('0.0.1'))
@@ -194,11 +189,11 @@ describe('controllers/registry/package/list.test.js', function () {
 
     it('should use tag gmt_modified', function (done) {
       request(app.listen())
-      .get('/testmodule-list-1')
+      .get('/@cnpmtest/testmodule-list-1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var data = res.body;
-        data.name.should.equal('testmodule-list-1');
+        data.name.should.equal('@cnpmtest/testmodule-list-1');
         data.time.modified.should.equal(tagModified);
         done();
       });

test/controllers/registry/package/list_all.test.js

@@ -17,6 +17,7 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var config = require('../../../../config');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
@@ -24,12 +25,8 @@ describe('controllers/registry/package/list_all.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-list_all', '0.0.1', utils.admin);
-    request(app.listen())
-    .put('/' + pkg.name)
-    .set('authorization', utils.adminAuth)
-    .send(pkg)
-    .expect(201, done);
+    mm(config, 'syncModel', 'all');
+    utils.sync('pedding', done);
   });
 
   describe('GET /-/all', function () {
@@ -41,7 +38,7 @@ describe('controllers/registry/package/list_all.test.js', function () {
         res.body.should.be.an.Object;
         res.body._updated.should.be.a.Number;
         Object.keys(res.body).length.should.above(1);
-        res.body['testmodule-list_all'].should.equal(true);
+        res.body.pedding.should.equal(true);
         done();
       });
     });

test/controllers/registry/package/list_shorts.test.js

@@ -17,6 +17,7 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var config = require('../../../../config');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
@@ -24,12 +25,8 @@ describe('controllers/registry/package/list_shorts.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-list_shorts', '0.0.1', utils.admin);
-    request(app.listen())
-    .put('/' + pkg.name)
-    .set('authorization', utils.adminAuth)
-    .send(pkg)
-    .expect(201, done);
+    mm(config, 'syncModel', 'all');
+    utils.sync('pedding', done);
   });
 
   describe('GET /-/short', function () {
@@ -40,7 +37,7 @@ describe('controllers/registry/package/list_shorts.test.js', function () {
         should.not.exist(err);
         res.body.should.be.an.Array;
         res.body.length.should.above(0);
-        res.body.indexOf('testmodule-list_shorts').should.above(-1);
+        res.body.indexOf('pedding').should.above(-1);
         done();
       });
     });

test/controllers/registry/package/list_since.test.js

@@ -24,7 +24,7 @@ describe('controllers/registry/package/list_since.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-list_since', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-list_since', '0.0.1', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
@@ -41,7 +41,7 @@ describe('controllers/registry/package/list_since.test.js', function () {
         res.body.should.be.an.Object;
         res.body._updated.should.be.a.Number;
         Object.keys(res.body).length.should.above(1);
-        res.body['testmodule-list_since'].should.equal(true);
+        res.body['@cnpmtest/testmodule-list_since'].should.equal(true);
         done();
       });
     });
@@ -54,7 +54,7 @@ describe('controllers/registry/package/list_since.test.js', function () {
         res.body.should.be.an.Object;
         res.body._updated.should.be.a.Number;
         Object.keys(res.body).length.should.above(1);
-        res.body['testmodule-list_since'].should.equal(true);
+        res.body['@cnpmtest/testmodule-list_since'].should.equal(true);
         done();
       });
     });

test/controllers/registry/package/remove.test.js

@@ -27,35 +27,27 @@ describe('controllers/registry/package/remove.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-remove-1', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-remove-1', '1.0.0', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-remove-1', '1.0.0', utils.admin);
-      request(app.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    .expect(201, done);
   });
 
   it('should delete 401 when no auth', function (done) {
     request(app)
-    .del('/testmodule-remove-1/-rev/1')
+    .del('/@cnpmtest/testmodule-remove-1/-rev/1')
     .expect({
       error: 'unauthorized',
-      reason: 'Login first.'
+      reason: 'Login first'
     })
     .expect(401, done);
   });
 
   it('should 404 when package not exists', function (done) {
     request(app)
-    .del('/testmodule-remove-1-not-exists/-rev/1')
+    .del('/@cnpmtest/testmodule-remove-1-not-exists/-rev/1')
     .set('authorization', utils.adminAuth)
     .expect({
       error: 'not_found',
@@ -67,7 +59,7 @@ describe('controllers/registry/package/remove.test.js', function () {
   it('should delete 403 when user is not admin on config.enablePrivate = true', function (done) {
     mm(config, 'enablePrivate', true);
     request(app)
-    .del('/testmodule-remove-1/-rev/1')
+    .del('/@cnpmtest/testmodule-remove-1/-rev/1')
     .set('authorization', utils.otherUserAuth)
     .expect({
       error: 'no_perms',
@@ -77,7 +69,6 @@ describe('controllers/registry/package/remove.test.js', function () {
   });
 
   it('should 400 when scope not exists', function (done) {
-    mm(config, 'enablePrivate', false);
     request(app)
     .del('/@cnpm-not-exists/testmodule-remove-1/-rev/1')
     .set('authorization', utils.otherUserAuth)
@@ -89,7 +80,6 @@ describe('controllers/registry/package/remove.test.js', function () {
   });
 
   it('should 403 when delete non scoped package', function (done) {
-    mm(config, 'enablePrivate', false);
     request(app)
     .del('/testmodule-remove-1/-rev/1')
     .set('authorization', utils.otherUserAuth)
@@ -100,20 +90,7 @@ describe('controllers/registry/package/remove.test.js', function () {
     .expect(403, done);
   });
 
-  it('should remove all versions ok', function (done) {
-    request(app)
-    .del('/testmodule-remove-1/-rev/1')
-    .set('authorization', utils.adminAuth)
-    .expect(200, function (err) {
-      should.not.exist(err);
-      request(app)
-      .get('/testmodule-remove-1')
-      .expect(404, done);
-    });
-  });
-
   it('should 403 when user not maintainer', function (done) {
-    mm(config, 'enablePrivate', false);
     request(app)
     .del('/@cnpmtest/testmodule-remove-1/-rev/1')
     .set('authorization', utils.otherUserAuth)
@@ -124,7 +101,7 @@ describe('controllers/registry/package/remove.test.js', function () {
     .expect(403, done);
   });
 
-  it('should remove scoped package all versions ok', function (done) {
+  it('should remove all versions ok', function (done) {
     request(app)
     .del('/@cnpmtest/testmodule-remove-1/-rev/1')
     .set('authorization', utils.adminAuth)

test/controllers/registry/package/remove_version.test.js

@@ -26,9 +26,8 @@ describe('controllers/registry/package/remove_version.test.js', function () {
   afterEach(mm.restore);
 
   var lastRev;
-  var lastRevScoped;
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-remove_version-1', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-remove_version-1', '0.0.1', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
@@ -36,22 +35,13 @@ describe('controllers/registry/package/remove_version.test.js', function () {
     .expect(201, function (err, res) {
       should.not.exist(err);
       lastRev = res.body.rev;
-      var pkg = utils.getPackage('@cnpmtest/testmodule-remove_version-1', '0.0.1', utils.admin);
-      request(app.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, function (err, res) {
-        should.not.exist(err);
-        lastRevScoped = res.body.rev;
-        done();
-      });
+      done();
     });
   });
 
   it('should 404 when version format error', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule_remove_version123.tgz/-rev/112312312321')
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule_remove_version123.tgz/-rev/112312312321')
     .set('authorization', utils.adminAuth)
     .expect({
       error: 'not_found',
@@ -62,7 +52,7 @@ describe('controllers/registry/package/remove_version.test.js', function () {
 
   it('should 404 when rev format error', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-1.0.1.tgz/-rev/abc')
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-1.0.1.tgz/-rev/abc')
     .set('authorization', utils.adminAuth)
     .expect({
       error: 'not_found',
@@ -73,7 +63,7 @@ describe('controllers/registry/package/remove_version.test.js', function () {
 
   it('should 404 when version not exists', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-1.0.1.tgz/-rev/112312312321')
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-1.0.1.tgz/-rev/112312312321')
     .set('authorization', utils.adminAuth)
     .expect({
       error: 'not_found',
@@ -84,47 +74,32 @@ describe('controllers/registry/package/remove_version.test.js', function () {
 
   it('should 401 when no auth', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
     .expect(401, done);
   });
 
   it('should 403 when auth error', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
     .set('authorization', utils.otherUserAuth)
     .expect(403, done);
   });
 
   it('should 200 when delete success', function (done) {
     request(app)
-    .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
-    .set('authorization', utils.adminAuth)
-    .expect(200, done);
-  });
-
-  it('should 200 when delete scoped package success', function (done) {
-    request(app)
-    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.1.tgz/-rev/123123123')
+    .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.1.tgz/-rev/' + lastRev)
     .set('authorization', utils.adminAuth)
     .expect(200, done);
   });
 
   describe('mock error', function () {
     before(function (done) {
-      var pkg = utils.getPackage('testmodule-remove_version-1', '0.0.2', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/testmodule-remove_version-1', '0.0.2', utils.admin);
       request(app.listen())
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
       .send(pkg)
-      .expect(201, function (err) {
-        should.not.exist(err);
-        var pkg = utils.getPackage('@cnpmtest/testmodule-remove_version-1', '0.0.2', utils.admin);
-        request(app.listen())
-        .put('/' + pkg.name)
-        .set('authorization', utils.adminAuth)
-        .send(pkg)
-        .expect(201, done);
-      });
+      .expect(201, done);
     });
 
     it('should auto add cdn key', function (done) {
@@ -136,17 +111,7 @@ describe('controllers/registry/package/remove_version.test.js', function () {
       });
 
       request(app)
-      .del('/testmodule-remove_version-1/download/testmodule-remove_version-1-0.0.2.tgz/-rev/' + lastRev)
-      .set('authorization', utils.adminAuth)
-      .expect(200, done);
-    });
-
-    it('should mock nfs.remove error', function (done) {
-      mm(nfs, 'remove', function* (key) {
-        throw new Error('mock remove ' + key + ' error');
-      });
-      request(app)
-      .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.2.tgz/-rev/1')
+      .del('/@cnpmtest/testmodule-remove_version-1/download/@cnpmtest/testmodule-remove_version-1-0.0.2.tgz/-rev/' + lastRev)
       .set('authorization', utils.adminAuth)
       .expect(200, done);
     });

test/controllers/registry/package/save.test.js

@@ -19,13 +19,20 @@ var request = require('supertest');
 var mm = require('mm');
 var packageService = require('../../../../services/package');
 var app = require('../../../../servers/registry');
+var config = require('../../../../config');
 var utils = require('../../../utils');
 
 describe('controllers/registry/package/save.test.js', function () {
   afterEach(mm.restore);
 
   describe('no @scoped package', function () {
+    beforeEach(function () {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'privatePackages', ['testmodule-new-1', 'testmodule-new-2', 'testmodule-no-latest']);
+    });
+
     before(function (done) {
+      mm(config, 'privatePackages', ['testmodule-new-1', 'testmodule-new-2']);
       var pkg = utils.getPackage('testmodule-new-1', '0.0.1', utils.admin);
       pkg.versions['0.0.1'].dependencies = {
         'bytetest-1': '~0.0.1',

test/controllers/registry/package/show.test.js

@@ -24,42 +24,34 @@ describe('controllers/registry/package/show.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-show', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-show', '0.0.1', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-show', '0.0.1', utils.admin);
-      request(app.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    .expect(201, done);
   });
 
   it('should return one version', function (done) {
     request(app.listen())
-    .get('/testmodule-show/0.0.1')
+    .get('/@cnpmtest/testmodule-show/0.0.1')
     .expect(200, function (err, res) {
       should.not.exist(err);
       var data = res.body;
-      data.name.should.equal('testmodule-show');
+      data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('0.0.1');
-      data.dist.tarball.should.containEql('/testmodule-show/download/testmodule-show-0.0.1.tgz');
+      data.dist.tarball.should.containEql('/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-0.0.1.tgz');
       done();
     });
   });
 
   it('should return latest tag', function (done) {
     request(app.listen())
-    .get('/testmodule-show/latest')
+    .get('/@cnpmtest/testmodule-show/latest')
     .expect(200, function (err, res) {
       should.not.exist(err);
       var data = res.body;
-      data.name.should.equal('testmodule-show');
+      data.name.should.equal('@cnpmtest/testmodule-show');
       data.version.should.equal('0.0.1');
       done();
     });
@@ -67,7 +59,7 @@ describe('controllers/registry/package/show.test.js', function () {
 
   it('should 404 when package not exist', function (done) {
     request(app.listen())
-    .get('/testmodule-show-not-exists/latest')
+    .get('/@cnpmtest/testmodule-show-not-exists/latest')
     .expect(404, done);
   });
 
@@ -89,21 +81,9 @@ describe('controllers/registry/package/show.test.js', function () {
     .expect(404, done);
   });
 
-  it('should 200 when source npm exists', function (done) {
-    request(app.listen())
-    .get('/baidu/latest')
-    .expect(200, done);
-  });
-
   describe('show sync package', function () {
-    var SyncModuleWorker = require('../../../../controllers/sync_module_worker');
     before(function (done) {
-      var worker = new SyncModuleWorker({
-        name: 'baidu',
-        noDep: true,
-      });
-      worker.start();
-      worker.on('end', done);
+      utils.sync('baidu', done);
     });
 
     it('should 200 when source npm exists', function (done) {

test/controllers/registry/package/tag.test.js

@@ -14,7 +14,6 @@
  * Module dependencies.
  */
 
-var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
 var app = require('../../../../servers/registry');
@@ -24,32 +23,15 @@ describe('controllers/registry/package/tag.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-tag-1', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-tag-1', '1.0.0', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-tag-1', '1.0.0', utils.admin);
-      request(app.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
-  });
-
-  it('should create new tag ok', function (done) {
-    request(app)
-    .put('/testmodule-tag-1/newtag')
-    .set('content-type', 'application/json')
-    .set('authorization', utils.adminAuth)
-    .send('"0.0.1"')
     .expect(201, done);
   });
 
-  it('should create new tag on scoped package', function (done) {
+  it('should create new tag ok', function (done) {
     request(app)
     .put('/@cnpmtest/testmodule-tag-1/newtag')
     .set('content-type', 'application/json')
@@ -60,16 +42,16 @@ describe('controllers/registry/package/tag.test.js', function () {
 
   it('should override exist tag ok', function (done) {
     request(app)
-    .put('/testmodule-tag-1/newtag')
+    .put('/@cnpmtest/testmodule-tag-1/newtag')
     .set('content-type', 'application/json')
     .set('authorization', utils.adminAuth)
-    .send('"0.0.1"')
+    .send('"1.0.0"')
     .expect(201, done);
   });
 
   it('should 400 when version missing', function (done) {
     request(app)
-    .put('/testmodule-tag-1/newtag')
+    .put('/@cnpmtest/testmodule-tag-1/newtag')
     .set('content-type', 'application/json')
     .set('authorization', utils.adminAuth)
     .send('""')
@@ -82,33 +64,33 @@ describe('controllers/registry/package/tag.test.js', function () {
 
   it('should tag invalid version 403', function (done) {
     request(app)
-    .put('/testmodule-tag-1/newtag')
+    .put('/@cnpmtest/testmodule-tag-1/newtag')
     .set('content-type', 'application/json')
     .set('authorization', utils.adminAuth)
     .send('"hello"')
     .expect(403)
     .expect({
       error: 'forbidden',
-      reason: 'setting tag newtag to invalid version: hello: testmodule-tag-1/newtag'
+      reason: 'setting tag newtag to invalid version: hello: @cnpmtest/testmodule-tag-1/newtag'
     }, done);
   });
 
   it('should tag not eixst version 403', function (done) {
     request(app)
-    .put('/testmodule-tag-1/newtag')
+    .put('/@cnpmtest/testmodule-tag-1/newtag')
     .set('content-type', 'application/json')
     .set('authorization', utils.adminAuth)
     .send('"5.0.0"')
     .expect(403)
     .expect({
       error: 'forbidden',
-      reason: 'setting tag newtag to unknown version: 5.0.0: testmodule-tag-1/newtag'
+      reason: 'setting tag newtag to unknown version: 5.0.0: @cnpmtest/testmodule-tag-1/newtag'
     }, done);
   });
 
   describe('update tag not maintainer', function () {
     before(function (done) {
-      var pkg = utils.getPackage('update-tag-not-maintainer', '1.0.0');
+      var pkg = utils.getPackage('@cnpmtest/update-tag-not-maintainer', '1.0.0');
       request(app)
       .put('/' + pkg.name)
       .set('content-type', 'application/json')
@@ -119,14 +101,14 @@ describe('controllers/registry/package/tag.test.js', function () {
 
     it('should not maintainer update tag return no permission 403', function (done) {
       request(app)
-      .put('/update-tag-not-maintainer/newtag')
+      .put('/@cnpmtest/update-tag-not-maintainer/newtag')
       .set('content-type', 'application/json')
       .set('authorization', utils.otherUserAuth)
       .send('"1.0.0"')
       .expect(403)
       .expect({
         error: 'forbidden user',
-        reason: 'cnpmjstest101 not authorized to modify update-tag-not-maintainer'
+        reason: 'cnpmjstest101 not authorized to modify @cnpmtest/update-tag-not-maintainer'
       }, done);
     });
   });

test/controllers/registry/package/update.test.js

@@ -25,43 +25,27 @@ var config = require('../../../../config');
 describe('controllers/registry/package/update.test.js', function () {
   afterEach(mm.restore);
 
-  beforeEach(function () {
-    mm(config, 'enablePrivate', false);
-    mm(config, 'forcePublishWithScope', false);
-  });
-
   before(function (done) {
-    mm(config, 'enablePrivate', false);
-    mm(config, 'forcePublishWithScope', false);
-
-    var pkg = utils.getPackage('testmodule-update-1', '0.0.1', utils.otherUser);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-update-1', '1.0.0', utils.otherUser);
     request(app.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.otherUserAuth)
     .send(pkg)
     .expect(201, function (err) {
       should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-update-1', '1.0.0', utils.otherUser);
+      var pkg = utils.getPackage('@cnpmtest/testmodule-update-1', '2.0.0', utils.otherUser);
       request(app.listen())
       .put('/' + pkg.name)
       .set('authorization', utils.otherUserAuth)
       .send(pkg)
-      .expect(201, function (err) {
-        should.not.exist(err);
-        var pkg = utils.getPackage('@cnpmtest/testmodule-update-1', '2.0.0', utils.otherUser);
-        request(app.listen())
-        .put('/' + pkg.name)
-        .set('authorization', utils.otherUserAuth)
-        .send(pkg)
-        .expect(201, done);
-      });
+      .expect(201, done);
     });
   });
 
   it('should 404 when update body wrong', function (done) {
     request(app)
-    .put('/testmodule-update-1/-rev/1')
-    .set('authorization', utils.adminAuth)
+    .put('/@cnpmtest/testmodule-update-1/-rev/1')
+    .set('authorization', utils.otherUserAuth)
     .send({
       foo: 'bar'
     })
@@ -74,11 +58,8 @@ describe('controllers/registry/package/update.test.js', function () {
 
   describe('PUT /:name/-rev/:rev updatePrivateModuleMaintainers()', function () {
     before(function (done) {
-      mm(config, 'enablePrivate', false);
-      mm(config, 'forcePublishWithScope', false);
-
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest101',
@@ -88,7 +69,7 @@ describe('controllers/registry/package/update.test.js', function () {
       .set('authorization', utils.otherUserAuth)
       .expect({
         ok: true,
-        id:"testmodule-update-1",
+        id:"@cnpmtest/testmodule-update-1",
         rev: "1"
       }, done);
     });
@@ -153,7 +134,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should add again new maintainers', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest101',
@@ -170,7 +151,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should add new maintainers by admin', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest101',
@@ -187,7 +168,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should rm maintainers', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest101',
@@ -201,7 +182,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should rm again maintainers', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest101',
@@ -211,7 +192,7 @@ describe('controllers/registry/package/update.test.js', function () {
       .set('authorization', utils.otherUserAuth)
       .expect(201)
       .expect({
-        id: 'testmodule-update-1',
+        id: '@cnpmtest/testmodule-update-1',
         rev: '1',
         ok: true
       }, done);
@@ -219,7 +200,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should rm all maintainers forbidden 403', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: []
       })
@@ -231,7 +212,7 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should 403 when not maintainer update', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
@@ -242,18 +223,12 @@ describe('controllers/registry/package/update.test.js', function () {
       .expect(403)
       .expect({
         error: 'forbidden user',
-        reason: 'cnpmjstest102 not authorized to modify testmodule-update-1'
+        reason: 'cnpmjstest102 not authorized to modify @cnpmtest/testmodule-update-1'
       }, done);
     });
 
     describe('forcePublishWithScope = true', function () {
-      beforeEach(function () {
-        mm(config, 'forcePublishWithScope', true);
-      });
-
       before(function (done) {
-        mm(config, 'forcePublishWithScope', true);
-        mm(config, 'enablePrivate', false);
         var pkg = utils.getPackage('@cnpm/testmodule-update-1', '0.0.1', utils.otherUser);
         request(app)
         .put('/' + pkg.name)
@@ -375,21 +350,21 @@ describe('controllers/registry/package/update.test.js', function () {
   describe('PUT /:name/-rev/:rev updateVersions()', function () {
     it('should update 401 when no auth', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/123')
+      .put('/@cnpmtest/testmodule-update-1/-rev/123')
       .expect(401, done);
     });
 
     it('should update 403 when auth error', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/123')
+      .put('/@cnpmtest/testmodule-update-1/-rev/123')
       .set('authorization', utils.thirdUserAuth)
       .expect(403, done);
     });
 
     it('should remove nothing removed ok', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
-      .set('authorization', utils.adminAuth)
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
+      .set('authorization', utils.otherUserAuth)
       .send({
         versions: {
           '0.0.1': {},
@@ -402,7 +377,7 @@ describe('controllers/registry/package/update.test.js', function () {
     it('should remove lastest tag with scoped', function (done) {
       request(app)
       .put('/@cnpmtest/testmodule-update-1/-rev/1')
-      .set('authorization', utils.adminAuth)
+      .set('authorization', utils.otherUserAuth)
       .send({
         versions: {
           '1.0.0': {},
@@ -413,7 +388,7 @@ describe('controllers/registry/package/update.test.js', function () {
         // again should work
         request(app)
         .put('/@cnpmtest/testmodule-update-1/-rev/1')
-        .set('authorization', utils.adminAuth)
+        .set('authorization', utils.otherUserAuth)
         .send({
           versions: {
             '1.0.0': {},
@@ -426,7 +401,7 @@ describe('controllers/registry/package/update.test.js', function () {
     it('should 404 when package not exists', function (done) {
       request(app)
       .put('/@cnpmtest/testmodule-update-1-not-exists/-rev/1')
-      .set('authorization', utils.adminAuth)
+      .set('authorization', utils.otherUserAuth)
       .send({
         versions: {
           '0.0.1': {},
@@ -438,8 +413,8 @@ describe('controllers/registry/package/update.test.js', function () {
 
     it('should remove all version ok', function (done) {
       request(app)
-      .put('/testmodule-update-1/-rev/1')
-      .set('authorization', utils.adminAuth)
+      .put('/@cnpmtest/testmodule-update-1/-rev/1')
+      .set('authorization', utils.otherUserAuth)
       .send({
         versions: {}
       })

test/controllers/registry/user/show.test.js

@@ -20,18 +20,13 @@ var mm = require('mm');
 var app = require('../../../../servers/registry');
 var config = require('../../../../config');
 var userService = require('../../../../services/user');
-var SyncModuleWorker = require('../../../../controllers/sync_module_worker');
+var utils = require('../../../utils');
 
 describe('controllers/registry/user/show.test.js, GET /-/user/org.couchdb.user:name', function () {
   afterEach(mm.restore);
 
   before(function (done) {
-    var worker = new SyncModuleWorker({
-      name: 'pedding',
-      noDep: true,
-    });
-    worker.start();
-    worker.on('end', done);
+    utils.sync('pedding', done);
   });
 
   beforeEach(function () {

test/controllers/registry/user_package.test.js

@@ -1,5 +1,5 @@
 /**!
- * cnpmjs.org - test/contributors/registry/user_package.test.js
+ * cnpmjs.org - test/controllers/registry/user_package.test.js
  *
  * Copyright(c) fengmk2 and other contributors.
  * MIT Licensed
@@ -19,24 +19,18 @@ var request = require('supertest');
 var pedding = require('pedding');
 var app = require('../../../servers/registry');
 var utils = require('../../utils');
-var SyncModuleWorker = require('../../../controllers/sync_module_worker');
 
-describe('contributors/registry/user_package.test.js', function () {
+describe('controllers/registry/user_package.test.js', function () {
   before(function (done) {
+    done = pedding(2, done);
     // sync pedding
-    var worker = new SyncModuleWorker({
-      name: 'pedding',
-      noDep: true,
-    });
-    worker.start();
-    worker.on('end', function () {
-      var pkg = utils.getPackage('test-user-package-module', '0.0.1', utils.otherAdmin2);
-      request(app)
-      .put('/' + pkg.name)
-      .set('authorization', utils.otherAdmin2Auth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    utils.sync('pedding', done);
+    var pkg = utils.getPackage('@cnpmtest/test-user-package-module', '0.0.1', utils.otherAdmin2);
+    request(app)
+    .put('/' + pkg.name)
+    .set('authorization', utils.otherAdmin2Auth)
+    .send(pkg)
+    .expect(201, done);
   });
 
   describe('listOne()', function () {
@@ -104,8 +98,6 @@ describe('contributors/registry/user_package.test.js', function () {
         should.not.exist(err);
         res.body.fengmk2.should.be.an.Array;
         res.body.fengmk2.should.containEql('pedding');
-        res.body.cnpmjstestAdmin2.should.be.an.Array;
-        res.body.cnpmjstestAdmin2.should.containEql('test-user-package-module');
         done();
       });
     });

test/controllers/sync_module_worker.test.js

@@ -28,10 +28,13 @@ describe('controllers/sync_module_worker.test.js', function () {
   afterEach(mm.restore);
 
   beforeEach(function () {
+    mm(config, 'syncModel', 'all');
     mm(config, 'sourceNpmRegistryIsCNpm', false);
+    mm(config, 'privatePackages', ['google']);
   });
 
   before(function (done) {
+    mm(config, 'privatePackages', ['google']);
     var pkg = utils.getPackage('google', '0.0.1', utils.admin);
     request(app.listen())
     .put('/' + pkg.name)
@@ -245,4 +248,17 @@ describe('controllers/sync_module_worker.test.js', function () {
       ];
     });
   });
+
+  describe('sync user', function () {
+    it('should sync fengmk2', function* () {
+      var worker = new SyncModuleWorker({
+        type: 'user',
+        name: 'fengmk2',
+        username: 'fengmk2',
+      });
+      worker.start();
+      var end = thunkify.event(worker, 'end');
+      yield end();
+    });
+  });
 });

test/controllers/web/badge.test.js

@@ -26,7 +26,7 @@ describe('controllers/web/badge.test.js', function () {
 
   describe('GET /badge/v/:name.svg', function () {
     it('should show blue version on >=1.0.0 when package exists', function (done) {
-      var pkg = utils.getPackage('badge-test-module', '1.0.1', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '1.0.1', utils.admin);
       request(registry)
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -34,14 +34,14 @@ describe('controllers/web/badge.test.js', function () {
       .end(function (err) {
         should.not.exists(err);
         request(app)
-        .get('/badge/v/badge-test-module.svg?style=flat-square')
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square')
         .expect('Location', 'https://img.shields.io/badge/cnpm-1.0.1-blue.svg?style=flat-square')
         .expect(302, done);
       });
     });
 
     it('should show tag', function (done) {
-      var pkg = utils.getPackage('badge-test-module', '2.0.1', utils.admin, 'v2');
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '2.0.1', utils.admin, 'v2');
       request(registry)
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -49,14 +49,14 @@ describe('controllers/web/badge.test.js', function () {
       .end(function (err) {
         should.not.exists(err);
         request(app)
-        .get('/badge/v/badge-test-module.svg?style=flat-square&tag=v2')
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square&tag=v2')
         .expect('Location', 'https://img.shields.io/badge/cnpm-2.0.1-blue.svg?style=flat-square')
         .expect(302, done);
       });
     });
 
     it('should support 1.0.0-beta1', function (done) {
-      var pkg = utils.getPackage('badge-test-module', '1.0.0-beta1', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '1.0.0-beta1', utils.admin);
       request(registry)
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -64,14 +64,14 @@ describe('controllers/web/badge.test.js', function () {
       .end(function (err) {
         should.not.exists(err);
         request(app)
-        .get('/badge/v/badge-test-module.svg?style=flat-square')
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square')
         .expect('Location', 'https://img.shields.io/badge/cnpm-1.0.0--beta1-blue.svg?style=flat-square')
         .expect(302, done);
       });
     });
 
     it('should show green version on <1.0.0 & >=0.1.0 when package exists', function (done) {
-      var pkg = utils.getPackage('badge-test-module', '0.1.0', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '0.1.0', utils.admin);
       request(registry)
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -79,14 +79,14 @@ describe('controllers/web/badge.test.js', function () {
       .end(function (err) {
         should.not.exists(err);
         request(app)
-        .get('/badge/v/badge-test-module.svg?style=flat-square')
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square')
         .expect('Location', 'https://img.shields.io/badge/cnpm-0.1.0-green.svg?style=flat-square')
         .expect(302, done);
       });
     });
 
     it('should show green version on <0.1.0 & >=0.0.0 when package exists', function (done) {
-      var pkg = utils.getPackage('badge-test-module', '0.0.0', utils.admin);
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '0.0.0', utils.admin);
       request(registry)
       .put('/' + pkg.name)
       .set('authorization', utils.adminAuth)
@@ -94,7 +94,7 @@ describe('controllers/web/badge.test.js', function () {
       .end(function (err) {
         should.not.exists(err);
         request(app)
-        .get('/badge/v/badge-test-module.svg?style=flat-square')
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square')
         .expect('Location', 'https://img.shields.io/badge/cnpm-0.0.0-red.svg?style=flat-square')
         .expect(302, done);
       });
@@ -102,7 +102,7 @@ describe('controllers/web/badge.test.js', function () {
 
     it('should show invalid when package not exists', function (done) {
       request(app)
-      .get('/badge/v/badge-test-module-not-exists.svg?style=flat')
+      .get('/badge/v/@cnpmtest/badge-test-module-not-exists.svg?style=flat')
       .expect('Location', 'https://img.shields.io/badge/cnpm-invalid-lightgrey.svg?style=flat')
       .expect(302, done);
     });

test/controllers/web/dist.test.js

@@ -1,215 +0,0 @@
-/**!
- * cnpmjs.org - test/controllers/web/dist.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.cnpmjs.org)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var should = require('should');
-var request = require('supertest');
-var mm = require('mm');
-var fs = require('fs');
-var nfs = require('../../../common/nfs');
-var app = require('../../../servers/web');
-var distService = require('../../../services/dist');
-
-describe('controllers/web/dist.test.js', function () {
-  before(function (done) {
-    app = app.listen(0, done);
-  });
-
-  afterEach(mm.restore);
-
-  describe('GET /dist/*', function () {
-    it('should GET /dist redirect to /dist/', function (done) {
-      request(app)
-      .get('/dist')
-      .expect(302)
-      .expect('Location', '/dist/', done);
-    });
-
-    it('should GET /dist/ show file list', function (done) {
-      request(app)
-      .get('/dist/')
-      .expect('Content-Type', 'text/html; charset=utf-8')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
-        done();
-      });
-    });
-
-    it('should mock return files and dirs', function (done) {
-      mm(distService, 'listdir', function* () {
-        return [
-          {name: 'ooxx/', date: '02-May-2014 00:54'},
-          {name: 'foo.txt', size: 1024, date: '02-May-2014 00:54'},
-        ];
-      });
-      request(app)
-      .get('/dist/v1.0.0/')
-      .expect('Content-Type', 'text/html; charset=utf-8')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/v1.0.0/</title>');
-        res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/v1.0.0/">http://nodejs.org/dist/v1.0.0/</a></h1>');
-        res.text.should.containEql('<a href="ooxx/">ooxx/</a>                                             02-May-2014 00:54                   -\n');
-        res.text.should.containEql('<a href="foo.txt">foo.txt</a>                                           02-May-2014 00:54                1024\n');
-        done();
-      });
-    });
-
-    it('should list files and dirs', function (done) {
-      mm(distService, 'listdir', function* () {
-        return [
-          {name: 'npm/', date: '02-May-2014 00:54'},
-          {name: 'npm-versions.txt', size: 1676, date: '02-May-2014 00:54'},
-        ];
-      });
-      request(app)
-      .get('/dist/')
-      .expect('Content-Type', 'text/html; charset=utf-8')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
-        res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/">http://nodejs.org/dist/</a></h1>');
-        res.text.should.containEql('<a href="npm/">npm/</a>                                              02-May-2014 00:54                   -\n');
-        res.text.should.containEql('<a href="npm-versions.txt">npm-versions.txt</a>                                  02-May-2014 00:54                1676\n');
-        done();
-      });
-    });
-  });
-
-  describe('GET /dist/ files', function () {
-    it('should pipe txt', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: 'foo.txt', size: 1024, date: '02-May-2014 00:54',
-          url: 'http://mock.com/dist/v0.10.28/SHASUMS.txt'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/SHASUMS.txt'), '6eff580cc8460741155d42ef1ef537961194443f');
-
-      request(app)
-      .get('/dist/v0.10.28/SHASUMS.txt')
-      .expect('Content-Type', 'text/plain; charset=utf-8')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        should.not.exist(res.headers['Content-Disposition']);
-        done();
-      });
-    });
-
-    it('should pipe html', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: 'foo.html', size: 1024, date: '02-May-2014 00:54',
-          url: 'http://mock.com/dist/v0.10.28/foo.html'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/foo.html'), '<p>hi</p>');
-
-      request(app)
-      .get('/dist/v0.10.28/foo.html')
-      .expect('Content-Type', 'text/html; charset=utf-8')
-      .expect('<p>hi</p>')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        should.not.exist(res.headers['Content-Disposition']);
-        done();
-      });
-    });
-
-    it('should pipe json', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: 'foo.json', date: '02-May-2014 00:54',
-          url: 'http://mock.com/dist/v0.10.28/foo.json'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/foo.json'), '{}');
-
-      request(app)
-      .get('/dist/v0.10.28/foo.json')
-      .expect('Content-Type', 'application/json; charset=utf-8')
-      .expect('{}')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        should.not.exist(res.headers['Content-Disposition']);
-        done();
-      });
-    });
-
-    it('should GET /dist/npm-versions.tgz redirect to nfs url', function (done) {
-      mm(distService, 'getfile', function* (fullname) {
-        fullname.should.equal('/npm-versions.tgz');
-        return {
-          name: 'npm-versions.txt', size: 1024, date: '02-May-2014 00:54',
-          url: 'http://mock.com/dist/npm-versions.tgz'
-        };
-      });
-
-      request(app)
-      .get('/dist/npm-versions.tgz')
-      .expect(302)
-      .expect('Location', 'http://mock.com/dist/npm-versions.tgz', done);
-    });
-
-    it('should download nfs txt file and send it', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: 'foo.txt',
-          size: 1264,
-          date: '02-May-2014 00:54',
-          url: '/dist/v0.10.28/SHASUMS.txt'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/SHASUMS.txt'), '6eff580cc8460741155d42ef1ef537961194443f');
-      request(app)
-      .get('/dist/v0.10.28/SHASUMS.txt')
-      .expect(200)
-      .expect(/6eff580cc8460741155d42ef1ef537961194443f/, done);
-    });
-
-    it('should download nfs tgz file and send it', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: 'foo.tgz',
-          size: 1264,
-          date: '02-May-2014 00:54',
-          url: '/dist/v0.10.28/foo.tgz'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/foo.tgz'), '6eff580cc8460741155d42ef1ef537961194443f');
-      request(app)
-      .get('/dist/v0.10.28/foo.tgz')
-      .expect('Content-Disposition', 'attachment; filename="foo.tgz"')
-      .expect(200, done);
-    });
-
-    it.skip('should download nfs no-ascii attachment file name', function (done) {
-      mm(distService, 'getfile', function* () {
-        return {
-          name: '中文名.tgz',
-          size: 1264,
-          date: '02-May-2014 00:54',
-          url: '/dist/v0.10.28/foo.tgz'
-        };
-      });
-      fs.writeFileSync(nfs._getpath('/dist/v0.10.28/foo.tgz'), '6eff580cc8460741155d42ef1ef537961194443f');
-      request(app)
-      .get('/dist/v0.10.28/foo.tgz')
-      .expect('Content-Disposition', 'attachment; filename="%E4%B8%AD%E6%96%87%E5%90%8D.tgz"')
-      .expect(200, done);
-    });
-  });
-});

test/controllers/web/package/list_privates.test.js

@@ -25,6 +25,7 @@ describe('controllers/web/package/list_privates.test.js', function () {
   afterEach(mm.restore);
 
   before(function (done) {
+    mm(config, 'privatePackages', ['testmodule-web-list_privates-no-scoped', 'hsf-haha']);
     var pkg = utils.getPackage('@cnpm/testmodule-web-list_privates', '0.0.1', utils.admin);
     request(registry.listen())
     .put('/' + pkg.name)

test/controllers/web/package/search.test.js

@@ -6,6 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -14,7 +15,6 @@
  * Module dependencies.
  */
 
-var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
 var app = require('../../../../servers/web');
@@ -23,41 +23,22 @@ var utils = require('../../../utils');
 
 describe('controllers/web/package/search.test.js', function () {
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-web-search', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-web-search', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
       bytetest: '~0.0.1',
-      mocha: '~1.0.0'
+      mocha: '~1.0.0',
+      'testmodule-web-show': '0.0.1'
     };
     request(registry.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-web-search', '0.0.1', utils.admin);
-      pkg.versions['0.0.1'].dependencies = {
-        bytetest: '~0.0.1',
-        mocha: '~1.0.0',
-        'testmodule-web-show': '0.0.1'
-      };
-      request(registry.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    .expect(201, done);
   });
 
   afterEach(mm.restore);
 
   describe('GET /browse/keyword/:word', function () {
-    it('should list by keyword ok', function (done) {
-      request(app)
-      .get('/browse/keyword/testmodule-web-search')
-      .expect(200)
-      .expect(/Packages match/, done);
-    });
-
     it('should list by keyword ok', function (done) {
       request(app)
       .get('/browse/keyword/@cnpmtest/testmodule-web-search')
@@ -67,11 +48,25 @@ describe('controllers/web/package/search.test.js', function () {
 
     it('should list by keyword with json ok', function (done) {
       request(app)
-      .get('/browse/keyword/testmodule-web-search?type=json')
+      .get('/browse/keyword/@cnpmtest/testmodule-web-search?type=json')
       .expect(200)
+      .expect({
+        keyword: '@cnpmtest/testmodule-web-search',
+        match: { name: '@cnpmtest/testmodule-web-search', description: '' },
+        packages: [ { name: '@cnpmtest/testmodule-web-search', description: '' } ],
+        keywords: []
+      })
       .expect('content-type', 'application/json; charset=utf-8', done);
     });
 
+    it('should search with jsonp work', function (done) {
+      request(app)
+      .get('/browse/keyword/@cnpmtest/testmodule-web-search?type=json&callback=foo')
+      .expect(200)
+      .expect('/**/ typeof foo === \'function\' && foo({"keyword":"@cnpmtest/testmodule-web-search","match":{"name":"@cnpmtest/testmodule-web-search","description":""},"packages":[{"name":"@cnpmtest/testmodule-web-search","description":""}],"keywords":[]});')
+      .expect('content-type', 'application/javascript; charset=utf-8', done);
+    });
+
     it('should list no match ok', function (done) {
       request(app)
       .get('/browse/keyword/notexistpackage')

test/controllers/web/package/search_range.test.js

@@ -23,29 +23,17 @@ var utils = require('../../../utils');
 
 describe('controllers/web/package/search_range.test.js', function () {
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-web-search_range', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-web-search_range', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
       bytetest: '~0.0.1',
-      mocha: '~1.0.0'
+      mocha: '~1.0.0',
+      'testmodule-web-show': '0.0.1'
     };
     request(registry.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-web-search_range', '0.0.1', utils.admin);
-      pkg.versions['0.0.1'].dependencies = {
-        bytetest: '~0.0.1',
-        mocha: '~1.0.0',
-        'testmodule-web-show': '0.0.1'
-      };
-      request(registry.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    .expect(201, done);
   });
 
   afterEach(mm.restore);

test/controllers/web/package/show.test.js

@@ -17,36 +17,24 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var config = require('../../../../config');
 var app = require('../../../../servers/web');
 var registry = require('../../../../servers/registry');
-var SyncModuleWorker = require('../../../../controllers/sync_module_worker');
 var utils = require('../../../utils');
 
 describe('controllers/web/package/show.test.js', function () {
   before(function (done) {
-    var pkg = utils.getPackage('testmodule-web-show', '0.0.1', utils.admin);
+    var pkg = utils.getPackage('@cnpmtest/testmodule-web-show', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
       bytetest: '~0.0.1',
-      mocha: '~1.0.0'
+      mocha: '~1.0.0',
+      'testmodule-web-show': '0.0.1'
     };
     request(registry.listen())
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
-    .expect(201, function (err) {
-      should.not.exist(err);
-      var pkg = utils.getPackage('@cnpmtest/testmodule-web-show', '0.0.1', utils.admin);
-      pkg.versions['0.0.1'].dependencies = {
-        bytetest: '~0.0.1',
-        mocha: '~1.0.0',
-        'testmodule-web-show': '0.0.1'
-      };
-      request(registry.listen())
-      .put('/' + pkg.name)
-      .set('authorization', utils.adminAuth)
-      .send(pkg)
-      .expect(201, done);
-    });
+    .expect(201, done);
   });
 
   afterEach(mm.restore);
@@ -54,7 +42,7 @@ describe('controllers/web/package/show.test.js', function () {
   describe('GET /package/:name', function () {
     it('should get 200', function (done) {
       request(app.listen())
-      .get('/package/testmodule-web-show')
+      .get('/package/@cnpmtest/testmodule-web-show')
       .expect(200)
       .expect('content-type', 'text/html; charset=utf-8')
       .expect(/testmodule-web-show/)
@@ -86,7 +74,7 @@ describe('controllers/web/package/show.test.js', function () {
 
     it('should get 404', function (done) {
       request(app)
-      .get('/package/not-exist-module')
+      .get('/package/@cnpmtest/not-exist-module')
       .expect(404, done);
     });
   });
@@ -94,7 +82,7 @@ describe('controllers/web/package/show.test.js', function () {
   describe('GET /package/:name/:version', function () {
     it('should 200 when get by version', function (done) {
       request(app)
-      .get('/package/testmodule-web-show/0.0.1')
+      .get('/package/@cnpmtest/testmodule-web-show/0.0.1')
       .expect(200)
       .expect(/testmodule-web-show/)
       .expect(/Maintainers/)
@@ -104,7 +92,7 @@ describe('controllers/web/package/show.test.js', function () {
 
     it('should 200 when get by tag', function (done) {
       request(app)
-      .get('/package/testmodule-web-show/latest')
+      .get('/package/@cnpmtest/testmodule-web-show/latest')
       .expect(200)
       .expect(/testmodule-web-show/)
       .expect(/Maintainers/)
@@ -114,34 +102,24 @@ describe('controllers/web/package/show.test.js', function () {
 
     it('should 404 when get by version not exist', function (done) {
       request(app)
-      .get('/package/testmodule-web-show/1.1.2')
+      .get('/package/@cnpmtest/testmodule-web-show/1.1.2')
       .expect(404, done);
     });
 
     it('should 404 when get by tag', function (done) {
       request(app)
-      .get('/package/testmodule-web-show/notexisttag')
+      .get('/package/@cnpmtest/testmodule-web-show/notexisttag')
       .expect(404, done);
     });
   });
 
   describe('unpublished package', function () {
     before(function (done) {
-      var worker = new SyncModuleWorker({
-        name: ['tnpm'],
-        username: 'fengmk2'
-      });
-
-      worker.start();
-      worker.on('end', function () {
-        var names = worker.successes.concat(worker.fails);
-        names.sort();
-        names.should.eql(['tnpm']);
-        done();
-      });
+      utils.sync('tnpm', done);
     });
 
     it('should display unpublished info', function (done) {
+      mm(config, 'syncModel', 'all');
       request(app)
       .get('/package/tnpm')
       .expect(200)
@@ -149,24 +127,36 @@ describe('controllers/web/package/show.test.js', function () {
     });
   });
 
-  describe('show npm package', function () {
+  describe('xss filter', function () {
     before(function (done) {
-      var worker = new SyncModuleWorker({
-        name: ['pedding'],
-        username: 'fengmk2',
-        noDep: true
-      });
+      var pkg = utils.getPackage('@cnpmtest/xss-test-ut', '0.0.1', utils.admin, null, '[xss link](javascript:alert(2)) \n\nfoo<script>alert(1)</script>/xss\'"&#');
+      request(registry.listen())
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .end(done);
+    });
 
-      worker.start();
-      worker.on('end', function () {
-        var names = worker.successes.concat(worker.fails);
-        names.sort();
-        names.should.eql(['pedding']);
+    it('should filter xss content', function (done) {
+      request(app.listen())
+      .get('/package/@cnpmtest/xss-test-ut')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.not.containEql('<script>alert(1)</script>');
+        res.text.should.not.containEql('alert(2)"');
         done();
       });
     });
+  });
+
+  describe('show npm package', function () {
+    before(function (done) {
+      mm(config, 'syncModel', 'exists');
+      utils.sync('pedding', done);
+    });
 
     it('should show pedding package info and contributors', function (done) {
+      mm(config, 'syncModel', 'exists');
       request(app)
       .get('/package/pedding')
       .expect(200)

test/controllers/web/show_sync.test.js

@@ -1,5 +1,5 @@
 /*!
- * cnpmjs.org - test/controllers/web/package/show_sync.test.js
+ * cnpmjs.org - test/controllers/web/show_sync.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -15,15 +15,15 @@
  */
 
 var request = require('supertest');
-var app = require('../../../../servers/web');
+var app = require('../../../servers/web');
 
-describe('controllers/web/package/show_sync.test.js', function () {
+describe('controllers/web/show_sync.test.js', function () {
   describe('GET /sync/:name', function () {
     it('should display ok', function (done) {
       request(app.listen())
       .get('/sync/cutter')
       .expect(200)
-      .expect(/Sync Package/)
+      .expect(/Sync package/)
       .expect(/Log/, done);
     });
   });

test/fixtures/code.md

@@ -0,0 +1,9 @@
+# test < >
+
+```html
+<body>hi</body>
+```
+
+```js
+var a = 1 > 2 ? 'a' : 'b';
+```

test/fixtures/eat-cpu-all.md

@@ -0,0 +1,357 @@
+# BOWER [![Build Status](https://secure.travis-ci.org/bower/bower.png?branch=master)](http://travis-ci.org/bower/bower)
+
+Bower is a package manager for the web. It offers a generic, unopinionated
+solution to the problem of **front-end package management**, while exposing the
+package dependency model via an API that can be consumed by a more opinionated
+build stack. There are no system wide dependencies, no dependencies are shared
+between different apps, and the dependency tree is flat.
+
+Bower runs over Git, and is package-agnostic. A packaged component can be made
+up of any type of asset, and use any type of transport (e.g., AMD, CommonJS,
+etc.).
+
+[View all packages available through Bower's registry](http://sindresorhus.com/bower-components/).
+
+
+## Installing Bower
+
+Bower depends on [Node](http://nodejs.org/) and [npm](http://npmjs.org/). It's
+installed globally using npm:
+
+```
+npm install -g bower
+```
+
+Also make sure that [git](http://git-scm.com/) is installed as some bower
+packages require it to be fetched and installed.
+
+
+## Usage
+
+Much more information is available via `bower help` once it's installed. This
+is just enough to get you started.
+
+#### Warning
+
+On `prezto` or `oh-my-zsh`, do not forget to `alias bower='noglob bower'` or `bower install jquery\#1.9.1`
+
+#### Running commands with sudo
+
+Bower is a user command, there is no need to execute it with superuser permissions.
+However, if you still want to run commands with sudo, use `--allow-root` option.
+
+### Installing packages and dependencies
+
+Bower offers several ways to install packages:
+
+```
+# Using the dependencies listed in the current directory's bower.json
+bower install
+# Using a local or remote package
+bower install <package>
+# Using a specific version of a package
+bower install <package>#<version>
+# Using a different name and a specific version of a package
+bower install <name>=<package>#<version>
+```
+
+Where `<package>` can be any one of the following:
+
+* A name that maps to a package registered with Bower, e.g, `jquery`. ‡
+* A remote Git endpoint, e.g., `git://github.com/someone/some-package.git`. Can be
+  public or private. ‡
+* A local endpoint, i.e., a folder that's a Git repository. ‡
+* A shorthand endpoint, e.g., `someone/some-package` (defaults to GitHub). ‡
+* A URL to a file, including `zip` and `tar` files. Its contents will be
+  extracted.
+
+‡ These types of `<package>` might have versions available. You can specify a
+[semver](http://semver.org/) compatible version to fetch a specific release, and lock the
+package to that version. You can also use ranges to specify a range of versions.
+
+All package contents are installed in the `bower_components` directory by default.
+You should **never** directly modify the contents of this directory.
+
+Using `bower list` will show all the packages that are installed locally.
+
+**N.B.** If you aren't authoring a package that is intended to be consumed by
+others (e.g., you're building a web app), you should always check installed
+packages into source control.
+
+### Finding packages
+
+To search for packages registered with Bower:
+
+```
+bower search [<name>]
+```
+
+Using just `bower search` will list all packages in the registry.
+
+### Using packages
+
+The easiest approach is to use Bower statically, just reference the package's
+installed components manually using a `script` tag:
+
+```html
+<script src="/bower_components/jquery/index.js"></script>
+```
+
+For more complex projects, you'll probably want to concatenate your scripts or
+use a module loader. Bower is just a package manager, but there are plenty of
+other tools -- such as [Sprockets](https://github.com/sstephenson/sprockets)
+and [RequireJS](http://requirejs.org/) -- that will help you do this.
+
+### Registering packages
+
+To register a new package:
+
+* There **must** be a valid manifest JSON in the current working directory.
+* Your package should use [semver](http://semver.org/) Git tags.
+* Your package **must** be available at a Git endpoint (e.g., GitHub); remember
+  to push your Git tags!
+
+Then use the following command:
+
+```
+bower register <my-package-name> <git-endpoint>
+```
+
+The Bower registry does not have authentication or user management at this point
+in time. It's on a first come, first served basis. Think of it like a URL
+shortener. Now anyone can run `bower install <my-package-name>`, and get your
+library installed.
+
+There is no direct way to unregister a package yet. For now, you can [request a
+package be unregistered](https://github.com/bower/bower/issues/120).
+
+### Uninstalling packages
+
+To uninstall a locally installed package:
+
+```
+bower uninstall <package-name>
+```
+
+
+## Configuration
+
+Bower can be configured using JSON in a `.bowerrc` file.
+
+The current spec can be read
+[here](https://docs.google.com/document/d/1APq7oA9tNao1UYWyOm8dKqlRP2blVkROYLZ2fLIjtWc/edit#heading=h.4pzytc1f9j8k)
+in the `Configuration` section.
+
+
+## Defining a package
+
+You must create a `bower.json` in your project's root, and specify all of its
+dependencies. This is similar to Node's `package.json`, or Ruby's `Gemfile`,
+and is useful for locking down a project's dependencies.
+
+*NOTE:* In versions of Bower before 0.9.0 the package metadata file was called
+`component.json` rather than `bower.json`. This has changed to avoid a name
+clash with another tool. You can still use `component.json` for now but it is
+deprecated and the automatic fallback is likely to be removed in an upcoming
+release.
+
+You can interactively create a `bower.json` with the following command:
+
+```
+bower init
+```
+
+The `bower.json` defines several options:
+
+* `name` (required): The name of your package.
+* `version`: A semantic version number (see [semver](http://semver.org/)).
+* `main` [string|array]: The primary endpoints of your package.
+* `ignore` [array]: An array of paths not needed in production that you want
+  Bower to ignore when installing your package.
+* `dependencies` [hash]: Packages your package depends upon in production.
+* `devDependencies` [hash]: Development dependencies.
+* `private` [boolean]: Set to true if you want to keep the package private and
+  do not want to register the package in future.
+
+```json
+{
+  "name": "my-project",
+  "version": "1.0.0",
+  "main": "path/to/main.css",
+  "ignore": [
+    ".jshintrc",
+    "**/*.txt"
+  ],
+  "dependencies": {
+    "<name>": "<version>",
+    "<name>": "<folder>",
+    "<name>": "<package>"
+  },
+  "devDependencies": {
+    "<test-framework-name>": "<version>"
+  }
+}
+```
+
+
+## Consuming a package
+
+Bower also makes available a source mapping. This can be used by build tools to
+easily consume Bower packages.
+
+If you pass the `--paths` option to Bower's `list` command, you will get a
+simple path-to-name mapping:
+
+```json
+{
+  "backbone": "bower_components/backbone/index.js",
+  "jquery": "bower_components/jquery/index.js",
+  "underscore": "bower_components/underscore/index.js"
+}
+```
+
+Alternatively, every command supports the `--json` option that makes bower
+output JSON. Command result is outputted to `stdout` and error/logs to
+`stderr`.
+
+
+## Programmatic API
+
+Bower provides a powerful, programmatic API. All commands can be accessed
+through the `bower.commands` object.
+
+```js
+var bower = require('bower');
+
+bower.commands
+.install(['jquery'], { save: true }, { /* custom config */ })
+.on('end', function (installed) {
+    console.log(installed);
+});
+
+bower.commands
+.search('jquery', {})
+.on('end', function (results) {
+    console.log(results);
+});
+```
+
+Commands emit four types of events: `log`, `prompt`, `end`, `error`.
+
+* `log` is emitted to report the state/progress of the command.
+* `prompt` is emitted whenever the user needs to be prompted.
+* `error` will only be emitted if something goes wrong.
+* `end` is emitted when the command successfully ends.
+
+For a better of idea how this works, you may want to check out [our bin
+file](https://github.com/bower/bower/blob/master/bin/bower).
+
+When using bower programmatically, prompting is disabled by default. Though you can enable it when calling commands with `interactive: true` in the config.
+This requires you to listen for the `prompt` event and handle the prompting yourself. The easiest way is to use the [inquirer](https://npmjs.org/package/inquirer) npm module like so:
+
+```js
+var inquirer =  require('inquirer');
+
+bower.commands
+.install(['jquery'], { save: true }, { interactive: true })
+// ..
+.on('prompt', function (prompts, callback) {
+    inquirer.prompt(prompts, callback);
+});
+```
+
+
+## Completion (experimental)
+
+_NOTE_: Completion is still not implemented for the 1.0.0 release
+
+Bower now has an experimental `completion` command that is based on, and works
+similarly to the [npm completion](https://npmjs.org/doc/completion.html). It is
+not available for Windows users.
+
+This command will output a Bash / ZSH script to put into your `~/.bashrc`,
+`~/.bash_profile`, or `~/.zshrc` file.
+
+```
+bower completion >> ~/.bash_profile
+```
+
+
+## A note for Windows users
+
+To use Bower on Windows, you must install
+[msysgit](http://code.google.com/p/msysgit/) correctly. Be sure to check the
+option shown below:
+
+![msysgit](http://f.cl.ly/items/2V2O3i1p3R2F1r2v0a12/mysgit.png)
+
+Note that if you use TortoiseGit and if Bower keeps asking for your SSH
+password, you should add the following environment variable: `GIT_SSH -
+C:\Program Files\TortoiseGit\bin\TortoisePlink.exe`. Adjust the `TortoisePlink`
+path if needed.
+
+
+## Contact
+
+Have a question?
+
+* [StackOverflow](http://stackoverflow.com/questions/tagged/bower)
+* [Mailinglist](http://groups.google.com/group/twitter-bower) - twitter-bower@googlegroups.com
+* [\#bower](http://webchat.freenode.net/?channels=bower) on Freenode
+
+
+## Contributing to this project
+
+Anyone and everyone is welcome to contribute. Please take a moment to
+review the [guidelines for contributing](CONTRIBUTING.md).
+
+* [Bug reports](CONTRIBUTING.md#bugs)
+* [Feature requests](CONTRIBUTING.md#features)
+* [Pull requests](CONTRIBUTING.md#pull-requests)
+
+
+## Authors
+
+* [@fat](https://github.com/fat)
+* [@maccman](https://github.com/maccman)
+* [@satazor](https://github.com/satazor)
+
+Thanks for assistance and contributions:
+
+[@addyosmani](https://github.com/addyosmani),
+[@angus-c](https://github.com/angus-c),
+[@borismus](https://github.com/borismus),
+[@carsonmcdonald](https://github.com/carsonmcdonald),
+[@chriseppstein](https://github.com/chriseppstein),
+[@danwrong](https://github.com/danwrong),
+[@davidmaxwaterman](https://github.com/davidmaxwaterman),
+[@desandro](https://github.com/desandro),
+[@hemanth](https://github.com/hemanth),
+[@isaacs](https://github.com/isaacs),
+[@josh](https://github.com/josh),
+[@jrburke](https://github.com/jrburke),
+[@marcelombc](https://github.com/marcelombc),
+[@marcooliveira](https://github.com/marcooliveira),
+[@mklabs](https://github.com/mklabs),
+[@MrDHat](https://github.com/MrDHat),
+[@necolas](https://github.com/necolas),
+[@paulirish](https://github.com/paulirish),
+[@richo](https://github.com/richo),
+[@rvagg](https://github.com/rvagg),
+[@sindresorhus](https://github.com/sindresorhus),
+[@SlexAxton](https://github.com/SlexAxton),
+[@sstephenson](https://github.com/sstephenson),
+[@svnlto](https://github.com/svnlto),
+[@tomdale](https://github.com/tomdale),
+[@uzquiano](https://github.com/uzquiano),
+[@visionmedia](https://github.com/visionmedia),
+[@wagenet](https://github.com/wagenet),
+[@wibblymat](https://github.com/wibblymat),
+[@wycats](https://github.com/wycats)
+
+
+## License
+
+Copyright 2012 Twitter, Inc.
+
+Licensed under the MIT License

test/fixtures/eat-cpu.md

@@ -0,0 +1,25 @@
+[@addyosmani](https://github.com/addyosmani),
+[@borismus](https://github.com/borismus),
+[@carsonmcdonald](https://github.com/carsonmcdonald),
+[@chriseppstein](https://github.com/chriseppstein),
+[@danwrong](https://github.com/danwrong),
+[@davidmaxwaterman](https://github.com/davidmaxwaterman),
+[@desandro](https://github.com/desandro),
+[@hemanth](https://github.com/hemanth),
+[@isaacs](https://github.com/isaacs),
+[@josh](https://github.com/josh),
+[@jrburke](https://github.com/jrburke),
+[@marcelombc](https://github.com/marcelombc),
+[@marcooliveira](https://github.com/marcooliveira),
+[@mklabs](https://github.com/mklabs),
+[@MrDHat](https://github.com/MrDHat),
+[@necolas](https://github.com/necolas),
+[@paulirish](https://github.com/paulirish),
+[@richo](https://github.com/richo),
+[@rvagg](https://github.com/rvagg),
+[@sindresorhus](https://github.com/sindresorhus),
+[@SlexAxton](https://github.com/SlexAxton),
+[@sstephenson](https://github.com/sstephenson),
+[@svnlto](https://github.com/svnlto),
+[@tomdale](https://github.com/tomdale),
+[@uzquiano](https://github.com/uzquiano),

test/fixtures/sonido.md

@@ -0,0 +1 @@
+Configuration Wizard: <!--- This is what the user will see during the configuration ->

test/init.js

@@ -20,6 +20,14 @@ if (process.env.DB) {
   config.database.dialect = process.env.DB;
 }
 
+if (process.env.DB_PORT) {
+  config.database.port = parseInt(process.env.DB_PORT);
+}
+
+if (process.env.DB_USER) {
+  config.database.username = process.env.DB_USER;
+}
+
 if (process.env.CNPM_SOURCE_NPM) {
   config.sourceNpmRegistry = process.env.CNPM_SOURCE_NPM;
 }

test/init_db.js

@@ -22,7 +22,8 @@ var config = require('../config');
 
 // init db first
 var initscript = path.join(__dirname, '..', 'models', 'init_script.js');
-var cmd = ['node', '--harmony', initscript, 'true', config.database.dialect].join(' ');
+var cmd = ['node', '--harmony', initscript, 'true',
+  config.database.dialect, config.database.port, config.database.username].join(' ');
 console.log('$ %s', cmd);
 var stdout = childProcess.execSync(cmd);
 process.stdout.write(stdout);
@@ -56,6 +57,7 @@ usernames.forEach(function (name) {
       process.exit(0);
     }
   }).catch(function (err) {
-    throw err;
+    console.log(err);
+    process.exit(1);
   });
 });