Release 0.5.0

Dist sync

.jshintrc

@@ -58,7 +58,7 @@
     "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
     "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
     "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
-    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+    "validthis"     : true,      // true: Tolerate using this in a non-constructor function
 
     // Environments
     "browser"       : true,     // Web Browser (window, document, etc)

History.md

@@ -1,4 +1,15 @@
 
+0.5.0 / 2014-05-13
+==================
+
+ * filter /nightlies/*
+ * use koa setter instead of set()
+ * add more info on error email
+ * add sync dist to sync/index.js
+ * show dist page
+ * sync dist file and save it to database
+ * disable gzip before #335 has fix
+
 0.4.3 / 2014-04-18 
 ==================
 

Makefile

@@ -2,7 +2,7 @@ TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
 REPORTER = tap
 TIMEOUT = 30000
 MOCHA_OPTS =
-REGISTRY = --registry=http://r.cnpmjs.org
+REGISTRY = --registry=http://registry.npm.taobao.org
 
 install:
 	@npm install $(REGISTRY) \

common/qnfs.js

@@ -16,10 +16,11 @@
 
 var thunkify = require('thunkify-wrap');
 var qn = require('qn');
+var fs = require('fs');
 var config = require('../config');
 var client = qn.create(config.qn);
 
-thunkify(client, ['delete', 'uploadFile', 'upload']);
+thunkify(client, ['delete', 'uploadFile', 'upload', 'download']);
 
 exports._client = client;
 
@@ -62,6 +63,14 @@ exports.url = function (key) {
   return client.resourceURL(key);
 };
 
+exports.download = function* (key, filepath, options) {
+  var writeStream = fs.createWriteStream(filepath);
+  yield client.download(key, {
+    timeout: options.timeout,
+    writeStream: writeStream
+  });
+};
+
 exports.remove = function *(key) {
   try {
     return yield client.delete(key);

config/index.js

@@ -76,7 +76,8 @@ var config = {
     debug: false
   },
 
-  disturl: 'http://dist.u.qiniudn.com',
+  disturl: 'http://nodejs.org/dist',
+  syncDist: false,
   logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
   registryHost: 'r.cnpmjs.org',
   // customReadmeFile: __dirname + '/web_readme.md',
@@ -107,7 +108,8 @@ var config = {
     whiteList: [],
     blackList: [],
     message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
-  }
+  },
+  enableCompress: false, // enable gzip response or not
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js

controllers/registry/module.js

@@ -37,6 +37,7 @@ var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var logger = require('../../common/logger');
 var ModuleDeps = require('../../proxy/module_deps');
 var ModuleStar = require('../../proxy/module_star');
+var downloadAsReadStream = require('../utils').downloadAsReadStream;
 
 /**
  * show all version of a module
@@ -218,8 +219,6 @@ exports.get = function *(next) {
 
 var _downloads = {};
 
-var DOWNLOAD_TIMEOUT = ms('10m');
-
 exports.download = function *(next) {
   var name = this.params.name;
   var filename = this.params.filename;
@@ -259,28 +258,13 @@ exports.download = function *(next) {
   _downloads[name] = (_downloads[name] || 0) + 1;
 
   if (typeof dist.size === 'number') {
-    this.set('Content-Length', dist.size);
+    this.length = dist.size;
   }
-  this.set('Content-Type', mime.lookup(dist.key));
-  this.set('Content-Disposition', 'attachment; filename="' + filename + '"');
-  this.set('ETag', dist.shasum);
+  this.type = mime.lookup(dist.key);
+  this.attachment = filename;
+  this.etag = dist.shasum;
 
-  // use download file api
-  var tmpPath = path.join(config.uploadDir,
-    utility.randomString() + dist.key.replace(/\//g, '-'));
-  function cleanup() {
-    fs.unlink(tmpPath, utility.noop);
-  }
-  try {
-    yield nfs.download(dist.key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
-  } catch (err) {
-    cleanup();
-    this.throw(err);
-  }
-  var tarball = fs.createReadStream(tmpPath);
-  tarball.once('error', cleanup);
-  tarball.once('end', cleanup);
-  this.body = tarball;
+  this.body = yield* downloadAsReadStream(dist.key);
 };
 
 setInterval(function () {

controllers/total.js

@@ -15,13 +15,12 @@
  * Module dependencies.
  */
 
-var microtime = require('microtime');
 var Total = require('../proxy/total');
 var Download = require('./download');
 var version = require('../package.json').version;
 var config = require('../config');
 
-var startTime = '' + microtime.now();
+var startTime = '' + Date.now();
 
 exports.show = function *() {
   var r = yield [Total.get(), Download.total()];

controllers/utils.js

@@ -0,0 +1,46 @@
+/**!
+ * cnpmjs.org - controllers/utils.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:utils');
+var path = require('path');
+var fs = require('fs');
+var utility = require('utility');
+var ms = require('ms');
+var nfs = require('../common/nfs');
+var config = require('../config');
+
+var DOWNLOAD_TIMEOUT = ms('10m');
+
+exports.downloadAsReadStream = function* (key) {
+  var tmpPath = path.join(config.uploadDir,
+    utility.randomString() + key.replace(/\//g, '-'));
+  function cleanup() {
+    debug('cleanup %s', tmpPath);
+    fs.unlink(tmpPath, utility.noop);
+  }
+  debug('downloadAsReadStream() %s to %s', key, tmpPath);
+  try {
+    yield nfs.download(key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
+  } catch (err) {
+    debug('downloadAsReadStream() %s to %s error: %s', key, tmpPath, err.stack);
+    cleanup();
+    throw err;
+  }
+  var tarball = fs.createReadStream(tmpPath);
+  tarball.once('error', cleanup);
+  tarball.once('end', cleanup);
+  return tarball;
+};

controllers/web/dist.js

@@ -14,10 +14,59 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:controllers:web:dist');
+var mime = require('mime');
+var Dist = require('../../proxy/dist');
 var config = require('../../config');
+var downloadAsReadStream = require('../utils').downloadAsReadStream;
 
-exports.redirect = function *(next) {
+function padding(max, current, pad) {
+  pad = pad || ' ';
+  var left = max - current;
+  var str = '';
+  for (var i = 0; i < left; i++) {
+    str += pad;
+  }
+  return str;
+}
+
+exports.list = function* (next) {
   var params = this.params;
-  var url = config.disturl + (params[0] || '/');
-  this.redirect(url);
+  var url = params[0] || '/';
+  var isDir = url[url.length - 1] === '/';
+  if (!isDir) {
+    return yield* download.call(this, next);
+  }
+
+  var items = yield* Dist.listdir(url);
+  yield this.render('dist', {
+    title: 'Mirror index of ' + config.disturl + url,
+    disturl: config.disturl,
+    dirname: url,
+    items: items,
+    padding: padding
+  });
 };
+
+ function* download(next) {
+  var fullname = this.params[0];
+  var info = yield* Dist.getfile(fullname);
+  debug('download %s got %j', fullname, info);
+  if (!info || !info.url) {
+    return yield* next;
+  }
+
+  if (info.url.indexOf('http') === 0) {
+    return this.redirect(info.url);
+  }
+
+  // download it from nfs
+  if (typeof info.size === 'number') {
+    this.length = info.size;
+  }
+  this.type = mime.lookup(info.url);
+  this.attachment = info.name;
+  this.etag = info.sha1;
+
+  this.body = yield* downloadAsReadStream(info.url);
+}

docs/db.sql

@@ -137,3 +137,30 @@ CREATE TABLE `module_deps` (
  UNIQUE KEY `name_deps` (`name`,`deps`),
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
+
+CREATE TABLE `dist_dir` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(200) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
+
+CREATE TABLE `dist_file` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
+ `sha1` varchar(40) COMMENT 'sha1 hex value',
+ `url` varchar(2048),
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `fullname` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';

docs/new.sql

@@ -0,0 +1,27 @@
+-- http://nodejs.org/dist/ mirror
+CREATE TABLE `dist_dir` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(200) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
+
+CREATE TABLE `dist_file` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
+ `sha1` varchar(40) COMMENT 'sha1 hex value',
+ `url` varchar(2048),
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `fullname` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';

middleware/registry_not_found.js

@@ -17,7 +17,7 @@
 module.exports = function *notFound(next) {
   yield *next;
 
-  if (this.status) {
+  if (this.status && this.status !== 404) {
     return;
   }
   this.status = 404;

middleware/web_not_found.js

@@ -14,14 +14,17 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:middleware:web_not_found');
+
 module.exports = function *notFound(next) {
   yield *next;
 
-  if (this.status) {
+  if (this.status && this.status !== 404) {
     return;
   }
 
   var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  debug('%s match %j', this.url, m);
   if (m) {
     return this.redirect('/package/' + m[1]);
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -10,41 +10,40 @@
     "stop": "./bin/nodejsctl stop"
   },
   "dependencies": {
-    "bytes": "0.3.0",
-    "co": "3.0.5",
+    "bytes": "1.0.0",
+    "co": "3.0.6",
     "co-defer": "0.1.0",
     "co-gather": "0.0.1",
     "co-read": "0.0.2",
     "co-redis": "1.1.0",
-    "co-urllib": "0.2.1",
+    "co-urllib": "0.2.2",
     "co-write": "0.3.0",
-    "copy-to": "0.0.3",
-    "debug": "0.8.0",
+    "copy-to": "1.0.1",
+    "debug": "0.8.1",
     "eventproxy": "0.3.1",
-    "giturl": "0.0.2",
+    "giturl": "0.0.3",
     "graceful": "0.0.6",
     "gravatar": "1.0.6",
     "humanize-number": "0.0.2",
-    "koa": "0.5.5",
-    "koa-limit": "1.0.0",
+    "koa": "0.6.1",
+    "koa-limit": "1.0.2",
     "koa-markdown": "0.0.3",
     "koa-middlewares": "0.1.3",
     "logfilestream": "0.1.0",
     "marked": "0.3.2",
-    "microtime": "0.5.1",
     "mime": "1.2.11",
-    "mkdirp": "0.3.5",
+    "mkdirp": "0.5.0",
     "moment": "2.6.0",
     "ms": "0.6.2",
-    "multiline": "0.3.2",
-    "mysql": "2.1.1",
+    "multiline": "0.3.4",
+    "mysql": "2.2.0",
     "nodemailer": "0.6.3",
     "qn": "0.2.1",
     "ready": "0.1.1",
     "redis": "0.10.1",
-    "semver": "2.2.1",
+    "semver": "2.3.0",
     "thunkify-wrap": "0.1.1",
-    "utility": "0.1.12"
+    "utility": "0.1.13"
   },
   "devDependencies": {
     "autod": ">=0.0.13",
@@ -58,7 +57,7 @@
     "mocha": "*",
     "pedding": "0.0.3",
     "should": "3.3.1",
-    "supertest": "0.11.0"
+    "supertest": "0.12.1"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {

proxy/dist.js

@@ -0,0 +1,82 @@
+/**!
+ * cnpmjs.org - proxy/dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var path = require('path');
+var multiline = require('multiline');
+var mysql = require('../common/mysql');
+
+var SAVE_FILE_SQL = multiline(function () {;/*
+  INSERT INTO
+    dist_file(gmt_create, gmt_modified, name, parent, date, size, url, sha1)
+  VALUES
+    (now(), now(), ?, ?, ?, ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    name=VALUES(name),
+    parent=VALUES(parent),
+    date=VALUES(date),
+    size=VALUES(size),
+    url=VALUES(url),
+    sha1=VALUES(sha1);
+*/});
+
+exports.savefile = function* (info) {
+  return yield mysql.query(SAVE_FILE_SQL, [
+    info.name, info.parent, info.date, info.size, info.url, info.sha1
+  ]);
+};
+
+var SAVE_DIR_SQL = multiline(function () {;/*
+  INSERT INTO
+    dist_dir(gmt_create, gmt_modified, name, parent, date)
+  VALUES
+    (now(), now(), ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    name=VALUES(name),
+    parent=VALUES(parent),
+    date=VALUES(date);
+*/});
+
+exports.savedir = function* (info) {
+  return yield mysql.query(SAVE_DIR_SQL, [
+    info.name, info.parent, info.date
+  ]);
+};
+
+var LIST_DIRS_SQL = multiline(function () {;/*
+  SELECT name, parent, date FROM dist_dir WHERE parent = ?;
+*/});
+var LIST_FILES_SQL = multiline(function () {;/*
+  SELECT name, parent, date, size, url, sha1
+  FROM dist_file WHERE parent = ?;
+*/});
+
+exports.listdir = function* (name) {
+  var rs = yield [
+    mysql.query(LIST_DIRS_SQL, [name]),
+    mysql.query(LIST_FILES_SQL, [name]),
+  ];
+  return rs[0].concat(rs[1]);
+};
+
+var GET_FILE_SQL = multiline(function () {;/*
+  SELECT name, parent, date, url, size, sha1 FROM dist_file WHERE parent = ? AND name = ?;
+*/});
+
+exports.getfile = function* (fullname) {
+  var name = path.basename(fullname);
+  var parent = path.dirname(fullname) + '/';
+  return yield mysql.queryOne(GET_FILE_SQL, [parent, name]);
+};

routes/web.js

@@ -36,7 +36,7 @@ function routes(app) {
 
   app.get('/_list/search/search', pkg.rangeSearch);
 
-  app.get(/^\/dist(\/.+)?/, dist.redirect);
+  app.get(/^\/dist(\/.+)?/, dist.list);
 }
 
 module.exports = routes;

servers/registry.js

@@ -18,7 +18,6 @@
 var koa = require('koa');
 var app = module.exports = koa();
 var http = require('http');
-var microtime = require('microtime');
 var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
@@ -28,7 +27,7 @@ var auth = require('../middleware/auth');
 var staticCache = require('../middleware/static');
 var notFound = require('../middleware/registry_not_found');
 
-app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
 app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
 app.use(staticCache);
 
@@ -40,7 +39,9 @@ app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
 app.use(notFound);
 
-app.use(middlewares.compress({threshold: 150}));
+if (config.enableCompress) {
+  app.use(middlewares.compress({threshold: 150}));
+}
 app.use(middlewares.conditional());
 app.use(middlewares.etag());
 

servers/web.js

@@ -18,7 +18,6 @@
 var path = require('path');
 var http = require('http');
 var fs = require('fs');
-var microtime = require('microtime');
 var koa = require('koa');
 var middlewares = require('koa-middlewares');
 var markdown = require('koa-markdown');
@@ -35,7 +34,7 @@ var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
 app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
 app.use(staticCache);
 
@@ -48,7 +47,10 @@ app.use(middlewares.bodyParser());
 app.use(auth());
 app.use(notFound);
 
-app.use(middlewares.compress({threshold: 150}));
+if (config.enableCompress) {
+  app.use(middlewares.compress({threshold: 150}));
+}
+
 app.use(middlewares.conditional());
 app.use(middlewares.etag());
 

sync/index.js

@@ -23,6 +23,7 @@ var config = require('../config');
 var mail = require('../common/mail');
 var Total = require('../proxy/total');
 var logger = require('../common/logger');
+var startSyncDist = require('./sync_dist');
 
 var sync = null;
 
@@ -76,6 +77,30 @@ if (sync) {
   setInterval(handleSync, ms(config.syncInterval));
 }
 
+var syncingDist = false;
+var syncDist = co(function* syncDist() {
+  if (syncingDist) {
+    return;
+  }
+  syncingDist = true;
+  logger.info('Start syncing dist...');
+  try {
+    yield * startSyncDist();
+  } catch (err) {
+    err.message += ' (sync dist error)';
+    logger.warn('Sync dist error: %s: %s\n%s', err.name, err.message, err.stack);
+    sendMailToAdmin(err, null, new Date());
+  }
+  syncingDist = false;
+});
+
+if (config.syncDist) {
+  syncDist();
+  setInterval(syncDist, ms(config.syncInterval));
+} else {
+  logger.info('sync dist disable');
+}
+
 function sendMailToAdmin(err, result, syncTime) {
   result = result || {};
   var to = [];
@@ -91,7 +116,7 @@ function sendMailToAdmin(err, result, syncTime) {
     subject = 'Sync Error';
     type = 'error';
     html = util.format('Sync packages from official registry failed.\n' +
-      'Start sync time is %s.\nError message is %s.', syncTime, err.stack);
+      'Start sync time is %s.\nError message is %s: %s\n%s.', syncTime, err.name, err.message, err.stack);
   } else if (result.fails && result.fails.length) {
     subject = 'Sync Finished But Some Packages Failed';
     type = 'warn';

sync/sync_dist.js

@@ -0,0 +1,213 @@
+/**!
+ * cnpmjs.org - sync/sync_dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:sync:sync_dist');
+var fs = require('fs');
+var urllib = require('co-urllib');
+var co = require('co');
+var bytes = require('bytes');
+var crypto = require('crypto');
+var utility = require('utility');
+var thunkify = require('thunkify-wrap');
+var common = require('../lib/common');
+var Dist = require('../proxy/dist');
+var config = require('../config');
+var nfs = require('../common/nfs');
+var logger = require('../common/logger');
+
+var disturl = config.disturl;
+var USER_AGENT = 'distsync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+
+// <a href="latest/">latest/</a>                                            02-May-2014 14:45                   -
+// <a href="node-v0.4.10.tar.gz">node-v0.4.10.tar.gz</a>                                26-Aug-2011 16:22            12410018
+var FILE_RE = /^<a[^>]+>([^<]+)<\/a>\s+(\d+\-\w+\-\d+ \d+\:\d+)\s+([\-\d]+)/;
+
+module.exports = sync;
+
+function* sync(name) {
+  name = name || '/';
+  yield* syncDir(name);
+}
+
+function* syncDir(fullname, info) {
+  var news = yield* listdiff(fullname);
+  var files = [];
+  var dirs = [];
+
+  for (var i = 0; i < news.length; i++) {
+    var item = news[i];
+    if (item.type === 'dir') {
+      dirs.push(item);
+    } else {
+      files.push(item);
+    }
+  }
+
+  logger.info('sync remote:%s got %d new items, %d dirs, %d files to sync',
+    fullname, news.length, dirs.length, files.length);
+
+  for (var i = 0; i < files.length; i++) {
+    yield* syncFile(files[i]);
+  }
+
+  for (var i = 0; i < dirs.length; i++) {
+    var dir = dirs[i];
+    yield* syncDir(dir.parent + dir.name, dir);
+  }
+
+  if (info) {
+    logger.info('Save dir:%s %j to database', fullname, info);
+    yield* Dist.savedir(info);
+  }
+
+  logger.info('Sync %s finished, %d dirs, %d files',
+    fullname, dirs.length, files.length);
+}
+
+function* syncFile(info) {
+  var name = info.parent + info.name;
+  name = process.pid + name.replace(/\//g, '_'); // make sure no parent dir
+  var downurl = disturl + info.parent + info.name;
+  var filepath = common.getTarballFilepath(name);
+  var ws = fs.createWriteStream(filepath);
+
+  var options = {
+    writeStream: ws,
+    followRedirect: true,
+    timeout: 6000000, // 100 minutes download
+    headers: {
+      'user-agent': USER_AGENT
+    }
+  };
+
+  try {
+    logger.info('downloading %s %s to %s', bytes(info.size), downurl, filepath);
+    // get tarball
+    var r = yield *urllib.request(downurl, options);
+    var statusCode = r.status || -1;
+    logger.info('download %s got status %s, headers: %j', downurl, statusCode, r.headers);
+    if (statusCode !== 200) {
+      var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
+      err.name = 'DownloadDistFileError';
+      throw err;
+    }
+
+    var shasum = crypto.createHash('sha1');
+    var dataSize = 0;
+    var rs = fs.createReadStream(filepath);
+    rs.on('data', function (data) {
+      shasum.update(data);
+      dataSize += data.length;
+    });
+    var end = thunkify.event(rs);
+    yield end(); // after end event emit
+
+    if (dataSize === 0) {
+      var err = new Error('Download ' + downurl + ' file size is zero');
+      err.name = 'DownloadDistFileZeroSizeError';
+      throw err;
+    }
+
+    if (dataSize !== info.size) {
+      var err = new Error('Download ' + downurl + ' file size is '
+        + dataSize + ' not match ' + info.size);
+      err.name = 'DownloadDistFileSizeError';
+      throw err;
+    }
+
+    shasum = shasum.digest('hex');
+    var args = {
+      key: '/dist' + info.parent + info.name,
+      size: info.size,
+      shasum: shasum,
+    };
+
+    // upload to NFS
+    logger.info('uploading %s to nfs:%s', filepath, args.key);
+    var result = yield nfs.upload(filepath, args);
+    info.url = result.url || result.key;
+    info.sha1 = shasum;
+
+    logger.info('upload %s to nfs:%s with size:%d, sha1:%s',
+      args.key, info.url, info.size, info.sha1);
+  } finally {
+    // remove tmp file whatever
+    fs.unlink(filepath, utility.noop);
+  }
+
+  logger.info('Sync dist file: %j done', info);
+  yield* Dist.savefile(info);
+}
+
+function* listdir(fullname) {
+  var url = disturl + fullname;
+  var result = yield* urllib.request(url, {
+    timeout: 60000,
+  });
+  debug('listdir %s got %s, %j', url, result.status, result.headers);
+  var html = result.data && result.data.toString() || '';
+  var lines = html.split('\n');
+  var items = [];
+  for (var i = 0; i < lines.length; i++) {
+    var m = FILE_RE.exec(lines[i].trim());
+    if (!m) {
+      continue;
+    }
+    var itemName = m[1].replace(/^\/+/, '');
+    if (!itemName) {
+      continue;
+    }
+
+    // filter /nightlies/*
+    if (itemName.indexOf('nightlies/') === 0) {
+      continue;
+    }
+
+    items.push({
+      name: itemName, // 'SHASUMS.txt', 'x64/'
+      date: m[2],
+      size: m[3] === '-' ? '-' : parseInt(m[3]),
+      type: m[3] === '-' ? 'dir' : 'file',
+      parent: fullname, // '/', '/v0.10.28/'
+    });
+  }
+  return items;
+}
+
+function* listdiff(fullname) {
+  var items = yield* listdir(fullname);
+  if (items.length === 0) {
+    return items;
+  }
+  var exists = yield* Dist.listdir(fullname);
+  debug('listdiff %s got %s exists items', fullname, exists.length);
+  var map = {};
+  for (var i = 0; i < exists.length; i++) {
+    var item = exists[i];
+    map[item.name] = item;
+  }
+  var news = [];
+  for (var i = 0; i < items.length; i++) {
+    var item = items[i];
+    var exist = map[item.name];
+    if (!exist || exist.date !== item.date || exist.size !== item.size) {
+      news.push(item);
+    } else {
+      debug('skip %s', item.name);
+    }
+  }
+  return news;
+}

test/controllers/registry/module.test.js

@@ -113,7 +113,7 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .get('/mk2testmodule')
       .set('accept-encoding', 'gzip')
-      .expect('content-encoding', 'gzip')
+      // .expect('content-encoding', 'gzip')
       .expect(200, function (err, res) {
         // console.log(res.headers)
         should.not.exist(err);
@@ -399,7 +399,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should upload tarball fail 404 when rev wrong', function (done) {
       var body = fs.readFileSync(path.join(fixtures, 'testputmodule-0.1.9.tgz'));
       request(app)
-      .put('/' + pkg.name + '/-/' + pkg.name + '-0.1.9.tgz/-rev/' + lastRev + '1')
+      .put('/' + pkg.name + '/-/' + pkg.name + '-0.1.9.tgz/-rev/' + '1231231')
       .set('authorization', baseauth)
       .set('content-type', 'application/octet-stream')
       .set('content-length', '' + body.length)

test/controllers/web/dist.test.js

@@ -16,22 +16,95 @@
 
 var should = require('should');
 var request = require('supertest');
+var pedding = require('pedding');
+var mm = require('mm');
 var app = require('../../../servers/web');
+var Dist = require('../../../proxy/dist');
 
 describe('controllers/web/dist.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
 
-  describe('GET /dist', function (done) {
-    it('should 302 to config.disturl', function (done) {
+  afterEach(mm.restore);
+
+  describe('GET /dist/*', function (done) {
+    it('should GET /dist show file list', function (done) {
+      done = pedding(2, done);
+
       request(app)
       .get('/dist')
-      .expect('Location', 'http://dist.u.qiniudn.com/')
-      .expect(302, done);
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.include('<title>Mirror index of http://nodejs.org/dist/</title>');
+        done();
+      });
+
+      request(app)
+      .get('/dist/')
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.include('<title>Mirror index of http://nodejs.org/dist/</title>');
+        done();
+      });
+    });
+
+    it('should mock return files and dirs', function (done) {
+      mm(Dist, 'listdir', function* () {
+        return [
+          {name: 'ooxx/', date: '02-May-2014 00:54'},
+          {name: 'foo.txt', size: 1024, date: '02-May-2014 00:54'},
+        ];
+      });
+      request(app)
+      .get('/dist/v1.0.0/')
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.include('<title>Mirror index of http://nodejs.org/dist/v1.0.0/</title>');
+        res.text.should.include('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/v1.0.0/">http://nodejs.org/dist/v1.0.0/</a></h1>');
+        res.text.should.include('<a href="ooxx/">ooxx/</a>                                             02-May-2014 00:54                   -\n');
+        res.text.should.include('<a href="foo.txt">foo.txt</a>                                           02-May-2014 00:54                1024\n');
+        done();
+      });
+    });
+  });
+
+  describe('GET /dist files', function () {
+    it('should redirect to nfs url', function (done) {
+      mm(Dist, 'getfile', function* () {
+        return {
+          name: 'foo.txt', size: 1024, date: '02-May-2014 00:54',
+          url: 'http://mock.com/dist/v0.10.28/SHASUMS.txt'
+        };
+      });
+
+      request(app)
+      .get('/dist/v0.10.28/SHASUMS.txt')
+      .expect(302)
+      .expect('Location', 'http://mock.com/dist/v0.10.28/SHASUMS.txt', done);
+    });
+
+    it('should download nfs file and send it', function (done) {
+      mm(Dist, 'getfile', function* () {
+        return {
+          name: 'foo.txt',
+          size: 1264,
+          date: '02-May-2014 00:54',
+          url: '/dist/v0.10.28/SHASUMS.txt'
+        };
+      });
+
+      request(app)
+      .get('/dist/v0.10.28/SHASUMS.txt')
+      .expect(200)
+      .expect(/6eff580cc8460741155d42ef1ef537961194443f/, done);
     });
   });
 });

test/controllers/web/package.test.js

@@ -95,7 +95,7 @@ describe('controllers/web/package.test.js', function () {
       request(app)
       .get('/package/mk2testmodule')
       .expect(200)
-      .expect('content-encoding', 'gzip')
+      // .expect('content-encoding', 'gzip')
       .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)

test/sync_dist.js

@@ -0,0 +1,24 @@
+/**!
+ * cnpmjs.org - test/sync_dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug');
+debug.enable('cnpmjs.org*');
+var co = require('co');
+var syncdist = require('../sync/sync_dist');
+
+co(function* () {
+  yield* syncdist('/v0.10.28/');
+})();

test/zlib_write_after_close.js

@@ -0,0 +1,9 @@
+var zlib = require('zlib');
+
+var stream = zlib.createGzip();
+
+stream.write('foo');
+stream.close();
+
+// Assertion failed: (!ctx->pending_close_ && "close is pending"), function Write, file ../src/node_zlib.cc, line 150.
+stream.write('again');

view/web/dist.html

@@ -0,0 +1,9 @@
+<h1>Mirror index of <a target="_blank" href="<%= disturl %><%= dirname %>"><%= disturl %><%= dirname %></a></h1>
+<hr>
+<pre><a href="../">../</a><% for (var i = 0; i < items.length; i++) {
+  var item = items[i];
+  var sizestr = '' + (item.size || '-');
+%>
+<a href="<%= item.name %>"><%= item.name %></a><%- padding(50, item.name.length, " ") %><%= item.date %><%- padding(20, sizestr.length, " ") %><%= sizestr %><% } %>
+</pre>
+<hr>

worker.js

@@ -24,7 +24,7 @@ var web = require('./servers/web');
 registry.listen(config.registryPort, config.bindingHost);
 web.listen(config.webPort, config.bindingHost);
 
-console.log('[%s] [worker:%d] Server started, registry server listen at %s:%d, web listen at %s%d, cluster: %s',
+console.log('[%s] [worker:%d] Server started, registry server listen at %s:%d, web listen at %s:%d, cluster: %s',
   new Date(), process.pid,
   config.bindingHost, config.registryPort,
   config.bindingHost, config.webPort,