Release 0.2.13

markdown tmpl not support footer, need to wrap on app start

.gitignore

@@ -19,3 +19,4 @@ config/config.js
 backup/*.json
 backup/*.gz
 docs/web/history.md
+view/web/_layout.html

.npmignore

@@ -8,3 +8,4 @@ logo.png
 public/dist/
 backup/*.json
 backup/*.gz
+view/web/_layout.html

.travis.yml

@@ -1,5 +1,5 @@
 language: node_js
 node_js:
   - '0.10'
-install: npm install --registry=http://registry.cnpmjs.org
+install: make install
 script: make test-coveralls

AUTHORS

@@ -1,5 +1,5 @@
 # Ordered by date of first contribution.
-# Auto-generated by 'contributors' on Fri, 20 Dec 2013 09:33:09 GMT.
+# Auto-generated by 'contributors' on Fri, 10 Jan 2014 02:56:32 GMT.
 # https://github.com/xingrz/node-contributors
 
 fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)

History.md

@@ -1,4 +1,96 @@
 
+0.2.13 / 2014-01-15 
+==================
+
+  * markdown tmpl not support footer, need to wrap on app start
+
+0.2.12 / 2014-01-15 
+==================
+
+  * add footer and npm client name customable
+
+0.2.11 / 2014-01-15 
+==================
+
+  * package page contributor link to search, default is true
+
+0.2.10 / 2014-01-14 
+==================
+
+  * fix #155 Content-Disposition wrong.
+
+0.2.9 / 2014-01-14 
+==================
+
+  * support startkey=c and startkey="c"
+  * support couch db search api. fixed #153
+  * fix fork me image link
+  * support sync by query.name
+
+0.2.8 / 2014-01-14 
+==================
+
+  * dont show err stack on test env
+  * add download link for package page
+
+0.2.7 / 2014-01-13 
+==================
+
+  * add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
+
+0.2.6 / 2014-01-13 
+==================
+
+  * support custom session store, fixed #146
+
+0.2.5 / 2014-01-13 
+==================
+
+  * add download timeout and unit test
+  * use downloadStream() first
+  * nfs download to a writeable stream.
+
+0.2.4 / 2014-01-10 
+==================
+
+  * set main script to  index.js, fixed #142
+
+0.2.3 / 2014-01-10 
+==================
+
+  * Dont show sync button on private package
+  * Sync package as publish with no deps. fixed #138
+
+0.2.2 / 2014-01-10 
+==================
+
+  * keep compatibility
+  * qnfs upload only callback a url
+  * compat remove package
+  * set tarball url
+  * new npm publish in one request, add _publish_in_cnpm
+  * support unsure name ufs
+  * contributors maybe a object
+  * Object #<Object> has no method 'forEach' fixed #134
+  * support custom config as a module, fixed issue #132
+  * support npm new publish flow. fixed #129
+  * add toString and constructor to test admin
+  * fix #119 hasOwnProperty check admin bug.
+
+0.2.0 / 2013-12-27 
+==================
+
+  * remove to lower case
+  * fix #127 execSync and execsync.
+  * add contributors list on package page
+  * mv blanket to config
+  * sync typeerror fix #statusCode
+  * add disturl
+  * fix #122 admin security bug
+  * fixed #121, let pkg 404 as success
+  * fix sql insert error
+  * fix typos
+
 0.1.3 / 2013-12-20
 ==================
 

Makefile

@@ -4,12 +4,13 @@ TIMEOUT = 30000
 MOCHA_OPTS =
 
 install:
-	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm
+	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm --disturl=http://dist.u.qiniudn.com
 
 test: install
 	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
+		--require should \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
@@ -27,4 +28,7 @@ test-coveralls: test
 
 test-all: test test-cov
 
+contributors: install
+	@./node_modules/contributors/bin/contributors -f plain -o AUTHORS
+
 .PHONY: test

README.md

@@ -1,20 +1,21 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)[![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org)[![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
-![logo](https://raw.github.com/fengmk2/cnpmjs.org/master/logo.png)
+![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
+
 ## Install
 
 ```bash
@@ -33,14 +34,14 @@ $ node dispatch.js
 $ git summary
 
  project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
+ repo age : 5 weeks
+ active   : 98 days
+ commits  : 239
+ files    : 85
  authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
+   140  fengmk2                 58.6%
+    98  dead_horse              41.0%
+     1  Alsotang                0.4%
 ```
 
 ## License

common/qnfs.js

@@ -19,6 +19,8 @@ var config = require('../config');
 
 var client = qn.create(config.qn);
 
+exports._client = client;
+
 /**
  * Upload file
  *
@@ -32,7 +34,23 @@ var client = qn.create(config.qn);
  */
 exports.upload = function (filepath, options, callback) {
   client.delete(options.key, function (err, data) {
-    client.uploadFile(filepath, {key: options.key, size: options.size}, callback);
+    client.uploadFile(filepath, {key: options.key, size: options.size}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
+  });
+};
+
+exports.uploadBuffer = function (buf, options, callback) {
+  client.delete(options.key, function (err, data) {
+    client.upload(buf, {key: options.key}, function (err, data) {
+      if (err) {
+        return callback(err);
+      }
+      callback(null, {url: data.url});
+    });
   });
 };
 

common/session.js

@@ -33,7 +33,7 @@ if (config.debug) {
   session = connect.session({
     key: key,
     secret: config.sessionSecret,
-    store: new RedisStore(config.redis),
+    store: config.sessionStore || new RedisStore(config.redis),
     cookie: cookie,
   });
 }

config/index.js

@@ -68,9 +68,13 @@ var config = {
     user: 'address@gmail.com',
     pass: 'your password',
     ssl: true,
-    debug: false 
+    debug: false
   },
 
+  registryHost: 'r.cnpmjs.org',
+  customFooter: '', // you can add copyright and site total script html here
+  npmClientName: 'cnpm', // use `${name} install package`
+  packagePageContributorSearch: true, // package page contributor link to search, default is true
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
@@ -98,3 +102,12 @@ mkdirp.sync(config.logdir);
 mkdirp.sync(config.uploadDir);
 
 module.exports = config;
+
+config.loadConfig = function (customConfig) {
+  if (!customConfig) {
+    return;
+  }
+  for (var key in customConfig) {
+    config[key] = customConfig[key];
+  }
+};

controllers/registry/module.js

@@ -23,6 +23,9 @@ var utility = require('utility');
 var eventproxy = require('eventproxy');
 var Bagpipe = require('bagpipe');
 var urlparse = require('url').parse;
+var mime = require('mime');
+var semver = require('semver');
+var ms = require('ms');
 var config = require('../../config');
 var Module = require('../../proxy/module');
 var Total = require('../../proxy/total');
@@ -32,7 +35,6 @@ var Log = require('../../proxy/module_log');
 var DownloadTotal = require('../../proxy/download');
 var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var logger = require('../../common/logger');
-var semver = require('semver');
 
 exports.show = function (req, res, next) {
   var name = req.params.name;
@@ -83,7 +85,7 @@ exports.show = function (req, res, next) {
         continue;
       }
       var pkg = row.package;
-      common.downloadURL(pkg, req);
+      common.setDownloadURL(pkg, req);
       versions[pkg.version] = pkg;
       times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
       if ((!distTags.latest && !latestMod) || distTags.latest === row.version) {
@@ -129,7 +131,6 @@ exports.get = function (req, res, next) {
   var name = req.params.name;
   var tag = req.params.version;
   var version = semver.valid(tag);
-
   var ep = eventproxy.create();
   ep.fail(next);
 
@@ -138,7 +139,7 @@ exports.get = function (req, res, next) {
 
   Module[method](name, queryLabel, ep.done(function (mod) {
     if (mod) {
-      common.downloadURL(mod.package, req);
+      common.setDownloadURL(mod.package, req);
       return res.json(mod.package);
     }
     ep.emit('notFound');
@@ -168,14 +169,86 @@ exports.get = function (req, res, next) {
 
 var _downloads = {};
 
+var DOWNLOAD_TIMEOUT = ms('10m');
+
 exports.download = function (req, res, next) {
   var name = req.params.name;
   var filename = req.params.filename;
-  var cdnurl = nfs.url(common.getCDNKey(name, filename));
-  res.statusCode = 302;
-  res.setHeader('Location', cdnurl);
-  res.end();
-  _downloads[name] = (_downloads[name] || 0) + 1;
+  var version = filename.slice(name.length + 1, -4);
+  var ep = eventproxy.create();
+  ep.fail(next);
+
+  Module.get(name, version, ep.doneLater('moduleInfo'));
+  ep.once('moduleInfo', function (row) {
+    if (!row || !row.package || !row.package.dist) {
+      return ep.emit('nodist');
+    }
+    var dist = row.package.dist;
+    if (dist.key) {
+      return ep.emit('key', dist);
+    } else {
+      return ep.emit('url', dist);
+    }
+    ep.emit('nodist');
+  });
+
+  ep.once('nodist', function () {
+    if (!nfs.url) {
+      return next();
+    }
+    ep.emit('url', nfs.url(common.getCDNKey(name, filename)));
+  });
+
+  ep.once('url', function (dist) {
+    res.statusCode = 302;
+    res.setHeader('Location', dist.tarball);
+    res.end();
+    _downloads[name] = (_downloads[name] || 0) + 1;
+  });
+
+  ep.once('key', function (dist) {
+    if (!nfs.downloadStream && !nfs.download) {
+      return next();
+    }
+
+    _downloads[name] = (_downloads[name] || 0) + 1;
+
+    if (typeof dist.size === 'number') {
+      res.setHeader('Content-Length', dist.size);
+    }
+    res.setHeader('Content-Type', mime.lookup(dist.key));
+    res.setHeader('Content-Disposition', 'attachment; filename="' + filename + '"');
+    res.setHeader('ETag', dist.shasum);
+
+    if (nfs.downloadStream) {
+      nfs.downloadStream(dist.key, res, {timeout: DOWNLOAD_TIMEOUT},
+      function (err) {
+        if (err) {
+          // TODO: just end or send error response?
+          return next(err);
+        }
+      });
+      return;
+    }
+
+    // use download file api
+    var tmpPath = path.join(config.uploadDir, utility.randomString() + dist.key);
+    function cleanup() {
+      fs.unlink(tmpPath, utility.noop);
+    }
+
+    nfs.download(dist.key, tmpPath, {timeout: DOWNLOAD_TIMEOUT},
+    function (err) {
+      if (err) {
+        cleanup();
+        return next(err);
+      }
+      var tarball = fs.createReadStream(tmpPath);
+      tarball.on('error', cleanup);
+      tarball.on('end', cleanup);
+      tarball.pipe(res);
+    });
+  });
 };
 
 setInterval(function () {
@@ -276,8 +349,13 @@ exports.upload = function (req, res, next) {
         });
       }
       shasum = shasum.digest('hex');
-      var key = common.getCDNKey(name, filename);
-      nfs.upload(filepath, {key: key, size: length}, function (err, result) {
+
+      var options = {
+        key: common.getCDNKey(name, filename),
+        size: length,
+        shasum: shasum
+      };
+      nfs.upload(filepath, options, function (err, result) {
         // remove tmp file whatever
         fs.unlink(filepath, utility.noop);
         if (err) {
@@ -285,10 +363,18 @@ exports.upload = function (req, res, next) {
         }
 
         var dist = {
-          tarball: result.url,
           shasum: shasum,
           size: length
         };
+
+        // if nfs upload return a key, record it
+        if (result.url) {
+          dist.tarball = result.url;
+        } else if (result.key) {
+          dist.key = result.key;
+          dist.tarball = result.key;
+        }
+
         mod.package.dist = dist;
         mod.package.version = version;
         debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
@@ -355,6 +441,7 @@ exports.updateLatest = function (req, res, next) {
     // change latest to version
     Module.update(nextMod, function (err) {
       if (err) {
+        debug('update nextMod %s error: %s', name, err);
         return next(err);
       }
       // set latest tag
@@ -375,14 +462,136 @@ exports.updateLatest = function (req, res, next) {
   });
 };
 
+exports.addPackageAndDist = function (req, res, next) {
+  // 'dist-tags': { latest: '0.0.2' },
+  //  _attachments:
+  // { 'nae-sandbox-0.0.2.tgz':
+  //    { content_type: 'application/octet-stream',
+  //      data: 'H4sIAAAAA
+  //      length: 9883
+  var pkg = req.body;
+  var username = req.session.name;
+  var name = req.params.name;
+  var filename = Object.keys(pkg._attachments)[0];
+  var attachment = pkg._attachments[filename];
+  var version = filename.match(/\-([\.\d\w\_]+?)\.tgz$/);
+  if (!version) {
+    return res.json(403, {
+      error: 'version_error',
+      reason: filename + ' version not found',
+    });
+  }
+  version = version[1];
+  var versionPackage = pkg.versions[version];
+  versionPackage._publish_on_cnpm = true;
+  var distTags = pkg['dist-tags'] || {};
+  var tags = []; // tag, version
+  for (var t in distTags) {
+    tags.push([t, distTags[t]]);
+  }
+
+  debug('addPackageAndDist %s:%s, attachment size: %s', name, version, attachment.length);
+
+  var ep = eventproxy.create();
+  ep.fail(next);
+  Module.get(name, version, ep.done('exists'));
+
+  var shasum;
+  ep.on('exists', function (exists) {
+    if (exists) {
+      return res.json(409, {
+        error: 'conflict',
+        reason: 'Document update conflict.'
+      });
+    }
+
+    // upload attachment
+    var tarballBuffer;
+    try {
+      tarballBuffer = new Buffer(attachment.data, 'base64');
+    } catch (e) {
+      return next(e);
+    }
+
+    if (tarballBuffer.length !== attachment.length) {
+      return res.json(403, {
+        error: 'size_wrong',
+        reason: 'Attachment size ' + attachment.length + ' not match download size ' + tarballBuffer.length,
+      });
+    }
+
+    shasum = crypto.createHash('sha1');
+    shasum.update(tarballBuffer);
+    shasum = shasum.digest('hex');
+
+    var options = {
+      key: common.getCDNKey(name, filename),
+      shasum: shasum
+    };
+    nfs.uploadBuffer(tarballBuffer, options, ep.done('upload'));
+  });
+
+  ep.on('upload', function (result) {
+    debug('upload %j', result);
+
+    var dist = {
+      shasum: shasum,
+      size: attachment.length
+    };
+
+    // if nfs upload return a key, record it
+    if (result.url) {
+      dist.tarball = result.url;
+    } else if (result.key) {
+      dist.key = result.key;
+      dist.tarball = result.key;
+    }
+
+    var mod = {
+      name: name,
+      version: version,
+      author: username,
+      package: versionPackage
+    };
+
+    mod.package.dist = dist;
+
+    Module.add(mod, ep.done(function (r) {
+      debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
+        r.id, dist.tarball, dist.size, shasum, dist, version);
+      ep.emit('saveModule', r.id);
+    }));
+  });
+
+  ep.on('saveModule', function () {
+    if (tags.length === 0) {
+      return ep.emit('saveTags');
+    }
+
+    tags.forEach(function (item) {
+      Module.addTag(name, item[0], item[1], ep.done('saveTag'));
+    });
+    ep.after('saveTag', tags.length, function () {
+      ep.emit('saveTags');
+    });
+  });
+
+  ep.all('saveModule', 'saveTags', function (moduleId) {
+    res.json(201, {ok: true, rev: String(moduleId)});
+  });
+};
+
 exports.add = function (req, res, next) {
   var username = req.session.name;
   var name = req.params.name;
-  var pkg = req.body;
+  var pkg = req.body || {};
   var maintainers = pkg.maintainers || [];
   var match = maintainers.filter(function (item) {
     return item.name === username;
   });
+
+  debug('add module %s maintainers match: %j, current user: %s', name, match, username);
+
   if (match.length === 0) {
     return res.json(403, {
       error: 'no_perms',
@@ -390,6 +599,10 @@ exports.add = function (req, res, next) {
     });
   }
 
+  if (pkg._attachments && Object.keys(pkg._attachments).length > 0) {
+    return exports.addPackageAndDist(req, res, next);
+  }
+
   var ep = eventproxy.create();
   ep.fail(next);
 
@@ -413,6 +626,7 @@ exports.add = function (req, res, next) {
         maintainers: pkg.maintainers,
       },
     };
+    debug('add next module: %s', name);
     Module.add(nextMod, ep.done(function (result) {
       nextMod.id = result.id;
       ep.emit('next', nextMod);
@@ -420,7 +634,8 @@ exports.add = function (req, res, next) {
   }));
 
   ep.all('latest', 'next', function (latestMod, nextMod) {
-    var maintainers = latestMod ? latestMod.package.maintainers : nextMod.package.maintainers;
+    var maintainers = latestMod && latestMod.package.maintainers.length > 0 ?
+      latestMod.package.maintainers : nextMod.package.maintainers;
     var match = maintainers.filter(function (item) {
       return item.name === username;
     });
@@ -432,7 +647,9 @@ exports.add = function (req, res, next) {
       });
     }
 
-    if (latestMod || nextMod.exists) {
+    debug('add %s rev: %s, version: %s', name, nextMod.id, nextMod.version);
+
+    if (latestMod || nextMod.version !== 'next') {
       return res.json(409, {
         error: 'conflict',
         reason: 'Document update conflict.'
@@ -513,8 +730,9 @@ exports.removeTar = function (req, res, next) {
         reason: 'Current user can not delete this tarball'
       });
     }
+    var key = mod.package.dist && mod.package.dist.key;
+    key = key || common.getCDNKey(mod.name, filename);
 
-    var key = common.getCDNKey(mod.name, filename);
     nfs.remove(key, ep.done(function () {
       res.json(200, {ok: true});
     }));
@@ -532,6 +750,7 @@ exports.removeAll = function (req, res, next) {
 
   Module.listByName(name, ep.doneLater('list'));
   ep.once('list', function (mods) {
+    debug('removeAll module %s: %d', name, mods.length);
     var mod = mods[0];
     if (!mod) {
       return next();
@@ -540,6 +759,10 @@ exports.removeAll = function (req, res, next) {
     var match = mod.package.maintainers.filter(function (item) {
       return item.name === username;
     });
+    if (req.session.isAdmin) {
+      match.push({name: username});
+    }
+
     if (!match.length || mod.name !== name) {
       return res.json(403, {
         error: 'no_perms',
@@ -559,7 +782,7 @@ exports.removeAll = function (req, res, next) {
     }
     var queue = new Bagpipe(5);
     keys.forEach(function (key) {
-      queue.push(nfs.remove, key, function () {
+      queue.push(nfs.remove.bind(nfs), key, function () {
         //ignore err here
         ep.emit('removeTar');
       });
@@ -587,7 +810,7 @@ function parseModsForList(updated, mods, req) {
     pkg['dist-tags'] = {
       latest: pkg.version
     };
-    common.downloadURL(pkg, req);
+    common.setDownloadURL(pkg, req);
     results[mod.name] = pkg;
   }
   return results;

controllers/sync.js

@@ -13,13 +13,27 @@
 /**
  * Module dependencies.
  */
+
 var Log = require('../proxy/module_log');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 
 exports.sync = function (req, res, next) {
   var username = req.session.name || 'anonymous';
   var name = req.params.name;
-  SyncModuleWorker.sync(name, username, function (err, result) {
+  var publish = req.query.publish === 'true';
+  var noDep = req.query.nodeps === 'true';
+  if (publish && !req.session.isAdmin) {
+    return res.json(403, {
+      error: 'no_perms',
+      reason: 'Only admin can publish'
+    });
+  }
+
+  var options = {
+    publish: publish,
+    noDep: noDep,
+  };
+  SyncModuleWorker.sync(name, username, options, function (err, result) {
     if (err) {
       return next(err);
     }

controllers/web/package.js

@@ -20,10 +20,12 @@ var semver = require('semver');
 var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
+var config = require('../../config');
 var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var setDownloadURL = require('../../lib/common').setDownloadURL;
 
 exports.display = function (req, res, next) {
   var params = req.params;
@@ -62,12 +64,30 @@ exports.display = function (req, res, next) {
       }
     }
 
+    if (pkg.contributors) {
+      // registry.cnpmjs.org/compressible
+      if (!Array.isArray(pkg.contributors)) {
+        pkg.contributors = [pkg.contributors];
+      }
+      for (var i = 0; i < pkg.contributors.length; i++) {
+        var contributor = pkg.contributors[i];
+        if (contributor.email) {
+          contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
+        }
+        if (config.packagePageContributorSearch || !contributor.url) {
+          contributor.url = '/~' + encodeURIComponent(contributor.name);
+        }
+      }
+    }
+
     setLicense(pkg);
 
     for (var k in download) {
       download[k] = humanize(download[k]);
     }
 
+    setDownloadURL(pkg, req, config.registryHost);
+
     res.render('package', {
       title: 'Package - ' + pkg.name,
       package: pkg,
@@ -92,8 +112,41 @@ exports.search = function (req, res, next) {
   });
 };
 
+exports.rangeSearch = function (req, res, next) {
+  var startKey = req.query.startkey || '';
+  if (startKey[0] === '"') {
+    startKey = startKey.substring(1);
+  }
+  if (startKey[startKey.length - 1] === '"') {
+    startKey = startKey.substring(0, startKey.length - 1);
+  }
+  var limit = Number(req.query.limit) || 20;
+  Module.search(startKey, {limit: limit}, function (err, packages) {
+    if (err) {
+      return next(err);
+    }
+
+    var rows = [];
+    for (var i = 0; i < packages.length; i++) {
+      var p = packages[i];
+      var row = {
+        key: p.name,
+        count: 1,
+        value: {
+          name: p.name,
+          description: p.description,
+        }
+      };
+      rows.push(row);
+    }
+    res.json({
+      rows: rows
+    });
+  });
+};
+
 exports.displaySync = function (req, res, next) {
-  var name = req.params.name;
+  var name = req.params.name || req.query.name;
   res.render('sync', {
     name: name,
     title: 'Sync - ' + name

docs/db.sql

@@ -19,7 +19,7 @@ CREATE TABLE `module` (
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `author` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `description` longtext,
  `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
@@ -38,23 +38,25 @@ CREATE TABLE `module` (
 -- ALTER TABLE `module` CHANGE `package` `package` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- ALTER TABLE `module` CHANGE `description` `description` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- show create table module\G
+-- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `module_log` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `username` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `log` longtext,
  PRIMARY KEY (`id`),
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+-- ALTER TABLE  `module_log` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `tag` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `tag` varchar(30) NOT NULL COMMENT 'tag name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
@@ -62,6 +64,7 @@ CREATE TABLE `tag` (
  UNIQUE KEY `name` (`name`, `tag`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
 -- ALTER TABLE  `tag` ADD  `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
+-- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
 
 CREATE TABLE `total` (
  `name` varchar(100) NOT NULL COMMENT 'total name',
@@ -77,7 +80,7 @@ CREATE TABLE `total` (
  `last_sync_module` varchar(100) COMMENT 'last sync success module name',
  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
-INSERT INTO total(name, gmt_modified, module_delete) VALUES('total', now(), 0, 0);
+INSERT INTO total(name, gmt_modified) VALUES('total', now());
 -- ALTER TABLE `total` ADD `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry'
 -- ALTER TABLE `total` ADD `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry'
 -- ALTER TABLE `total` ADD `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status'
@@ -92,8 +95,9 @@ CREATE TABLE `download_total` (
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
  PRIMARY KEY (`id`),
  UNIQUE KEY `date_name` (`date`, `name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';

docs/web/install.md

@@ -91,23 +91,25 @@ $ open http://localhost:7001
 $ open http://localhost:7002
 ```
 
-## use cnpm cli with your own registry  
-You do not need to write another command line tool with your own registry, 
+## use cnpm cli with your own registry
+You do not need to write another command line tool with your own registry,
 just alias [cnpm](http://github.com/fengmk2/cnpm), then you can get a npm client for you own registry.
 
 ```
 # install cnpm first
-npm install -g cnpm 
+npm install -g cnpm
 
 # then alias lnpm to cnpm, but change config to your own registry
 alias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://dist.u.qiniudn.com\
  --userconfig=$HOME/.lnpmrc'
 
  #or put this in .zshrc or .bashrc
  echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
  --registryweb=http://localhost:7002\
  --cache=$HOME/.npm/.cache/lnpm\
+ --disturl=http://dist.u.qiniudn.com\
  --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
 ```

docs/web/readme.md

@@ -1,12 +1,12 @@
 # cnpmjs.org: Private npm registry and web for Enterprise
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org) [![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/cnpm/cnpmjs.org/badge.png)](https://coveralls.io/r/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
 
 [![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
 ## What is this?
 
-> Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+> Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
@@ -16,7 +16,7 @@
 
 ## Registry
 
-* Our public registry: [registry.cnpmjs.org](http://registry.cnpmjs.org)
+* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
 * Current [cnpmjs.org](/) version: <span id="app-version"></span>
 
 <table class="downloads">
@@ -119,13 +119,15 @@ $(function () {
 alias it:
 
 ```bash
-alias cnpm="npm --registry=http://registry.cnpmjs.org \
+alias cnpm="npm --registry=http://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
+--disturl=http://dist.u.qiniudn.com \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
+  --disturl=http://dist.u.qiniudn.com \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -137,7 +139,7 @@ $ npm install cnpm -g
 
 ### install
 
-Install package from [registry.cnpmjs.org](http://registry.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
 ```
 $ cnpm install [name]
@@ -166,12 +168,17 @@ $ cnpm publish [name]
 $ cnpm unpublish [name]
 ```
 
-### Orther command
-Support all the other npm command.
+### Other commands
+
+Support all the other npm commands. e.g.:
+
+```bash
+$ cnpm info cnpm
+```
 
 ## TODO list
 
-@see Github [Issues](https://github.com/fengmk2/cnpmjs.org/issues)
+@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
 
 ## Authors
 
@@ -181,14 +188,14 @@ Release [History](/history).
 $ git summary
 
  project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
+ repo age : 5 weeks
+ active   : 98 days
+ commits  : 239
+ files    : 85
  authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
+   140  fengmk2                 58.6%
+    98  dead_horse              41.0%
+     1  Alsotang                0.4%
 ```
 
 ## npm and cnpm relation

index.js

@@ -0,0 +1,30 @@
+/*!
+ * cnpmjs.org - index.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('./config');
+
+exports.loadConfig = config.loadConfig;
+exports.config = config;
+
+exports.startWorker = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./worker');
+};
+
+exports.startSync = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./sync');
+};

lib/common.js

@@ -17,6 +17,7 @@
 var crypto = require('crypto');
 var path = require('path');
 var config = require('../config');
+var util = require('util');
 
 exports.getTarballFilepath = function (filename) {
   // ensure download file path unique
@@ -28,8 +29,15 @@ exports.getCDNKey = function (name, filename) {
   return '/' + name + '/-/' + filename;
 };
 
-exports.downloadURL = function (pkg, req) {
-  if (pkg.dist && pkg.dist.tarball) {
-    pkg.dist.tarball = 'http://' + req.headers.host + '/' + pkg.name + '/download/' + path.basename(pkg.dist.tarball);
+exports.setDownloadURL = function (pkg, req, host) {
+  if (pkg.dist) {
+    host = host || req.headers.host;
+    pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
+      req.connection.encrypted ? 'https' : 'http',
+      host, pkg.name, pkg.name, pkg.version);
   }
 };
+
+exports.isAdmin = function (username) {
+  return typeof config.admins[username] === 'string';
+};

middleware/auth.js

@@ -17,14 +17,13 @@
 var debug = require('debug')('cnpmjs.org:middleware:auth');
 var User = require('../proxy/user');
 var config = require('../config');
+var common = require('../lib/common');
 
 module.exports = function (options) {
   return function auth(req, res, next) {
     req.session.onlySync = config.enablePrivate ? true : false;
     if (req.session.name) {
-      if (config.admins[req.session.name]) {
-        req.session.isAdmin = true;
-      }
+      req.session.isAdmin = common.isAdmin(req.session.name);
       debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
         req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
       return next();
@@ -52,9 +51,7 @@ module.exports = function (options) {
       }
 
       req.session.name = row.name;
-      if (config.admins.hasOwnProperty(req.session.name)) {
-        req.session.isAdmin = true;
-      }
+      req.session.isAdmin = common.isAdmin(req.session.name);
       debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
         row, req.session.onlySync, req.session.isAdmin, req.headers);
       next();

middleware/sync_by_install.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:middleware:sync_by_install');
 var config = require('../config');
 
 /**
@@ -36,5 +37,9 @@ module.exports = function (req, res, next) {
     req.session.allowSync = false;
   }
 
+  // TODO: allow sync will let publish sync package...
+  req.session.allowSync = false;
+
+  debug('%s allowSync: %s', req.session.name, req.session.allowSync);
   next();
 };

package.json

@@ -1,13 +1,15 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.1.4",
-  "description": "Private npm registry and web",
-  "main": "dispatch.js",
+  "version": "0.2.13",
+  "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
+  "main": "index.js",
   "scripts": {
     "test": "make test-all",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
-    "stop": "./bin/nodejsctl stop",
+    "stop": "./bin/nodejsctl stop"
+  },
+  "config": {
     "blanket": {
       "pattern": "//^((?!(node_modules|test|common)).)*$/",
       "data-cover-flags": {
@@ -19,54 +21,56 @@
     }
   },
   "dependencies": {
-    "forward": ">=0.0.4",
-    "humanize-number": ">=0.0.2",
-    "gravatar": "1.0.6",
-    "urllib": ">=0.5.5",
-    "connect-markdown": "0.0.3",
-    "qn": ">=0.2.0",
-    "microtime": "0.5.1",
-    "debug": ">=0.7.4",
-    "utility": ">=0.1.9",
-    "ready": "0.1.1",
+    "bagpipe": "0.3.5",
     "connect": "2.12.0",
-    "connect-rt": "0.0.2",
+    "connect-markdown": "0.0.3",
     "connect-redis": "1.4.6",
     "connect-render": "0.3.2",
-    "urlrouter": ">=0.5.4",
-    "graceful": ">=0.0.5",
-    "moment": "2.4.0",
-    "logfilestream": ">=0.1.0",
-    "ms": "0.6.2",
+    "connect-rt": "0.0.2",
+    "debug": "0.7.4",
+    "eventproxy": "0.2.6",
+    "forward": "0.0.4",
+    "graceful": "0.0.5",
+    "gravatar": "1.0.6",
+    "humanize-number": "0.0.2",
+    "logfilestream": "0.1.0",
+    "marked": "0.3.0",
+    "microtime": "0.5.1",
+    "mime": "1.2.11",
     "mkdirp": "0.3.5",
-    "mysql": "2.0.0-rc2",
-    "response-patch": "0.1.1",
+    "moment": "2.5.0",
+    "ms": "0.6.2",
+    "mysql": "2.0.0",
+    "nodemailer": "0.6.0",
+    "qn": "0.2.0",
+    "ready": "0.1.1",
     "response-cookie": "0.0.2",
-    "bagpipe": "0.3.5",
+    "response-patch": "0.1.1",
     "semver": "2.2.1",
-    "marked": "0.2.10",
-    "nodemailer": "0.5.15",
-    "eventproxy": ">=0.2.6"
+    "urllib": "0.5.5",
+    "urlrouter": "0.5.4",
+    "utility": "0.1.10"
   },
   "devDependencies": {
-    "supertest": "*",
-    "should": "*",
     "blanket": "*",
-    "travis-cov": "*",
+    "contributors": "*",
     "coveralls": "*",
-    "mocha-lcov-reporter": "*",
+    "mm": "0.1.8",
     "mocha": "*",
-    "mm": "*",
-    "pedding": "*"
+    "mocha-lcov-reporter": "*",
+    "pedding": "0.0.3",
+    "should": "2.1.1",
+    "supertest": "0.8.3",
+    "travis-cov": "*"
   },
-  "homepage": "https://github.com/fengmk2/cnpmjs.org",
+  "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {
     "type": "git",
-    "url": "git://github.com/fengmk2/cnpmjs.org.git",
-    "web": "https://github.com/fengmk2/cnpmjs.org"
+    "url": "git://github.com/cnpm/cnpmjs.org.git",
+    "web": "https://github.com/cnpm/cnpmjs.org"
   },
   "bugs": {
-    "url": "https://github.com/fengmk2/cnpmjs.org/issues",
+    "url": "https://github.com/cnpm/cnpmjs.org/issues",
     "email": "fengmk2@gmail.com"
   },
   "keywords": [

proxy/module.js

@@ -296,14 +296,20 @@ exports.listByAuthor = function (author, callback) {
 };
 
 var SEARCH_SQLS = [
-  'SELECT module_id FROM tag WHERE name LIKE ? AND  tag="latest" ORDER BY name LIMIT 100;',
+  'SELECT module_id FROM tag WHERE name LIKE ? AND  tag="latest" ORDER BY name LIMIT ?;',
   'SELECT name, description FROM module WHERE id IN (?) ORDER BY name;'
 ];
-exports.search = function (word, callback) {
+exports.search = function (word, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
+  var limit = options.limit || 100;
   word = word.replace(/^%/, '') + '%'; //ignore prefix %
   var ep = eventproxy.create();
   ep.fail(callback);
-  mysql.query(SEARCH_SQLS[0], [word], ep.done(function (rows) {
+  mysql.query(SEARCH_SQLS[0], [word, limit], ep.done(function (rows) {
     if (!rows || rows.length === 0) {
       return callback(null, []);
     }

proxy/npm.js

@@ -20,11 +20,9 @@ var config = require('../config');
 function request(url, options, callback) {
   if (typeof options === 'function') {
     callback = options;
-    options = {
-      dataType: 'json',
-      timeout: 10000
-    };
+    options = null;
   }
+  options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
   url = config.sourceNpmRegistry + url;

proxy/sync_module_worker.js

@@ -40,13 +40,15 @@ function SyncModuleWorker(options) {
   }
 
   this.names = options.name;
-  for (var i = 0; i < this.names.length; i++) {
-    // ensure package name is lower case
-    this.names[i] = this.names[i].toLowerCase();
-  }
+  // for (var i = 0; i < this.names.length; i++) {
+  //   // ensure package name is lower case
+  //   this.names[i] = this.names[i].toLowerCase();
+  // }
 
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
+  this._publish = options.publish; // _publish_on_cnpm
+
   this.syncingNames = {};
   this.nameMap = {};
   this.names.forEach(function (name) {
@@ -82,7 +84,8 @@ SyncModuleWorker.prototype.log = function (format, arg1, arg2) {
 };
 
 SyncModuleWorker.prototype.start = function () {
-  this.log('user: %s, sync %s worker start, %d concurrency.', this.username, this.names[0], this.concurrency);
+  this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s',
+    this.username, this.names[0], this.concurrency, this.noDep, this._publish);
   for (var i = 0; i < this.concurrency; i++) {
     this.next(i);
   }
@@ -106,22 +109,27 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
   var that = this;
   that.syncingNames[name] = true;
   npm.get(name, function (err, pkg, response) {
+    var statusCode = response && response.statusCode || -1;
     if (!err && !pkg) {
-      err = new Error('Module ' + name + ' not exists, http status ' + response.statusCode);
+      err = new Error('Module ' + name + ' not exists, http status ' + statusCode);
       err.name = 'NpmModuleNotExsitsError';
     }
     if (err) {
-      that.fails.push(name);
+      if (statusCode === 404) {
+        that.successes.push(name);
+      } else {
+        that.fails.push(name);
+      }
       that.log('[error] [%s] get package error: %s', name, err.stack);
       delete that.syncingNames[name];
       return that.next(concurrencyId);
     }
 
-    that.log('[c#%d] [%s] start...', concurrencyId, pkg.name);
-    that._sync(pkg, function (err, versions) {
+    that.log('[c#%d] [%s] start...', concurrencyId, name);
+    that._sync(name, pkg, function (err, versions) {
       delete that.syncingNames[name];
       if (err) {
-        that.fails.push(pkg.name);
+        that.fails.push(name);
         that.log('[error] [%s] sync error: %s', name, err.stack);
         return that.next(concurrencyId);
       }
@@ -134,9 +142,8 @@ SyncModuleWorker.prototype.next = function (concurrencyId) {
   });
 };
 
-SyncModuleWorker.prototype._sync = function (pkg, callback) {
-  var username = this.name;
-  var name = pkg.name;
+SyncModuleWorker.prototype._sync = function (name, pkg, callback) {
+  var username = this.username;
   var that = this;
   var ep = eventproxy.create();
   ep.fail(callback);
@@ -154,7 +161,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
 
       if (r.package && r.package._publish_on_cnpm) {
         // publish on cnpm, dont sync this version package
-        that.log('  [%s] publish on local cnpm, don\'t sync.', pkg.name);
+        that.log('  [%s] publish on local cnpm, don\'t sync', name);
         ep.unbind();
         callback(null, []);
         return;
@@ -195,7 +202,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       versionNames = Object.keys(pkg.versions);
     }
     if (versionNames.length === 0) {
-      that.log('  [%s] no times and no versions, hasModules: %s', pkg.name, hasModules);
+      that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
       if (!hasModules) {
         // save a next module
         var maintainer = pkg.maintainers && pkg.maintainers[0];
@@ -206,11 +213,11 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
           maintainer = '-';
         }
         var nextMod = {
-          name: pkg.name,
+          name: name,
           version: 'next',
           author: maintainer,
           package: {
-            name: pkg.name,
+            name: name,
             version: 'next',
             description: pkg.description || '',
             readme: pkg.readme || '',
@@ -220,7 +227,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
           },
         };
         Module.add(nextMod, function (err, result) {
-          that.log('  [%s] save next module, %j, error: %s', pkg.name, result, err);
+          that.log('  [%s] save next module, %j, error: %s', name, result, err);
         });
       }
     }
@@ -245,7 +252,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
         // * shasum make sure equal
         if ((version.publish_time === exists.publish_time) || (!version.publish_time && exists.publish_time)) {
           debug('  [%s] %s publish_time equal: %s, %s',
-            pkg.name, version.version, version.publish_time, exists.publish_time);
+            name, version.version, version.publish_time, exists.publish_time);
           // * publish_time make sure equal
           if (exists.description === null && version.description) {
             // * make sure description exists
@@ -269,7 +276,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
     }
 
     if (versions.length === 0) {
-      that.log('  [%s] all versions are exists', pkg.name);
+      that.log('  [%s] all versions are exists', name);
       return ep.emit('syncDone');
     }
 
@@ -277,7 +284,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return a.publish_time - b.publish_time;
     });
     missingVersions = versions;
-    that.log('  [%s] %d versions', pkg.name, versions.length);
+    that.log('  [%s] %d versions', name, versions.length);
     ep.emit('syncModule', missingVersions.shift());
   });
 
@@ -306,7 +313,7 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return ep.emit('descriptionDone');
     }
 
-    that.log('  [%s] saving %d descriptions', pkg.name, missingDescriptions.length);
+    that.log('  [%s] saving %d descriptions', name, missingDescriptions.length);
     missingDescriptions.forEach(function (item) {
       Module.updateDescription(item.id, item.description, function (err, result) {
         if (err) {
@@ -327,10 +334,10 @@ SyncModuleWorker.prototype._sync = function (pkg, callback) {
       return ep.emit('tagDone');
     }
 
-    that.log('  [%s] adding %d tags', pkg.name, missingTags.length);
+    that.log('  [%s] adding %d tags', name, missingTags.length);
     // sync tags
     missingTags.forEach(function (item) {
-      Module.addTag(pkg.name, item[0], item[1], ep.done(function (result) {
+      Module.addTag(name, item[0], item[1], ep.done(function (result) {
         that.log('    added tag %s:%s, module_id: %s', item[0], item[1], result && result.module_id);
         ep.emit('addTag');
       }));
@@ -366,8 +373,9 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
     callback(err);
   });
 
-  that.log('    [%s:%d] syncing, version: %s, dist: %j',
-    sourcePackage.name, versionIndex, sourcePackage.version, sourcePackage.dist);
+  that.log('    [%s:%d] syncing, version: %s, dist: %j, no deps: %s, publish on cnpm: %s',
+    sourcePackage.name, versionIndex, sourcePackage.version,
+    sourcePackage.dist, that.noDep, that._publish);
   if (!that.noDep) {
     for (var k in sourcePackage.dependencies) {
       that.add(k);
@@ -414,8 +422,12 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
         return ep.emit('error', err);
       }
 
-      var key = common.getCDNKey(sourcePackage.name, filename);
-      nfs.upload(filepath, {key: key, size: dataSize}, ep.done('uploadResult'));
+      var options = {
+        key: common.getCDNKey(sourcePackage.name, filename),
+        size: dataSize,
+        shasum: shasum
+      };
+      nfs.upload(filepath, options, ep.done('uploadResult'));
     });
   }));
 
@@ -437,24 +449,44 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
       author: author,
       publish_time: sourcePackage.publish_time,
     };
+
+    if (that._publish) {
+      // sync as publish
+      mod.package._publish_on_cnpm = true;
+    }
+
     var dist = {
-      tarball: result.url,
       shasum: shasum,
       size: dataSize,
       noattachment: dataSize === 0,
     };
+
+    if (result.url) {
+      dist.tarball = result.url;
+    } else if (result.key) {
+      dist.key = result.key;
+      dist.tarball = result.key;
+    }
+
     mod.package.dist = dist;
     Module.add(mod, ep.done(function (result) {
-      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j',
+      that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, size: %d, publish_time: %j, publish on cnpm: %s',
         sourcePackage.name, versionIndex,
         result.id,
-        author, mod.version, dataSize, new Date(mod.publish_time));
+        author, mod.version, dataSize,
+        new Date(mod.publish_time),
+        that._publish);
       callback(null, result);
     }));
   });
 };
 
-SyncModuleWorker.sync = function (name, username, callback) {
+SyncModuleWorker.sync = function (name, username, options, callback) {
+  if (typeof options === 'function') {
+    callback = options;
+    options = null;
+  }
+  options = options || {};
   npm.get(name, function (err, pkg, response) {
     if (err) {
       return callback(err);
@@ -474,6 +506,8 @@ SyncModuleWorker.sync = function (name, username, callback) {
         logId: result.id,
         name: name,
         username: username,
+        noDep: options.noDep,
+        publish: options.publish,
       });
       worker.start();
       callback(null, {

routes/web.js

@@ -31,6 +31,9 @@ function routes(app) {
   app.get('/sync/:name', pkg.displaySync);
   app.put('/sync/:name', sync.sync);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
+  app.get('/sync', pkg.displaySync);
+
+  app.get('/_list/search/search', pkg.rangeSearch);
 }
 
 module.exports = routes;

servers/registry.js

@@ -65,9 +65,9 @@ app.use(function (req, res, next) {
 
 app.use(function (err, req, res, next) {
   err.url = err.url || req.url;
+  logger.error(err);
   if (process.env.NODE_ENV !== 'test') {
     console.error(err.stack);
-    logger.error(err);
   }
   if (config.debug) {
     return next(err);

servers/web.js

@@ -18,6 +18,7 @@
 require('response-patch');
 var path = require('path');
 var http = require('http');
+var fs = require('fs');
 var connect = require('connect');
 var rt = require('connect-rt');
 var urlrouter = require('urlrouter');
@@ -49,9 +50,14 @@ app.use(connect.json());
 var viewDir = path.join(rootdir, 'view', 'web');
 var docDir = path.join(rootdir, 'docs', 'web');
 
+var layoutFile = path.join(viewDir, '_layout.html');
+var footer = config.customFooter || fs.readFileSync(path.join(viewDir, 'footer.html'), 'utf8');
+var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8').replace('{{footer}}', footer);
+fs.writeFileSync(layoutFile, layout);
+
 app.use('/', connectMarkdown({
   root: docDir,
-  layout: path.join(viewDir, 'layout.html'),
+  layout: layoutFile,
   titleHolder: '<%- locals.title %>',
   bodyHolder: '<%- locals.body %>',
   indexName: 'readme'
@@ -81,8 +87,10 @@ app.use(urlrouter(routes));
 
 app.use(function (err, req, res, next) {
   err.url = err.url || req.url;
-  console.error(err.stack);
   logger.error(err);
+  if (process.env.NODE_ENV !== 'test') {
+    console.error(err.stack);
+  }
   if (config.debug) {
     return next(err);
   }

sync/sync_all.js

@@ -31,13 +31,11 @@ function subtract(subtracter, minuend) {
   var map = {};
   var results = [];
   minuend.forEach(function (name) {
-    map[name.toLowerCase()] = true;
+    map[name] = true;
   });
   subtracter.forEach(function (name) {
-    var lowerName = name.toLowerCase();
-    if (!map[lowerName] && !/[A-Z]/.test(name)) {
-      // ensure package name is lower case
-      results.push(lowerName);
+    if (!map[name]) {
+      results.push(name);
     }
   });
   return results;

test/controllers/registry/module.test.js

@@ -88,14 +88,14 @@ describe('controllers/registry/module.test.js', function () {
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.0.0')
+      .get('/cnpmjs.org/0.2.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.0.0');
-        body._id.should.equal('cnpmjs.org@0.0.0');
-        body.dist.tarball.should.include('cnpmjs.org-0.0.0.tgz');
+        body.version.should.equal('0.2.1');
+        body._id.should.equal('cnpmjs.org@0.2.1');
+        body.dist.tarball.should.include('cnpmjs.org-0.2.1.tgz');
         done();
       });
     });
@@ -114,14 +114,14 @@ describe('controllers/registry/module.test.js', function () {
       });
     });
 
-    it('should get cnpmjs.org@0.1.0 with _publish_on_cnpm=true', function (done) {
+    it('should get cnpmjs.org@0.2.1 with _publish_on_cnpm=true', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.1.0')
+      .get('/cnpmjs.org/0.2.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
         body.name.should.equal('cnpmjs.org');
-        body.version.should.equal('0.1.0');
+        body.version.should.equal('0.2.1');
         body._publish_on_cnpm.should.equal(true);
         done();
       });
@@ -172,14 +172,7 @@ describe('controllers/registry/module.test.js', function () {
       .put('/' + pkg.name)
       .set('authorization', baseauth)
       .send(pkg)
-      .expect(409, function (err, res) {
-        should.not.exist(err);
-        res.body.should.eql({
-          error: 'conflict',
-          reason: 'Document update conflict.'
-        });
-        done();
-      });
+      .expect(201, done);
     });
 
     it('should try to add return 403 when not module user and only next module exists',
@@ -453,6 +446,43 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
+  describe('GET /:name/download/:filename', function () {
+    it('should download a file with 302 redirect', function (done) {
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
+      .expect(302, done)
+    });
+
+    it('should download a file direct from nfs stream', function (done) {
+      var nfs = require('../../../common/nfs');
+      mm(nfs, 'downloadStream', function (key, writeStream, options, callback) {
+        options.timeout.should.equal(600000);
+        nfs._client.download(key, {writeStream: writeStream, timeout: options.timeout}, callback);
+      });
+      Module.__get__ = Module.get;
+      mm(Module, 'get', function (name, version, callback) {
+        Module.__get__(name, version, function (err, info) {
+          info.package.dist.key = 'cutter/-/cutter-0.0.2.tgz';
+          callback(err, info);
+        });
+      });
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('ETag', 'c61fde5e8c26d053574d0c722097029fd1bc963a')
+      .expect('Content-Type', 'application/octet-stream')
+      .expect('Content-Length', '3139')
+      .expect('Content-Disposition', 'attachment; filename="cutter-0.0.2.tgz"')
+      .expect(200)
+      .end(function (err, res) {
+        should.not.exist(err);
+        // TODO: why supertest change buffer to text?
+        // res.text.length.should.equal(3139);
+        done();
+      });
+    });
+  });
+
   describe('DELETE /:name/download/:filename/-rev/:rev', function () {
     var lastRev;
     before(function (done) {

test/controllers/sync.test.js

@@ -40,6 +40,53 @@ describe('controllers/sync.test.js', function () {
   describe('sync source npm package', function () {
     var logIdRegistry;
     var logIdWeb;
+
+    it('should sync as publish success', function (done) {
+      request(registryApp)
+      .del('/utility/-rev/123')
+      .set('authorization', baseauth)
+      // .expect(200)
+      // .expect({ok: true})
+      .end(function (err, res) {
+        should.not.exist(err);
+
+        mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+        request(registryApp)
+        .put('/utility/sync?publish=true&nodeps=true')
+        .set('authorization', baseauth)
+        .end(function (err, res) {
+          should.not.exist(err);
+          res.body.should.have.keys('ok', 'logId');
+          logIdRegistry = res.body.logId;
+          setTimeout(function () {
+            request(registryApp)
+            .get('/utility')
+            .expect(200)
+            .end(function (err, res) {
+              should.not.exist(err);
+              Object.keys(res.body.versions).length.should.above(0);
+              for (var v in res.body.versions) {
+                var pkg = res.body.versions[v];
+                pkg.should.have.property('_publish_on_cnpm', true);
+              }
+              done();
+            });
+          }, 3000);
+        });
+      });
+    });
+
+    it('should sync as publish 403 when user not admin', function (done) {
+      mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
+      request(registryApp)
+      .put('/utility_unit_test/sync?publish=true&nodeps=true')
+      .expect(403)
+      .expect({
+        error: 'no_perms',
+        reason: 'Only admin can publish'
+      }, done);
+    });
+
     it('should sync success', function (done) {
       mm.data(Npm, 'get', require(path.join(fixtures, 'utility.json')));
       done = pedding(2, done);

test/controllers/web/package.test.js

@@ -31,6 +31,48 @@ describe('controllers/web/package.test.js', function () {
 
   afterEach(mm.restore);
 
+  describe('GET /_list/search/search', function () {
+    it('should search with "c"', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="c"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.should.eql([
+          { key: 'c', count: 1, value: { name: 'c', description: 'Give folders or directories comments and view them easy.' } },
+          { key: 'charset', count: 1,
+            value: { name: 'charset', description: 'Get the content charset from header and html content-type.' } }
+        ]);
+        done();
+      });
+    });
+
+    it('should search with c', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey=c&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.have.keys('rows');
+        res.body.rows.should.eql([
+          { key: 'c', count: 1, value: { name: 'c', description: 'Give folders or directories comments and view them easy.' } },
+          { key: 'charset', count: 1,
+            value: { name: 'charset', description: 'Get the content charset from header and html content-type.' } }
+        ]);
+        done();
+      });
+    });
+
+    it('should search return empty', function (done) {
+      request(app)
+      .get('/_list/search/search?startkey="cddddsdasdaasds"&limit=2')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({rows: []});
+        done();
+      });
+    });
+  });
+
   describe('GET /package/:name', function (done) {
     it('should get 200', function (done) {
       request(app)

test/lib/common.test.js

@@ -0,0 +1,29 @@
+/*!
+ * cnpmjs.org - test/lib/common.test.js.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com>
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var common = require('../../lib/common');
+
+describe('lib/common.test.js', function () {
+  describe('isAdmin()', function () {
+    it('should admin is admin', function () {
+      common.isAdmin('admin').should.equal(true);
+      common.isAdmin('fengmk2').should.equal(true);
+      common.isAdmin('constructor').should.equal(false);
+      common.isAdmin('toString').should.equal(false);
+    });
+  });
+});

test/middleware/auth.test.js

@@ -29,8 +29,9 @@ describe('middleware/auth.test.js', function () {
     app.close(done);
   });
 
+  afterEach(mm.restore);
+
   describe('auth()', function () {
-    afterEach(mm.restore);
     it('should pass if no authorization', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:cnpmjstest1')

test/sync.js

@@ -30,6 +30,8 @@ Log.create({
     name: names,
     username: 'fengmk2',
     concurrency: names.length,
+    noDep: true,
+    publish: true,
   });
 
   worker.start();

test/sync/sync_exist.test.js

@@ -44,7 +44,7 @@ describe('sync/sync_exist.js', function () {
         should.not.exist(err);
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });      
+      });
     });
   });
 });

view/web/footer.html

@@ -0,0 +1,5 @@
+Copyright 2013 - 2014 © cnpmjs.org
+|
+<a href="/">Home</a>
+|
+<script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>

view/web/layout.html

@@ -54,13 +54,9 @@
     <div class="bottom">
       <hr/>
       <p>
-        Copyright 2013 - 2014 &copy; cnpmjs.org
-        |
-        <a href="/">Home</a>
-        |
-        <script>var cnzz_protocol = (("https:" == document.location.protocol) ? " https://" : " http://");document.write(unescape("%3Cspan id='cnzz_stat_icon_5757157'%3E%3C/span%3E%3Cscript src='" + cnzz_protocol + "s17.cnzz.com/stat.php%3Fid%3D5757157%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));</script>
+        {{footer}}
       </p>
-      <a href="https://github.com/fengmk2/cnpmjs.org" id="fork" target="_blank">
+      <a href="https://github.com/cnpm/cnpmjs.org" id="fork" target="_blank">
         <img alt="Fork me on GitHub" src="http://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
     </a>
     </div>

view/web/package.html

@@ -1,7 +1,13 @@
   <div id="package">
     <h1>
       <%= package.name %>
-      <small>(<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)</small>
+      <small>
+      <% if (package._publish_on_cnpm) { %>
+      (Private package)
+      <% } else { %>
+      (<a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from official npm registry)
+      <% } %>
+      </small>
     </h1>
 
     <% if (package.deprecated) { %>
@@ -12,7 +18,7 @@
       <p class="description"><%= package.description %></p>
     <% } %>
 
-    <pre class="sh"><code>$ cnpm install <%= package.name %></code></pre>
+    <pre class="sh"><code>$ <%- config.npmClientName %> install <%= package.name %></code></pre>
 
     <table class="downloads">
       <tbody>
@@ -30,19 +36,19 @@
     </table>
 
     <table class="metadata">
-      <% if (package.maintainers) { %>
+      <% if (Array.isArray(package.maintainers) && package.maintainers.length > 0) { %>
       <tr>
         <th>Maintainers</th>
         <td>
         <% package.maintainers.forEach(function (m) { %>
-          <div class="user">
+          <span class="user">
             <a class="username" href="/~<%= m.name %>">
               <% if (m.gravatar) { %>
               <img src="<%- m.gravatar %>" class="avatar">
               <% } %>
               <%= m.name %>
             </a>
-          </div>
+          </span>
         <% }) %>
         </td>
       </tr>
@@ -168,6 +174,35 @@
         </td>
       </tr>
       <% } %>
+
+      <% if (Array.isArray(package.contributors) && package.contributors.length > 0) { %>
+      <tr>
+        <th>Contributors</th>
+        <td>
+        <% package.contributors.forEach(function (m) { %>
+          <span class="user">
+            <a class="username" target="_blank" href="<%= m.url %>">
+              <% if (m.gravatar) { %>
+              <img src="<%- m.gravatar %>" class="avatar">
+              <% } %>
+              <%= m.name %>
+            </a>
+          </span>
+        <% }) %>
+        </td>
+      </tr>
+      <% } %>
+
+      <% if (package.dist && package.dist.tarball) { %>
+      <tr>
+        <th>Download</th>
+        <td>
+          <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
+            <%= package.dist.tarball %>
+          </a>
+        </td>
+      </tr>
+      <% } %>
     </table>
 
     <div class="details">

view/web/sync.html

@@ -11,6 +11,7 @@
   var $notify = $('#sync-notify');
   var timer;
   var name = '<%= name %>';
+  var resourceURL = '/sync/' + name;
   $(function() {
     var checkLogId = location.hash.match(/logid=(\d+)/);
     var logid = checkLogId ? checkLogId[1] : '';
@@ -18,7 +19,7 @@
       return getSyncLog(logid);
     }
     $.ajax({
-      url: location.pathname,
+      url: resourceURL,
       type: 'PUT',
       dataType: 'json',
       success: handleSyncSucess,
@@ -53,7 +54,7 @@
   var hasFail = false;
   function getSyncLog(id) {
     $.ajax({
-      url: location.pathname + '/log/' + id + '?offset=' + offset,
+      url: resourceURL + '/log/' + id + '?offset=' + offset,
       type: 'GET',
       dataType: 'json',
       success: function (data) {