Release 0.2.8

add download link for package page

History.md

@@ -1,4 +1,32 @@
 
+0.2.8 / 2014-01-14 
+==================
+
+  * dont show err stack on test env
+  * add download link for package page
+
+0.2.7 / 2014-01-13 
+==================
+
+  * add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
+
+0.2.6 / 2014-01-13 
+==================
+
+  * support custom session store, fixed #146
+
+0.2.5 / 2014-01-13 
+==================
+
+  * add download timeout and unit test
+  * use downloadStream() first
+  * nfs download to a writeable stream.
+
+0.2.4 / 2014-01-10 
+==================
+
+  * set main script to  index.js, fixed #142
+
 0.2.3 / 2014-01-10 
 ==================
 

README.md

@@ -9,12 +9,13 @@ cnpmjs.org
 
 ## What is this?
 
-Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
+
 ## Install
 
 ```bash

common/qnfs.js

@@ -19,6 +19,8 @@ var config = require('../config');
 
 var client = qn.create(config.qn);
 
+exports._client = client;
+
 /**
  * Upload file
  *

common/session.js

@@ -33,7 +33,7 @@ if (config.debug) {
   session = connect.session({
     key: key,
     secret: config.sessionSecret,
-    store: new RedisStore(config.redis),
+    store: config.sessionStore || new RedisStore(config.redis),
     cookie: cookie,
   });
 }

config/index.js

@@ -71,6 +71,7 @@ var config = {
     debug: false
   },
 
+  registryHost: 'r.cnpmjs.org',
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {

controllers/registry/module.js

@@ -23,6 +23,9 @@ var utility = require('utility');
 var eventproxy = require('eventproxy');
 var Bagpipe = require('bagpipe');
 var urlparse = require('url').parse;
+var mime = require('mime');
+var semver = require('semver');
+var ms = require('ms');
 var config = require('../../config');
 var Module = require('../../proxy/module');
 var Total = require('../../proxy/total');
@@ -32,7 +35,6 @@ var Log = require('../../proxy/module_log');
 var DownloadTotal = require('../../proxy/download');
 var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var logger = require('../../common/logger');
-var semver = require('semver');
 
 exports.show = function (req, res, next) {
   var name = req.params.name;
@@ -167,6 +169,8 @@ exports.get = function (req, res, next) {
 
 var _downloads = {};
 
+var DOWNLOAD_TIMEOUT = ms('10m');
+
 exports.download = function (req, res, next) {
   var name = req.params.name;
   var filename = req.params.filename;
@@ -181,9 +185,9 @@ exports.download = function (req, res, next) {
     }
     var dist = row.package.dist;
     if (dist.key) {
-      return ep.emit('key', dist.key);
+      return ep.emit('key', dist);
     } else {
-      return ep.emit('url', dist.tarball);
+      return ep.emit('url', dist);
     }
     ep.emit('nodist');
   });
@@ -195,23 +199,46 @@ exports.download = function (req, res, next) {
     ep.emit('url', nfs.url(common.getCDNKey(name, filename)));
   });
 
-  ep.once('url', function (url) {
+  ep.once('url', function (dist) {
     res.statusCode = 302;
-    res.setHeader('Location', url);
+    res.setHeader('Location', dist.tarball);
     res.end();
     _downloads[name] = (_downloads[name] || 0) + 1;
   });
 
-  ep.once('key', function (key) {
-    if (!nfs.download) {
+  ep.once('key', function (dist) {
+    if (!nfs.downloadStream && !nfs.download) {
       return next();
     }
-    var tmpPath = path.join(config.uploadDir, utility.randomString() + key);
+
+    _downloads[name] = (_downloads[name] || 0) + 1;
+
+    if (typeof dist.size === 'number') {
+      res.setHeader('Content-Length', dist.size);
+    }
+    res.setHeader('Content-Type', mime.lookup(dist.key));
+    res.setHeader('Content-Disposition: attachment; filename="' + filename + '"');
+    res.setHeader('ETag', dist.shasum);
+
+    if (nfs.downloadStream) {
+      nfs.downloadStream(dist.key, res, {timeout: DOWNLOAD_TIMEOUT},
+      function (err) {
+        if (err) {
+          // TODO: just end or send error response?
+          return next(err);
+        }
+      });
+      return;
+    }
+
+    // use download file api
+    var tmpPath = path.join(config.uploadDir, utility.randomString() + dist.key);
     function cleanup() {
       fs.unlink(tmpPath, utility.noop);
     }
 
-    nfs.download(key, tmpPath, function (err) {
+    nfs.download(dist.key, tmpPath, {timeout: DOWNLOAD_TIMEOUT},
+    function (err) {
       if (err) {
         cleanup();
         return next(err);
@@ -220,7 +247,6 @@ exports.download = function (req, res, next) {
       tarball.on('error', cleanup);
       tarball.on('end', cleanup);
       tarball.pipe(res);
-      _downloads[name] = (_downloads[name] || 0) + 1;
     });
   });
 };
@@ -323,8 +349,13 @@ exports.upload = function (req, res, next) {
         });
       }
       shasum = shasum.digest('hex');
-      var key = common.getCDNKey(name, filename);
-      nfs.upload(filepath, {key: key, size: length}, function (err, result) {
+
+      var options = {
+        key: common.getCDNKey(name, filename),
+        size: length,
+        shasum: shasum
+      };
+      nfs.upload(filepath, options, function (err, result) {
         // remove tmp file whatever
         fs.unlink(filepath, utility.noop);
         if (err) {
@@ -492,9 +523,12 @@ exports.addPackageAndDist = function (req, res, next) {
     shasum = crypto.createHash('sha1');
     shasum.update(tarballBuffer);
     shasum = shasum.digest('hex');
-    var key = common.getCDNKey(name, filename);
 
-    nfs.uploadBuffer(tarballBuffer, {key: key}, ep.done('upload'));
+    var options = {
+      key: common.getCDNKey(name, filename),
+      shasum: shasum
+    };
+    nfs.uploadBuffer(tarballBuffer, options, ep.done('upload'));
   });
 
   ep.on('upload', function (result) {

controllers/web/package.js

@@ -20,10 +20,12 @@ var semver = require('semver');
 var marked = require('marked');
 var gravatar = require('gravatar');
 var humanize = require('humanize-number');
+var config = require('../../config');
 var Module = require('../../proxy/module');
 var down = require('../download');
 var sync = require('../sync');
 var Log = require('../../proxy/module_log');
+var setDownloadURL = require('../../lib/common').setDownloadURL;
 
 exports.display = function (req, res, next) {
   var params = req.params;
@@ -81,6 +83,8 @@ exports.display = function (req, res, next) {
       download[k] = humanize(download[k]);
     }
 
+    setDownloadURL(pkg, req, config.registryHost);
+
     res.render('package', {
       title: 'Package - ' + pkg.name,
       package: pkg,

dispatch.js

@@ -24,46 +24,36 @@ var workerPath = path.join(__dirname, 'worker.js');
 var childProcess = require('child_process');
 var syncPath = path.join(__dirname, 'sync');
 
-function start(customConfig) {
-  config.loadConfig(customConfig);
+if (config.enableCluster) {
+  cluster.setupMaster({
+    exec: workerPath
+  });
 
-  if (config.enableCluster) {
-    cluster.setupMaster({
-      exec: workerPath
-    });
+  cluster.on('fork', function (worker) {
+    console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
+  });
 
-    cluster.on('fork', function (worker) {
-      console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
-    });
+  cluster.on('disconnect', function (worker) {
+    var w = cluster.fork();
+    console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
+      new Date(), process.pid, worker.process.pid, w.process.pid);
+  });
 
-    cluster.on('disconnect', function (worker) {
-      var w = cluster.fork();
-      console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
-        new Date(), process.pid, worker.process.pid, w.process.pid);
-    });
+  cluster.on('exit', function (worker, code, signal) {
+    var exitCode = worker.process.exitCode;
+    var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
+    err.name = 'WorkerDiedError';
+    console.error(err);
+  });
 
-    cluster.on('exit', function (worker, code, signal) {
-      var exitCode = worker.process.exitCode;
-      var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
-      err.name = 'WorkerDiedError';
-      console.error(err);
-    });
-
-    var numCPUs = require('os').cpus().length;
-    // Fork workers.
-    for (var i = 0; i < numCPUs; i++) {
-      cluster.fork();
-    }
-
-    childProcess.fork(syncPath);
-  } else {
-    require(workerPath);
-    require(syncPath);
+  var numCPUs = require('os').cpus().length;
+  // Fork workers.
+  for (var i = 0; i < numCPUs; i++) {
+    cluster.fork();
   }
-}
 
-if (!module.parent) {
-  start();
+  childProcess.fork(syncPath);
 } else {
-  module.exports = start;
+  require(workerPath);
+  require(syncPath);
 }

docs/web/readme.md

@@ -6,7 +6,7 @@
 
 ## What is this?
 
-> Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+> Private npm registry and web for Enterprise, base on MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
 @[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
 
@@ -16,7 +16,7 @@
 
 ## Registry
 
-* Our public registry: [registry.cnpmjs.org](http://registry.cnpmjs.org)
+* Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
 * Current [cnpmjs.org](/) version: <span id="app-version"></span>
 
 <table class="downloads">
@@ -119,13 +119,13 @@ $(function () {
 alias it:
 
 ```bash
-alias cnpm="npm --registry=http://registry.cnpmjs.org \
+alias cnpm="npm --registry=http://r.cnpmjs.org \
 --cache=$HOME/.npm/.cache/cnpm \
 --disturl=http://dist.u.qiniudn.com \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://r.cnpmjs.org \
   --cache=$HOME/.npm/.cache/cnpm \
   --disturl=http://dist.u.qiniudn.com \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
@@ -139,7 +139,7 @@ $ npm install cnpm -g
 
 ### install
 
-Install package from [registry.cnpmjs.org](http://registry.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](http://r.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
 ```
 $ cnpm install [name]

index.js

@@ -0,0 +1,30 @@
+/*!
+ * cnpmjs.org - index.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var config = require('./config');
+
+exports.loadConfig = config.loadConfig;
+exports.config = config;
+
+exports.startWorker = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./worker');
+};
+
+exports.startSync = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./sync');
+};

lib/common.js

@@ -29,11 +29,12 @@ exports.getCDNKey = function (name, filename) {
   return '/' + name + '/-/' + filename;
 };
 
-exports.setDownloadURL = function (pkg, req) {
+exports.setDownloadURL = function (pkg, req, host) {
   if (pkg.dist) {
+    host = host || req.headers.host;
     pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
       req.connection.encrypted ? 'https' : 'http',
-      req.headers.host, pkg.name, pkg.name, pkg.version);
+      host, pkg.name, pkg.name, pkg.version);
   }
 };
 

package.json

@@ -1,8 +1,8 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.2.3",
-  "description": "Private npm registry and web",
-  "main": "dispatch.js",
+  "version": "0.2.8",
+  "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
+  "main": "index.js",
   "scripts": {
     "test": "make test-all",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
@@ -36,6 +36,7 @@
     "logfilestream": "0.1.0",
     "marked": "0.3.0",
     "microtime": "0.5.1",
+    "mime": "1.2.11",
     "mkdirp": "0.3.5",
     "moment": "2.5.0",
     "ms": "0.6.2",

proxy/sync_module_worker.js

@@ -422,8 +422,12 @@ SyncModuleWorker.prototype._syncOneVersion = function (versionIndex, sourcePacka
         return ep.emit('error', err);
       }
 
-      var key = common.getCDNKey(sourcePackage.name, filename);
-      nfs.upload(filepath, {key: key, size: dataSize}, ep.done('uploadResult'));
+      var options = {
+        key: common.getCDNKey(sourcePackage.name, filename),
+        size: dataSize,
+        shasum: shasum
+      };
+      nfs.upload(filepath, options, ep.done('uploadResult'));
     });
   }));
 

servers/registry.js

@@ -65,9 +65,9 @@ app.use(function (req, res, next) {
 
 app.use(function (err, req, res, next) {
   err.url = err.url || req.url;
+  logger.error(err);
   if (process.env.NODE_ENV !== 'test') {
     console.error(err.stack);
-    logger.error(err);
   }
   if (config.debug) {
     return next(err);

servers/web.js

@@ -81,8 +81,10 @@ app.use(urlrouter(routes));
 
 app.use(function (err, req, res, next) {
   err.url = err.url || req.url;
-  console.error(err.stack);
   logger.error(err);
+  if (process.env.NODE_ENV !== 'test') {
+    console.error(err.stack);
+  }
   if (config.debug) {
     return next(err);
   }

test/controllers/registry/module.test.js

@@ -446,6 +446,45 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
+  describe('GET /:name/download/:filename', function () {
+    it('should download a file with 302 redirect', function (done) {
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
+      .expect(302, done)
+    });
+
+    it('should download a file direct from nfs stream', function (done) {
+      var nfs = require('../../../common/nfs');
+      mm(nfs, 'downloadStream', function (key, writeStream, options, callback) {
+        options.timeout.should.equal(600000);
+        nfs._client.download(key, {writeStream: writeStream, timeout: options.timeout}, callback);
+      });
+      Module.__get__ = Module.get;
+      mm(Module, 'get', function (name, version, callback) {
+        Module.__get__(name, version, function (err, info) {
+          info.package.dist.key = 'cutter/-/cutter-0.0.2.tgz';
+          callback(err, info);
+        });
+      });
+      request(app)
+      .get('/cutter/download/cutter-0.0.2.tgz')
+      .expect('ETag', 'c61fde5e8c26d053574d0c722097029fd1bc963a')
+      .expect('Content-Type', 'application/octet-stream')
+      .expect('Content-Length', '3139')
+      // TODO supertest has a bug
+      // Error: expected "Content-Disposition" of "inline; filename="testputmodule-0.1.9.tgz"", got "attachment; filename="testputmodule-0.1.9.tgz": undefined"
+      // .expect('Content-Disposition', 'inline; filename="testputmodule-0.1.9.tgz"')
+      .expect(200)
+      .end(function (err, res) {
+        should.not.exist(err);
+        // TODO: why supertest change buffer to text?
+        // res.text.length.should.equal(3139);
+        done();
+      });
+    });
+  });
+
   describe('DELETE /:name/download/:filename/-rev/:rev', function () {
     var lastRev;
     before(function (done) {

test/sync/sync_exist.test.js

@@ -44,7 +44,7 @@ describe('sync/sync_exist.js', function () {
         should.not.exist(err);
         data.successes.should.eql(['cnpmjs.org', 'cutter']);
         done();
-      });      
+      });
     });
   });
 });

view/web/package.html

@@ -192,6 +192,17 @@
         </td>
       </tr>
       <% } %>
+
+      <% if (package.dist && package.dist.tarball) { %>
+      <tr>
+        <th>Download</th>
+        <td>
+          <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
+            <%= package.dist.tarball %>
+          </a>
+        </td>
+      </tr>
+      <% } %>
     </table>
 
     <div class="details">