Release 2.2.1

fix: support nfs.url is generator

.jshintignore

@@ -2,3 +2,6 @@ node_modules/
 coverage/
 .tmp/
 .git/
+tools/
+public/
+test/

.travis.yml

@@ -1,8 +1,11 @@
 sudo: false
 language: node_js
 node_js:
-  - 'iojs-2'
+  - '1'
+  - '2'
+  - '4'
 addons:
   - postgresql: '9.3'
 script: 'make test-travis-all'
-after_script: 'npm install coveralls@2 && cat ./coverage/lcov.info | coveralls'
+after_script:
+  - "npm i codecov.io && cat ./coverage/coverage.json | ./node_modules/codecov.io/bin/codecov.io.js"

History.md

@@ -1,4 +1,44 @@
 
+2.2.1 / 2015-09-30
+==================
+
+ * test: use istanbul
+ * pref: move out try/catch block
+ * fix: support nfs.url is generator
+
+2.2.0 / 2015-09-29
+==================
+
+ * feat: list packages by username
+ * test: use codecov
+ * feat(badge): support custom subject
+ * fix(sync): add recover logic
+ * feat(sync): add sync scripts
+
+2.1.5 / 2015-09-05
+==================
+
+ * fix: only sync update packages
+
+2.1.4 / 2015-09-05
+==================
+
+ * fix: support new array and old map format both
+ * fix: /-/all/since had been redirect to /-/all/static/today.json
+ * fix(list): let koa-etag to caculate the etag
+
+2.1.3 / 2015-08-18
+==================
+
+ * fix: sync public scope package download url is wrong
+ * fix: default registry change to taobao registry
+
+2.1.2 / 2015-08-09
+==================
+
+ * fix(syncer): sync worker pkg null bug
+ * feat(web): add downloads badge
+
 2.1.1 / 2015-07-27
 ==================
 

Makefile

@@ -3,12 +3,12 @@ REPORTER = spec
 TIMEOUT = 30000
 MOCHA_OPTS =
 DB = sqlite
-DISTURL = http://npm.taobao.org/mirrors/iojs
+DISTURL = https://npm.taobao.org/mirrors/iojs
 BIN = iojs
 
 ifeq ($(findstring io.js, $(shell which node)),)
 	BIN = node
-	DISTURL = http://npm.taobao.org/mirrors/node
+	DISTURL = https://npm.taobao.org/mirrors/node
 endif
 
 install:
@@ -56,7 +56,7 @@ test-pg: init-pg
 test-all: test-sqlite test-mysql
 
 test-cov cov: install init-database
-	@NODE_ENV=test DB=${DB} $(BIN) --harmony \
+	@NODE_ENV=test DB=${DB} $(BIN) \
 		node_modules/.bin/istanbul cover --preserve-comments \
 		node_modules/.bin/_mocha \
 		-- -u exports \
@@ -77,10 +77,9 @@ test-cov-mysql: init-mysql
 
 test-travis: install init-database
 	@NODE_ENV=test DB=${DB} CNPM_SOURCE_NPM=http://registry.npmjs.com CNPM_SOURCE_NPM_ISCNPM=false \
-		$(BIN) --harmony \
+		$(BIN) \
 		node_modules/.bin/istanbul cover --preserve-comments \
 		node_modules/.bin/_mocha \
-		--report lcovonly \
 		-- -u exports \
 		--reporter dot \
 		--timeout $(TIMEOUT) \

README.md

@@ -3,29 +3,20 @@ cnpmjs.org
 
 [![NPM version][npm-image]][npm-url]
 [![build status][travis-image]][travis-url]
-[![Test coverage][coveralls-image]][coveralls-url]
-[![Gittip][gittip-image]][gittip-url]
+[![Test coverage][cov-image]][cov-url]
 [![David deps][david-image]][david-url]
-[![node version][node-image]][node-url]
 [![npm download][download-image]][download-url]
-[![gitter][gitter-image]][gitter-url]
 
 [npm-image]: http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square
 [npm-url]: http://cnpmjs.org/package/cnpmjs.org
 [travis-image]: https://img.shields.io/travis/cnpm/cnpmjs.org.svg?style=flat-square
 [travis-url]: https://travis-ci.org/cnpm/cnpmjs.org
-[coveralls-image]: https://img.shields.io/coveralls/cnpm/cnpmjs.org.svg?style=flat-square
-[coveralls-url]: https://coveralls.io/r/cnpm/cnpmjs.org?branch=master
-[gittip-image]: https://img.shields.io/gittip/fengmk2.svg?style=flat-square
-[gittip-url]: https://www.gittip.com/fengmk2/
+[cov-image]: http://codecov.io/github/cnpm/cnpmjs.org/coverage.svg?branch=master
+[cov-url]: http://codecov.io/github/cnpm/cnpmjs.org?branch=master
 [david-image]: https://img.shields.io/david/cnpm/cnpmjs.org.svg?style=flat-square
 [david-url]: https://david-dm.org/cnpm/cnpmjs.org
-[node-image]: https://img.shields.io/badge/node.js-%3E=_0.11-red.svg?style=flat-square
-[node-url]: http://nodejs.org/download/
 [download-image]: https://img.shields.io/npm/dm/cnpmjs.org.svg?style=flat-square
 [download-url]: https://npmjs.org/package/cnpmjs.org
-[gitter-image]: https://badges.gitter.im/Join%20Chat.svg
-[gitter-url]: https://gitter.im/cnpm/cnpmjs.org?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge
 
 ![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
@@ -120,8 +111,8 @@ Tips: make sure your code is following the [node-style-guide](https://github.com
 
 ## Sponsors
 
-- [![UCloud云计算](https://www.ucloud.cn/www/img/www/logo.png)](http://www.ucloud.cn/sdk?sem=sdk-CNPMJS)
+- [![UCloud云计算](https://www.ucloud.cn/static/style/images/about/logo.png)](http://www.ucloud.cn?sem=sdk-CNPMJS)
 
 ## License
 
-MIT
+[MIT](LICENSE.txt)

common/nfs.js

@@ -1,11 +1,9 @@
 /*!
- * cnpmjs.org - common/nfs.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';

config/index.js

@@ -1,6 +1,4 @@
 /**!
- * cnpmjs.org - config/index.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *

controllers/registry/package/download.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - controllers/registry/package/download.js
- *
- * Copyright(c) fengmk2 and other contributors.
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -18,6 +16,7 @@ var debug = require('debug')('cnpmjs.org:controllers:registry:download');
 var mime = require('mime');
 var utility = require('utility');
 var defer = require('co-defer');
+var is = require('is-type-of');
 var nfs = require('../../../common/nfs');
 var logger = require('../../../common/logger');
 var common = require('../../../lib/common');
@@ -32,12 +31,16 @@ module.exports = function* download(next) {
   var name = this.params.name || this.params[0];
   var filename = this.params.filename || this.params[1];
   var version = filename.slice(name.length + 1, -4);
-  var row = yield* packageService.getModule(name, version);
+  var row = yield packageService.getModule(name, version);
   // can not get dist
   var url = null;
 
   if (typeof nfs.url === 'function') {
-    url = nfs.url(common.getCDNKey(name, filename));
+    if (is.generatorFunction(nfs.url)) {
+      url = yield nfs.url(common.getCDNKey(name, filename));
+    } else {
+      url = nfs.url(common.getCDNKey(name, filename));
+    }
   }
 
   debug('download %s %s %s %s', name, filename, version, url);
@@ -78,7 +81,7 @@ module.exports = function* download(next) {
   this.attachment(filename);
   this.etag = dist.shasum;
 
-  this.body = yield* downloadAsReadStream(dist.key);
+  this.body = yield downloadAsReadStream(dist.key);
 };
 
 defer.setInterval(function* () {

controllers/registry/package/list.js

@@ -45,9 +45,6 @@ module.exports = function* list() {
       }
     }
 
-    // use modifiedTime as etag
-    this.set('ETag', '"' + modifiedTime.getTime() + '"');
-
     // must set status first
     this.status = 200;
     if (this.fresh) {

controllers/registry/package/list_by_user.js

@@ -0,0 +1,29 @@
+/**!
+ * list packages by username
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+const packageService = require('../../../services/package');
+
+module.exports = function*() {
+  const username = this.params.user;
+  const packages = yield packageService.listModulesByUser(username);
+
+  this.body = {
+    user: {
+      name: username,
+    },
+    packages: packages,
+  };
+};

controllers/sync_module_worker.js

@@ -334,7 +334,7 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   var unpublishedInfo = null;
   if (status === 404) {
     // check if it's unpublished
-    if (pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+    if (pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
       unpublishedInfo = pkg.time.unpublished;
     } else {
       pkg = null;

controllers/web/badge.js

@@ -5,7 +5,7 @@
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -18,6 +18,7 @@ var utility = require('utility');
 var util = require('util');
 var config = require('../../config');
 var packageService = require('../../services/package');
+var DownloadTotal = require('../../services/download_total');
 
 exports.version = function* () {
   var color = 'lightgrey';
@@ -40,9 +41,22 @@ exports.version = function* () {
   }
 
   var subject = config.badgeSubject.replace(/\-/g, '--');
+  if (this.query.subject) {
+    subject = this.query.subject.replace(/\-/g, '--');
+  }
   version = version.replace(/\-/g, '--');
   var style = this.query.style || 'flat-square';
   var url = util.format('https://img.shields.io/badge/%s-%s-%s.svg?style=%s',
-    subject, version, color, utility.encodeURIComponent(style));
+    utility.encodeURIComponent(subject), version, color, utility.encodeURIComponent(style));
+  this.redirect(url);
+};
+
+exports.downloads = function* () {
+  // https://img.shields.io/badge/downloads-100k/month-brightgreen.svg?style=flat-square
+  var name = this.params[0];
+  var count = yield DownloadTotal.getTotalByName(name);
+  var style = this.query.style || 'flat-square';
+  var url = util.format('https://img.shields.io/badge/downloads-%s-brightgreen.svg?style=%s',
+    count, utility.encodeURIComponent(style));
   this.redirect(url);
 };

dispatch.js

@@ -1,12 +1,10 @@
 /**!
- * cnpmjs.org - dispatch.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';

docs/web/readme.md

@@ -52,36 +52,42 @@ So `cnpm` is meaning: **Company npm**.
 
 <script src="/js/readme.js"></script>
 
-## Version Badge
+## Badges
 
 Default style is `flat-square`.
 
-Badge URL: `https://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-badge](//cnpmjs.org/badge/v/cnpmjs.org.svg)
+### Version
+
+Badge URL: `http://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-version-badge](//cnpmjs.org/badge/v/cnpmjs.org.svg)
 
 * `<0.1.0 & >=0.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.0.1-red.svg?style=flat-square)
 * `<1.0.0 & >=0.1.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.1.0-green.svg?style=flat-square)
 * `>=1.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-1.0.0-blue.svg?style=flat-square)
 
+### Downloads
+
+Badge URL: `http://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-badge](//cnpmjs.org/badge/d/cnpmjs.org.svg)
+
 ## Usage
 
 use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
 
 ```bash
-$ npm install -g cnpm --registry=https://r.cnpmjs.org
+$ npm install -g cnpm --registry=http://registry.npm.taobao.org
 ```
 
 Or you can alias NPM to use it:
 
 ```bash
-alias cnpm="npm --registry=https://r.cnpmjs.org \
+alias cnpm="npm --registry=http://registry.npm.taobao.org \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=https://cnpmjs.org/mirrors/node \
+--disturl=http://registry.npm.taobao.org/mirrors/node \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://r.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.npm.taobao.org \
   --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=https://cnpmjs.org/mirrors/node \
+  --disturl=http://registry.npm.taobao.org/mirrors/node \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -104,7 +110,7 @@ $ cnpm sync connect
 sync package on web: [sync/connect](/sync/connect)
 
 ```bash
-$ open https://cnpmjs.org/sync/connect
+$ open http://registry.npm.taobao.org/sync/connect
 ```
 
 ### publish / unpublish
@@ -138,4 +144,4 @@ Release [History](/history).
 
 ## Sponsors
 
-- [![UCloud云计算](https://www.ucloud.cn/www/img/www/logo.png)](http://www.ucloud.cn/sdk?sem=sdk-CNPMJS)
+- [![UCloud云计算](https://www.ucloud.cn/static/style/images/about/logo.png)](http://www.ucloud.cn?sem=sdk-CNPMJS)

index.js

@@ -1,6 +1,4 @@
 /**!
- * cnpmjs.org - index.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *

lib/common.js

@@ -27,6 +27,11 @@ exports.getTarballFilepath = function (filename) {
 };
 
 exports.getCDNKey = function (name, filename) {
+  // if name is scope package name, need to auto fix filename as a scope package file name
+  // e.g.: @scope/foo, filename: foo-1.0.0.tgz => filename: @scope/foo-1.0.0.tgz
+  if (name[0] === '@' && filename[0] !== '@') {
+    filename = name.split('/')[0] + '/' + filename;
+  }
   return '/' + name + '/-/' + filename;
 };
 

models/index.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - models/index.js
- *
  * Copyright(c) fengmk2 and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -15,7 +13,6 @@
  */
 
 var path = require('path');
-var Sequelize = require('sequelize');
 var sequelize = require('../common/sequelize');
 
 function load(name) {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.1.1",
+  "version": "2.2.1",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -32,6 +32,7 @@
     "gravatar": "~1.1.0",
     "humanize-ms": "~1.0.1",
     "humanize-number": "~0.0.2",
+    "is-type-of": "~0.3.1",
     "kcors": "~1.0.1",
     "koa": "~0.17.0",
     "koa-limit": "~1.0.2",
@@ -60,7 +61,7 @@
     "chunkstream": "*",
     "co-mocha": "*",
     "contributors": "*",
-    "istanbul-harmony": "*",
+    "istanbul": "*",
     "jshint": "*",
     "mm": "*",
     "mocha": "*",
@@ -91,7 +92,7 @@
     "registry"
   ],
   "engines": {
-    "node": ">= 0.11.14"
+    "node": ">= 1.0.0"
   },
   "author": [
     "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",

routes/registry.js

@@ -1,12 +1,10 @@
 /**!
- * cnpmjs.org - routes/registry.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 "use strict";
@@ -36,6 +34,7 @@ var removeOneVersion = require('../controllers/registry/package/remove_version')
 var updatePackage = require('../controllers/registry/package/update');
 var downloadPackage = require('../controllers/registry/package/download');
 var downloadTotal = require('../controllers/registry/package/download_total');
+var listPackagesByUser = require('../controllers/registry/package/list_by_user');
 
 var addUser = require('../controllers/registry/user/add');
 var showUser = require('../controllers/registry/user/show');
@@ -113,6 +112,7 @@ function routes(app) {
 
   // list all packages of user
   app.get('/-/by-user/:user', userPackage.list);
+  app.get('/-/users/:user/packages', listPackagesByUser);
 
   // download times
   app.get('/downloads/range/:range/:name', downloadTotal);

routes/web.js

@@ -54,6 +54,7 @@ function routes(app) {
   app.get('/_list/search/search', searchRange);
 
   app.get(/^\/badge\/v\/([@\w\-\.\/]+)\.svg$/, badge.version);
+  app.get(/^\/badge\/d\/([@\w\-\.\/]+)\.svg$/, badge.downloads);
 }
 
 module.exports = routes;

services/download_total.js

@@ -32,6 +32,27 @@ exports.getModuleTotal = function* (name, start, end) {
   return formatRows(rows, start, end);
 };
 
+exports.getTotalByName = function* (name) {
+  var rows = yield DownloadTotal.findAll({
+    where: {
+      name: name
+    }
+  });
+  var count = 0;
+  rows.forEach(function (row) {
+    for (var i = 1; i <= 31; i++) {
+      var day = i < 10 ? '0' + i : String(i);
+      var field = 'd' + day;
+      var val = row[field];
+      if (typeof val === 'string') {
+        val = utility.toSafeNumber(val);
+      }
+      count += val;
+    }
+  });
+  return count;
+};
+
 exports.plusModuleTotal = function* (data) {
   var yearMonth = parseYearMonth(data.date);
   // all module download total

services/npm.js

@@ -15,6 +15,7 @@
  * Module dependencies.
  */
 
+var ms = require('humanize-ms');
 var urllib = require('../common/urllib');
 var config = require('../config');
 
@@ -69,6 +70,83 @@ exports.get = function* (name) {
   return data;
 };
 
+exports.fetchUpdatesSince = function* (lastSyncTime, timeout) {
+  var lastModified = lastSyncTime - ms('10m');
+  var data = yield exports.getAllSince(lastModified, timeout);
+  var result = {
+    lastModified: lastSyncTime,
+    names: [],
+  };
+  if (!data) {
+    return result;
+  }
+  if (Array.isArray(data)) {
+    // support https://registry.npmjs.org/-/all/static/today.json
+    var maxModified;
+    data.forEach(function (pkg) {
+      if (pkg.time && pkg.time.modified) {
+        var modified = Date.parse(pkg.time.modified);
+        if (modified >= lastModified) {
+          result.names.push(pkg.name);
+        }
+        if (!maxModified || modified > maxModified) {
+          maxModified = modified;
+        }
+      } else {
+        result.names.push(pkg.name);
+      }
+    });
+    if (maxModified) {
+      result.lastModified = maxModified;
+    }
+  } else {
+    // /-/all/since
+    if (data._updated) {
+      result.lastModified = data._updated;
+      delete data._updated;
+    }
+    result.names = Object.keys(data);
+  }
+  return result;
+};
+
+exports.fetchAllPackagesSince = function* (timestamp) {
+  var r = yield request('/-/all/static/all.json', {
+    registry: 'http://registry.npmjs.org',
+    timeout: 600000
+  });
+  // {"_updated":1441520402174,"0":{"name":"0","dist-tags
+  // "time":{"modified":"2014-06-17T06:38:43.495Z"}
+  var data = r.data;
+  var result = {
+    lastModified: timestamp,
+    lastModifiedName: null,
+    names: [],
+  };
+  var maxModified;
+  for (var key in data) {
+    if (key === '_updated') {
+      continue;
+    }
+    var pkg = data[key];
+    if (!pkg.time || !pkg.time.modified) {
+      continue;
+    }
+    var modified = Date.parse(pkg.time.modified);
+    if (modified >= timestamp) {
+      result.names.push(pkg.name);
+    }
+    if (!maxModified || modified > maxModified) {
+      maxModified = modified;
+      result.lastModifiedName = pkg.name;
+    }
+  }
+  if (maxModified) {
+    result.lastModified = maxModified;
+  }
+  return result;
+};
+
 exports.getAllSince = function* (startkey, timeout) {
   var r = yield* request('/-/all/since?stale=update_after&startkey=' + startkey, {
     timeout: timeout || 300000
@@ -76,6 +154,14 @@ exports.getAllSince = function* (startkey, timeout) {
   return r.data;
 };
 
+exports.getAllToday = function* (timeout) {
+  var r = yield* request('/-/all/static/today.json', {
+    timeout: timeout || 300000
+  });
+  // data is array: see https://registry.npmjs.org/-/all/static/today.json
+  return r.data;
+};
+
 exports.getShort = function* (timeout) {
   var r = yield* request('/-/short', {
     timeout: timeout || 300000,

services/package.js

@@ -28,19 +28,23 @@ var NpmModuleMaintainer = models.NpmModuleMaintainer;
 
 // module
 
+var _parseRow = function (row) {
+  if (row.package.indexOf('%7B%22') === 0) {
+    // now store package will encodeURIComponent() after JSON.stringify
+    row.package = decodeURIComponent(row.package);
+  }
+  row.package = JSON.parse(row.package);
+  if (typeof row.publish_time === 'string') {
+    // pg bigint is string
+    row.publish_time = Number(row.publish_time);
+  }
+};
+
 // module:read
 function parseRow(row) {
   if (row && row.package) {
     try {
-      if (row.package.indexOf('%7B%22') === 0) {
-        // now store package will encodeURIComponent() after JSON.stringify
-        row.package = decodeURIComponent(row.package);
-      }
-      row.package = JSON.parse(row.package);
-      if (typeof row.publish_time === 'string') {
-        // pg bigint is string
-        row.publish_time = Number(row.publish_time);
-      }
+      _parseRow(row);
     } catch (e) {
       console.warn('parse package error: %s, id: %s version: %s, error: %s', row.name, row.id, row.version, e);
     }
@@ -128,7 +132,7 @@ exports.listModules = function* (names) {
       id: ids
     },
     attributes: [
-      'name', 'description'
+      'name', 'description', 'version',
     ]
   });
   return rows;

sync/sync_all.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - sync/sync_all.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -6,6 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -14,7 +15,6 @@
  * Module dependencies.
  */
 
-var ms = require('humanize-ms');
 var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var Status = require('./status');
@@ -40,19 +40,6 @@ function* getFirstSyncPackages(lastSyncModule) {
   }
 }
 
-/**
- * get all the packages that update time > lastSyncTime
- * @param {Number} lastSyncTime
- */
-function* getCommonSyncPackages(lastSyncTime) {
-  var data = yield* npmService.getAllSince(lastSyncTime);
-  if (!data) {
-    return [];
-  }
-  delete data._updated;
-  return Object.keys(data);
-}
-
 module.exports = function* sync() {
   var syncTime = Date.now();
   var info = yield* totalService.getTotalInfo();
@@ -66,7 +53,9 @@ module.exports = function* sync() {
     logger.syncInfo('First time sync all packages from official registry');
     packages = yield* getFirstSyncPackages(info.last_sync_module);
   } else {
-    packages = yield* getCommonSyncPackages(info.last_sync_time - ms('10m'));
+    var result = yield npmService.fetchUpdatesSince(info.last_sync_time);
+    syncTime = result.lastModified;
+    packages = result.names;
   }
 
   packages = packages || [];

sync/sync_exist.js

@@ -1,6 +1,4 @@
 /*!
- * cnpmjs.org - sync/sync_exist.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
@@ -16,7 +14,6 @@
 
 var debug = require('debug')('cnpmjs.org:sync:sync_exist');
 var Status = require('./status');
-var ms = require('humanize-ms');
 var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var npmService = require('../services/npm');
@@ -63,16 +60,10 @@ module.exports = function* sync() {
     }
     allPackages = pkgs;
   } else {
-    debug('sync new module from last exist sync time: %s', info.last_sync_time);
-    var data = yield* npmService.getAllSince(info.last_exist_sync_time - ms('10m'));
-    if (!data) {
-      allPackages = [];
-    }
-    if (data._updated) {
-      syncTime = data._updated;
-      delete data._updated;
-    }
-    allPackages = Object.keys(data);
+    debug('sync new module from last exist sync time: %s', info.last_exist_sync_time);
+    var result = yield npmService.fetchUpdatesSince(info.last_exist_sync_time);
+    allPackages = result.names;
+    syncTime = result.lastModified;
   }
 
   var packages = intersection(existPackages, allPackages);

sync/sync_since.js

@@ -0,0 +1,66 @@
+/**!
+ * sync packages since by some days ago
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+const thunkify = require('thunkify-wrap');
+const co = require('co');
+const ms = require('humanize-ms');
+const npmService = require('../services/npm');
+const SyncModuleWorker = require('../controllers/sync_module_worker');
+
+function* sync(sinceTimestamp) {
+  console.log('Fetching packages since: %s', new Date(sinceTimestamp));
+  var result = yield npmService.fetchAllPackagesSince(sinceTimestamp);
+  var packages = result.names;
+
+  packages = packages || [];
+  if (!packages.length) {
+    console.log('no packages need be sync');
+    process.exit(0);
+  }
+  // var news = [];
+  // for (var i = 0; i < packages.length; i++) {
+  //   if (packages[i] === 'elwms') {
+  //     news = packages.slice(i);
+  //     break;
+  //   }
+  // }
+  // packages = news;
+  console.log('lastModified: %s, lastModified package: %s, total %d packages to sync: %j',
+    new Date(result.lastModified), result.lastModifiedName, packages.length, packages);
+
+  var worker = new SyncModuleWorker({
+    username: 'sync_since',
+    name: packages,
+    noDep: true,
+    concurrency: 1,
+    syncUpstreamFirst: false,
+  });
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  console.log('All packages sync done, successes %d, fails %d',
+      worker.successes.length, worker.fails.length);
+  process.exit(0);
+}
+
+co(function* () {
+  var timestamp = Date.now() - ms(process.argv[2] || '30d');
+  yield sync(timestamp);
+}).catch(function (err) {
+  console.error(err.stack);
+  process.exit(1);
+});

test/controllers/registry/package/download.test.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - test/controllers/registry/package/download.test.js
- *
- * Copyright(c) fengmk2 and other contributors.
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -15,36 +13,66 @@
  */
 
 var request = require('supertest');
+var mm = require('mm');
+var config = require('../../../../config');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
-describe('controllers/registry/package/download.test.js', function () {
+describe('test/controllers/registry/package/download.test.js', function () {
   before(function (done) {
     var pkg = utils.getPackage('@cnpmtest/download-test-module', '1.0.0', utils.admin);
-    request(app.listen())
+    request(app)
     .put('/' + pkg.name)
     .set('authorization', utils.adminAuth)
     .send(pkg)
     .expect(201, done);
   });
 
+  afterEach(mm.restore);
+
   describe('GET /:name/download/:filename', function () {
     it('should download a file with 200', function (done) {
-      request(app.listen())
+      request(app)
       .get('/@cnpmtest/download-test-module/download/@cnpmtest/download-test-module-1.0.0.tgz')
       .expect(200, done);
     });
 
     it('should alias /:name/-/:filename to /:name/download/:filename', function (done) {
-      request(app.listen())
+      request(app)
       .get('/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
       .expect(200, done);
     });
 
     it('should 404 when package not exists', function (done) {
-      request(app.listen())
+      request(app)
       .get('/@cnpmtest/download-test-module-not-exists/download/@cnpmtest/download-test-module-not-exists-1.0.0.tgz')
       .expect(404, done);
     });
+
+    describe('nfs.url is function', function() {
+      it('should work with nfs.url is generatorFunction', function(done) {
+        mm(config.nfs, 'url', function*(key) {
+          return 'http://foo.test.com' + key;
+        });
+        mm(config, 'downloadRedirectToNFS', true);
+
+        request(app)
+        .get('/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
+        .expect('Location', 'http://foo.test.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
+        .expect(302, done);
+      });
+
+      it('should work with nfs.url is function', function(done) {
+        mm(config.nfs, 'url', function(key) {
+          return 'http://foo.test.com' + key;
+        });
+        mm(config, 'downloadRedirectToNFS', true);
+
+        request(app)
+        .get('/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
+        .expect('Location', 'http://foo.test.com/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
+        .expect(302, done);
+      });
+    });
   });
 });

test/controllers/registry/package/list_by_user.test.js

@@ -0,0 +1,87 @@
+/**!
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var request = require('supertest');
+var mm = require('mm');
+var should = require('should');
+var app = require('../../../../servers/registry');
+var utils = require('../../../utils');
+
+describe('test/controllers/registry/package/list_by_user.test.js', function () {
+  var user = 'cnpmjstest_list_by_user';
+  var userauth = 'Basic ' + new Buffer(user + ':' + user).toString('base64');
+
+  afterEach(mm.restore);
+
+  before(function (done) {
+    var pkg = utils.getPackage('@cnpmtest/list_by_user_module1', '1.0.1', user);
+    request(app)
+    .put('/' + pkg.name)
+    .set('authorization', userauth)
+    .send(pkg)
+    .expect(201, function (err) {
+      should.not.exist(err);
+
+      var pkg2 = utils.getPackage('@cnpmtest/list_by_user_module2', '2.0.0', user);
+      request(app)
+      .put('/' + pkg2.name)
+      .set('authorization', userauth)
+      .send(pkg2)
+      .expect(201, done);
+    });
+  });
+
+  describe('GET /-/users/:user/packages', function () {
+    it('should get 200', function (done) {
+      var url = '/-/users/' + user + '/packages';
+      request(app)
+      .get(url)
+      .expect(function(res) {
+        var data = res.body;
+        data.user.name.should.equal(user);
+        var map = {};
+        data.packages.forEach(function(pkg) {
+          map[pkg.name] = pkg;
+        });
+        map['@cnpmtest/list_by_user_module1'].should.be.an.Object;
+        map['@cnpmtest/list_by_user_module1'].should.eql({
+          name: '@cnpmtest/list_by_user_module1',
+          description: '',
+          version: '1.0.1',
+        });
+
+        map['@cnpmtest/list_by_user_module2'].should.be.an.Object;
+        map['@cnpmtest/list_by_user_module2'].should.eql({
+          name: '@cnpmtest/list_by_user_module2',
+          description: '',
+          version: '2.0.0',
+        });
+      })
+      .expect(200, done);
+    });
+
+    it('should get empty packages list when user not exists', function (done) {
+      var url = '/-/users/not-exist-username/packages';
+      request(app)
+      .get(url)
+      .expect({
+        user: {
+          name: 'not-exist-username',
+        },
+        packages: [],
+      })
+      .expect(200, done);
+    });
+  });
+});

test/controllers/sync_module_worker.test.js

@@ -17,6 +17,7 @@
 var mm = require('mm');
 var thunkify = require('thunkify-wrap');
 var request = require('supertest');
+var urllib = require('urllib');
 var config = require('../../config');
 var SyncModuleWorker = require('../../controllers/sync_module_worker');
 var logService = require('../../services/module_log');
@@ -24,7 +25,7 @@ var packageService = require('../../services/package');
 var utils = require('../utils');
 var app = require('../../servers/registry');
 
-describe('controllers/sync_module_worker.test.js', function () {
+describe('test/controllers/sync_module_worker.test.js', function () {
   afterEach(mm.restore);
 
   beforeEach(function () {
@@ -81,6 +82,24 @@ describe('controllers/sync_module_worker.test.js', function () {
     worker.start();
     var end = thunkify.event(worker, 'end');
     yield end();
+
+    var tgzUrl;
+    function checkResult() {
+      return function (done) {
+        request(app.listen())
+        .get('/@sindresorhus/df')
+        .expect(function (res) {
+          var latest = res.body.versions[res.body['dist-tags']['latest']];
+          tgzUrl = latest.dist.tarball;
+        })
+        .expect(200, done);
+      };
+    }
+
+    yield checkResult();
+
+    var r = yield urllib.request(tgzUrl);
+    r.status.should.equal(200);
   });
 
   it('should start a sync worker and dont sync deps', function* () {

test/controllers/web/badge.test.js

@@ -5,7 +5,7 @@
  * MIT Licensed
  *
  * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -55,6 +55,21 @@ describe('controllers/web/badge.test.js', function () {
       });
     });
 
+    it('should support custom subject', function (done) {
+      var pkg = utils.getPackage('@cnpmtest/badge-test-module', '3.0.1', utils.admin, 'v3');
+      request(registry)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .end(function (err) {
+        should.not.exists(err);
+        request(app)
+        .get('/badge/v/@cnpmtest/badge-test-module.svg?style=flat-square&tag=v3&subject=ant-design')
+        .expect('Location', 'https://img.shields.io/badge/ant--design-3.0.1-blue.svg?style=flat-square')
+        .expect(302, done);
+      });
+    });
+
     it('should support 1.0.0-beta1', function (done) {
       var pkg = utils.getPackage('@cnpmtest/badge-test-module', '1.0.0-beta1', utils.admin);
       request(registry)
@@ -107,4 +122,21 @@ describe('controllers/web/badge.test.js', function () {
       .expect(302, done);
     });
   });
+
+  describe('GET /badge/d/:name.svg', function () {
+    it('should show downloads count', function (done) {
+      var pkg = utils.getPackage('@cnpmtest/badge-download-module', '1.0.1', utils.admin);
+      request(registry)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .end(function (err) {
+        should.not.exists(err);
+        request(app)
+        .get('/badge/d/@cnpmtest/badge-download-module.svg?style=flat-square')
+        .expect('Location', 'https://img.shields.io/badge/downloads-0-brightgreen.svg?style=flat-square')
+        .expect(302, done);
+      });
+    });
+  });
 });

test/init_db.js

@@ -38,6 +38,7 @@ var usernames = [
   'cnpmjstest10', // admin
   'cnpmjstestAdmin2', // other admin
   'cnpmjstestAdmin3', // other admin
+  'cnpmjstest_list_by_user',
 ];
 
 var count = usernames.length;

test/lib/common.test.js

@@ -6,7 +6,7 @@
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -17,7 +17,7 @@
 
 var common = require('../../lib/common');
 
-describe('lib/common.test.js', function () {
+describe('test/lib/common.test.js', function () {
   describe('isAdmin()', function () {
     it('should admin is admin', function () {
       common.isAdmin('admin').should.equal(true);
@@ -26,4 +26,13 @@ describe('lib/common.test.js', function () {
       common.isAdmin('toString').should.equal(false);
     });
   });
+
+  describe('getCDNKey()', function () {
+    it('should auto fix scope filename', function () {
+      common.getCDNKey('foo', 'foo-1.0.0.tgz').should.equal('/foo/-/foo-1.0.0.tgz');
+      common.getCDNKey('@bar/foo', 'foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
+      common.getCDNKey('@bar/foo', '@bar/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
+      common.getCDNKey('@bar/foo', '@bar1/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar1/foo-1.0.0.tgz');
+    });
+  });
 });

test/services/download_total.test.js

@@ -16,7 +16,7 @@
 
 var DownloadTotal = require('../../services/download_total');
 
-describe('services/download_total.test.js', function () {
+describe('test/services/download_total.test.js', function () {
   describe('plusModuleTotal()', function () {
     it('should plus one module download count', function* () {
       var data = {
@@ -89,4 +89,25 @@ describe('services/download_total.test.js', function () {
       rows[2].count.should.equal(3);
     });
   });
+
+  describe('getTotalByName()', function () {
+    it('should get total downloads', function* () {
+      var data = {
+        date: '2014-10-21',
+        name: 'getTotalByName-module',
+        count: 1000
+      };
+      yield* DownloadTotal.plusModuleTotal(data);
+
+      data = {
+        date: '2015-10-22',
+        name: 'getTotalByName-module',
+        count: 2
+      };
+      yield* DownloadTotal.plusModuleTotal(data);
+
+      var count = yield DownloadTotal.getTotalByName('getTotalByName-module');
+      count.should.equal(1002);
+    });
+  });
 });

test/services/package.test.js

@@ -124,7 +124,7 @@ describe('test/services/package.test.js', function () {
       var mods = yield* Package.listPublicModulesByUser('listPublicModuleNamesByUser-user');
       mods.should.length(3);
       mods.forEach(function (mod) {
-        mod.toJSON().should.have.keys('name', 'description');
+        mod.toJSON().should.have.keys('name', 'description', 'version');
         mod.name.should.containEql('listPublicModuleNamesByUser-module');
       });
     });

test/sync/sync_all.test.js

@@ -21,7 +21,7 @@ var npmSerivce = require('../../services/npm');
 var totalService = require('../../services/total');
 var packageService = require('../../services/package');
 
-describe('sync/sync_all.test.js', function () {
+describe('test/sync/sync_all.test.js', function () {
   beforeEach(function () {
     mm(config, 'syncModel', 'all');
   });
@@ -48,5 +48,31 @@ describe('sync/sync_all.test.js', function () {
       data.successes.should.eql(['mk2testmodule']);
       mm.restore();
     });
+
+    it('should sync with array data format ok', function* () {
+      mm.data(npmSerivce, 'getAllSince', [
+        {
+          name: 'mk2testmodule',
+          versions: {
+            '0.0.2': 'latest'
+          }
+        },
+        {
+          name: 'mk2testmodule1',
+          time: {
+            modified: '2015-09-05T07:31:35.734Z',
+          },
+          versions: {
+            '0.0.2': 'latest'
+          }
+        },
+      ]);
+      mm.data(npmSerivce, 'getShort', ['mk2testmodule']);
+      mm.data(totalService, 'getTotalInfo', {last_sync_time: Date.now()});
+      mm.data(packageService, 'listAllPublicModuleNames', [ 'mk2testmodule' ]);
+      var data = yield sync;
+      data.successes.should.eql(['mk2testmodule']);
+      mm.restore();
+    });
   });
 });

test/sync/sync_exist.test.js

@@ -21,7 +21,7 @@ var npmService = require('../../services/npm');
 var totalService = require('../../services/total');
 var utils = require('../utils');
 
-describe('sync/sync_exist.test.js', function () {
+describe('test/sync/sync_exist.test.js', function () {
   beforeEach(function () {
     mm(config, 'syncModel', 'all');
   });
@@ -43,15 +43,28 @@ describe('sync/sync_exist.test.js', function () {
     it('should sync common ok', function *() {
       mm.data(npmService, 'getAllSince', {
         _updated: Date.now(),
-        'byte': {},
+        byte: {},
       });
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield* sync();
       data.successes.should.eql(['byte']);
 
-      mm.data(npmService, 'getAllSince', {
-        _updated: Date.now(),
-      });
+      mm.data(npmService, 'getAllSince', []);
+      var data = yield* sync();
+      data.successes.should.eql([]);
+    });
+
+    it('should sync with array format data', function *() {
+      mm.data(npmService, 'getAllSince', [
+        {
+          name: 'byte',
+        }
+      ]);
+      mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
+      var data = yield* sync();
+      data.successes.should.eql(['byte']);
+
+      mm.data(npmService, 'getAllSince', []);
       var data = yield* sync();
       data.successes.should.eql([]);
     });

test/utils.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - test/utils.js
- *
- * Copyright(c) fengmk2 and other contributors.
+ * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';

view/web/profile.html

@@ -59,10 +59,12 @@
   <% if (!packages.length) { %>
     <div class="alert alert-warning">Can not found any package by <%= user.name %>.</div>
   <% } %>
-  <% for (var i = 0; i < packages.length; i++) { %>
+  <% for (var i = 0; i < packages.length; i++) {
+    var package = packages[i];
+  %>
     <div class="package">
-      <a href="/package/<%= packages[i].name %>" class="package-name"><%= packages[i].name %></a>
-      <span class="package-description"><%= packages[i].description %></span>
+      <a href="/package/<%= packages[i].name %>" class="package-name"><%= package.name %></a>
+      <span class="package-description">(latest: <%= package.version %>) <%= package.description %></span>
     </div>
   <% } %>
 </div>

worker.js

@@ -1,6 +1,4 @@
 /**!
- * cnpmjs.org - worker.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *