Release 2.12.2

closes #981

History.md

@@ -1,4 +1,30 @@
 
+2.12.2 / 2016-07-11
+==================
+
+  * fix: getModuleByRange don't list all packages (#990)
+  * fix: should show new version package count (#984)
+
+2.12.1 / 2016-07-01
+==================
+
+  * fix: make sure chagnes stream destroy (#982)
+  * chore(package): update semver to version 5.2.0 (#978)
+  * deps: use ^ instead of ~ (#976)
+  * chore(package): update mini-logger to version 1.1.1 (#973)
+
+2.12.0 / 2016-06-26
+==================
+
+  * fix: logger seperator should be one EOL ([#972](https://github.com/cnpm/cnpmjs.org/issues/972))
+  * feat: add security check badge for public package ([#971](https://github.com/cnpm/cnpmjs.org/issues/971))
+
+2.11.0 / 2016-06-25
+==================
+
+  * feat: add changes stream syncer ([#970](https://github.com/cnpm/cnpmjs.org/issues/970))
+  * chore(package): update pg to version 5.1.0 ([#953](https://github.com/cnpm/cnpmjs.org/issues/953))
+
 2.10.1 / 2016-06-05
 ==================
 

Makefile

@@ -93,7 +93,7 @@ test-travis-pg:
 	@psql -c 'CREATE DATABASE cnpmjs_test;' -U postgres
 	@DB_PORT=5432 DB_USER=postgres $(MAKE) test-travis DB=postgres
 
-test-travis-all: test-travis-sqlite test-travis-mysql test-travis-pg
+test-travis-all: jshint test-travis-sqlite test-travis-mysql test-travis-pg
 
 dev:
 	@NODE_ENV=development node node_modules/.bin/node-dev dispatch.js

README.md

@@ -3,18 +3,21 @@ cnpmjs.org
 
 [![NPM version][npm-image]][npm-url]
 [![build status][travis-image]][travis-url]
-[![Test coverage][cov-image]][cov-url]
+[![Test coverage][codecov-image]][codecov-url]
 [![David deps][david-image]][david-url]
+[![Known Vulnerabilities][snyk-image]][snyk-url]
 [![npm download][download-image]][download-url]
 
 [npm-image]: http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square
 [npm-url]: http://cnpmjs.org/package/cnpmjs.org
 [travis-image]: https://img.shields.io/travis/cnpm/cnpmjs.org.svg?style=flat-square
 [travis-url]: https://travis-ci.org/cnpm/cnpmjs.org
-[cov-image]: http://codecov.io/github/cnpm/cnpmjs.org/coverage.svg?branch=master
-[cov-url]: http://codecov.io/github/cnpm/cnpmjs.org?branch=master
+[codecov-image]: https://codecov.io/gh/cnpm/cnpmjs.org/branch/master/graph/badge.svg
+[codecov-url]: https://codecov.io/gh/cnpm/cnpmjs.org
 [david-image]: https://img.shields.io/david/cnpm/cnpmjs.org.svg?style=flat-square
 [david-url]: https://david-dm.org/cnpm/cnpmjs.org
+[snyk-image]: https://snyk.io/test/npm/cnpmjs.org/badge.svg?style=flat-square
+[snyk-url]: https://snyk.io/test/npm/cnpmjs.org
 [download-image]: https://img.shields.io/npm/dm/cnpmjs.org.svg?style=flat-square
 [download-url]: https://npmjs.org/package/cnpmjs.org
 

common/logger.js

@@ -1,62 +1,56 @@
-/**!
- * cnpmjs.org - common/logger.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+const debug = require('debug')('cnpmjs.org:logger');
+const formater = require('error-formater');
+const Logger = require('mini-logger');
+const utility = require('utility');
+const util = require('util');
+const os = require('os');
+const config = require('../config');
+const mail = require('./mail');
 
-var formater = require('error-formater');
-var Logger = require('mini-logger');
-var utility = require('utility');
-var util = require('util');
-var config = require('../config');
-var mail = require('./mail');
+const isTEST = process.env.NODE_ENV === 'test';
+const categories = ['sync_info', 'sync_error'];
 
-var isTEST = process.env.NODE_ENV === 'test';
-var categories = ['sync_info', 'sync_error'];
-
-var logger = module.exports = Logger({
+const logger = module.exports = Logger({
   categories: categories,
   dir: config.logdir,
   duration: '1d',
   format: '[{category}.]YYYY-MM-DD[.log]',
   stdout: config.debug && !isTEST,
-  errorFormater: errorFormater
+  errorFormater: errorFormater,
+  seperator: os.EOL,
 });
 
-var to = [];
+const to = [];
 for (var user in config.admins) {
   to.push(config.admins[user]);
 }
 
 function errorFormater(err) {
-  var msg = formater.both(err);
+  const msg = formater.both(err);
   mail.error(to, msg.json.name, msg.text);
   return msg.text;
 }
 
 logger.syncInfo = function () {
-  var args = [].slice.call(arguments);
+  const args = [].slice.call(arguments);
   if (typeof args[0] === 'string') {
     args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
   }
+  if (debug.enabled) {
+    debug.apply(debug, args);
+  }
   logger.sync_info.apply(logger, args);
 };
 
 logger.syncError =function () {
-  var args = [].slice.call(arguments);
+  const args = [].slice.call(arguments);
   if (typeof args[0] === 'string') {
     args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
   }
+  if (debug.enabled) {
+    debug.apply(debug, args);
+  }
   logger.sync_error.apply(logger, arguments);
 };

config/index.js

@@ -1,18 +1,5 @@
-/**
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var mkdirp = require('mkdirp');
 var copy = require('copy-to');
 var path = require('path');
@@ -26,6 +13,7 @@ var dataDir = path.join(process.env.HOME || root, '.cnpmjs.org');
 
 var config = {
   version: version,
+  dataDir: dataDir,
 
   /**
    * Cluster mode
@@ -209,6 +197,10 @@ var config = {
   // sync devDependencies or not, default is false
   syncDevDependencies: false,
 
+  // changes streaming sync
+  syncChangesStream: false,
+  handleSyncRegistry: 'http://127.0.0.1:7001',
+
   // badge subject on http://shields.io/
   badgePrefixURL: 'https://img.shields.io/badge',
   badgeSubject: 'cnpm',

controllers/sync_module_worker.js

@@ -1,18 +1,5 @@
-/**!
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:sync_module_worker');
 var co = require('co');
 var gather = require('co-gather');
@@ -72,6 +59,7 @@ function SyncModuleWorker(options) {
 
   this.successes = [];
   this.fails = [];
+  this.updates = [];
 }
 
 util.inherits(SyncModuleWorker, EventEmitter);
@@ -140,7 +128,7 @@ SyncModuleWorker.prototype.start = function () {
     // sync upstream
     if (that.syncUpstreamFirst) {
       try {
-        yield* that.syncUpstream(that.startName);
+        yield that.syncUpstream(that.startName);
       } catch (err) {
         logger.error(err);
       }
@@ -193,7 +181,7 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
   var that = this;
   // relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
   defer.setImmediate(function* () {
-    yield* that.next(concurrencyId);
+    yield that.next(concurrencyId);
   });
 };
 
@@ -313,13 +301,13 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   if (common.isPrivateScopedPackage(name)) {
     this.log('[c#%d] [%s] ignore sync private scoped %j package',
       concurrencyId, name, config.scopes);
-    yield* this._doneOne(concurrencyId, name, true);
+    yield this._doneOne(concurrencyId, name, true);
     return;
   }
 
   // get from npm
   try {
-    var result = yield* npmSerivce.request('/' + name.replace('/', '%2f'));
+    var result = yield npmSerivce.request('/' + name.replace('/', '%2f'));
     pkg = result.data;
     status = result.status;
   } catch (err) {
@@ -346,7 +334,7 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   if (!pkg) {
     that.log('[c#%s] [error] [%s] get package error: package not exists, status: %s',
       concurrencyId, name, status);
-    yield* that._doneOne(concurrencyId, name, true);
+    yield that._doneOne(concurrencyId, name, true);
     return;
   }
 
@@ -354,31 +342,36 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
 
   if (unpublishedInfo) {
     try {
-      yield* that._unpublished(name, unpublishedInfo);
+      yield that._unpublished(name, unpublishedInfo);
     } catch (err) {
       that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
-      yield* that._doneOne(concurrencyId, name, false);
+      yield that._doneOne(concurrencyId, name, false);
       return;
     }
-    return yield* that._doneOne(concurrencyId, name, true);
+    return yield that._doneOne(concurrencyId, name, true);
   }
 
   var versions;
   try {
-    versions = yield* that._sync(name, pkg);
+    versions = yield that._sync(name, pkg);
   } catch (err) {
     that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
-    yield* that._doneOne(concurrencyId, name, false);
+    yield that._doneOne(concurrencyId, name, false);
     return;
   }
 
+  // has new version
+  if (versions.length > 0) {
+    that.updates.push(name);
+  }
+
   this.log('[c#%d] [%s] synced success, %d versions: %s',
     concurrencyId, name, versions.length, versions.join(', '));
-  yield* this._doneOne(concurrencyId, name, true);
+  yield this._doneOne(concurrencyId, name, true);
 };
 
 function* _listStarUsers(modName) {
-  var users = yield* packageService.listStarUserNames(modName);
+  var users = yield packageService.listStarUserNames(modName);
   var userMap = {};
   users.forEach(function (user) {
     userMap[user] = true;
@@ -676,7 +669,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
       continue;
     }
     try {
-      yield* that._syncOneVersion(index, syncModule);
+      yield that._syncOneVersion(index, syncModule);
       syncedVersionNames.push(syncModule.version);
     } catch (err) {
       that.log('    [%s:%d] sync error, version: %s, %s: %s',
@@ -936,7 +929,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   }
 
   // add module dependence
-  yield* packageService.addDependencies(sourcePackage.name, dependencies);
+  yield packageService.addDependencies(sourcePackage.name, dependencies);
 
   var shasum = crypto.createHash('sha1');
   var dataSize = 0;
@@ -958,6 +951,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -974,6 +968,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       var err = new Error('Download ' + downurl + ' file size is zero');
       err.name = 'DownloadTarballSizeZeroError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -984,6 +979,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
         ' not match ' + sourcePackage.dist.shasum);
       err.name = 'DownloadTarballShasumError';
       err.data = sourcePackage;
+      logger.syncInfo('[sync_module_worker] %s', err.message);
       throw err;
     }
 
@@ -1002,8 +998,9 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       throw err;
     }
     logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result);
-    var r = yield *afterUpload(result);
-    logger.syncInfo('[sync_module_worker] sync %s@%s done!', sourcePackage.name, sourcePackage.version);
+    var r = yield afterUpload(result);
+    logger.syncInfo('[sync_module_worker] sync %s@%s done!',
+      sourcePackage.name, sourcePackage.version);
     return r;
   } finally {
     // remove tmp file whatever
@@ -1047,7 +1044,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     }
 
     mod.package.dist = dist;
-    var r = yield* packageService.saveModule(mod);
+    var r = yield packageService.saveModule(mod);
 
     that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, '
       + 'size: %d, publish_time: %j, publish on cnpm: %s',

controllers/total.js

@@ -1,18 +1,5 @@
-/**!
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 const Total = require('../services/total');
 const version = require('../package.json').version;
 const config = require('../config');

controllers/web/package/show.js

@@ -1,18 +1,5 @@
-/**!
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:controllers:web:package:show');
 var bytes = require('bytes');
 var giturl = require('giturl');
@@ -183,6 +170,17 @@ module.exports = function* show(next) {
     }
   }
 
+  if (pkg._publish_on_cnpm) {
+    pkg.isPrivate = true;
+  } else {
+    pkg.isPrivate = false;
+    // add security check badge
+    pkg.snyk = {
+      badge: `https://snyk.io/test/npm/${pkg.name}/badge.svg?style=flat-square`,
+      url: `https://snyk.io/test/npm/${pkg.name}`,
+    };
+  }
+
   yield this.render('package', {
     title: 'Package - ' + pkg.name,
     package: pkg,

dispatch.js

@@ -1,18 +1,5 @@
-/**!
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var childProcess = require('child_process');
 var path = require('path');
 var util = require('util');

package.json

@@ -1,9 +1,10 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.10.1",
+  "version": "2.12.2",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
+    "dev": "DEBUG=cnpm* node dispatch.js",
     "test": "make jshint && make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
@@ -13,46 +14,49 @@
     "cnpmjs.org": "bin/cli.js"
   },
   "dependencies": {
-    "agentkeepalive": "~2.1.0",
-    "bytes": "~2.4.0",
-    "cfork": "~1.4.0",
-    "co": "~4.6.0",
-    "co-defer": "~1.0.0",
-    "co-gather": "~0.0.1",
-    "co-sleep": "~0.0.1",
-    "commander": "~2.9.0",
-    "copy-to": "~2.0.1",
-    "debug": "~2.2.0",
-    "error-formater": "~1.0.3",
-    "fs-cnpm": "~1.2.0",
+    "agentkeepalive": "^2.2.0",
+    "await-event": "^1.0.0",
+    "bytes": "^2.4.0",
+    "cfork": "^1.5.1",
+    "changes-stream": "^1.1.0",
+    "co": "^4.6.0",
+    "co-defer": "^1.0.0",
+    "co-gather": "^0.0.1",
+    "co-sleep": "^0.0.1",
+    "commander": "^2.9.0",
+    "copy-to": "^2.0.1",
+    "debug": "^2.2.0",
+    "error-formater": "^1.0.3",
+    "fs-cnpm": "^1.2.0",
     "giturl": "^1.0.0",
-    "graceful": "~1.0.0",
-    "gravatar": "~1.5.0",
-    "humanize-ms": "~1.2.0",
-    "humanize-number": "~0.0.2",
-    "is-type-of": "~1.0.0",
-    "kcors": "~1.2.1",
-    "koa": "~1.2.0",
-    "koa-limit": "~1.0.2",
-    "koa-markdown": "~2.0.1",
-    "koa-middlewares": "~2.1.0",
-    "koa-mock": "~1.6.1",
-    "koa-safe-jsonp": "~0.3.0",
-    "markdown-it": "~3.0.6",
-    "mime": "~1.3.4",
-    "mini-logger": "~1.1.0",
-    "mkdirp": "~0.5.0",
-    "moment": "~2.12.0",
-    "mysql": "~2.10.2",
-    "nodemailer": "~1.3.0",
-    "semver": "~5.1.0",
-    "sequelize": "~3.21.0",
-    "thunkify-wrap": "~1.0.4",
-    "treekill": "~1.0.0",
+    "graceful": "^1.0.0",
+    "gravatar": "^1.5.0",
+    "humanize-ms": "^1.2.0",
+    "humanize-number": "^0.0.2",
+    "is-type-of": "^1.0.0",
+    "kcors": "^1.2.1",
+    "koa": "^1.2.0",
+    "koa-limit": "^1.0.2",
+    "koa-markdown": "^2.0.1",
+    "koa-middlewares": "^2.1.0",
+    "koa-mock": "^1.6.1",
+    "koa-safe-jsonp": "^0.3.1",
+    "markdown-it": "^3.0.6",
+    "mime": "^1.3.4",
+    "mini-logger": "^1.1.1",
+    "mkdirp": "^0.5.0",
+    "moment": "^2.12.0",
+    "mysql": "^2.10.2",
+    "mz": "^2.4.0",
+    "nodemailer": "^1.3.0",
+    "semver": "^5.2.0",
+    "sequelize": "^3.23.4",
+    "thunkify-wrap": "^1.0.4",
+    "treekill": "^1.0.0",
     "tunnel-agent": "^0.4.0",
-    "urllib": "~2.9.0",
-    "utility": "~1.7.0",
-    "xss": "~0.2.10"
+    "urllib": "^2.11.0",
+    "utility": "^1.8.0",
+    "xss": "^0.2.13"
   },
   "devDependencies": {
     "autod": "*",
@@ -64,12 +68,13 @@
     "mocha": "*",
     "node-dev": "*",
     "pedding": "*",
-    "pg": "~4.5.2",
-    "pg-hstore": "~2.3.2",
-    "should": "~8.4.0",
+    "pg": "5",
+    "pg-hstore": "2",
+    "should": "8",
     "should-http": "*",
     "sqlite3": "*",
-    "supertest": "*",
+    "supertest": "1",
+    "supertest-as-promised": "3",
     "thunk-mocha": "1"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",

public/css/main.css

@@ -201,6 +201,12 @@ pre code {
   font-size: 28px;
   line-height: 48px;
 }
+
+.pack-ver a.badge-link {
+  font-size: inherit;
+  border-bottom: 0px;
+}
+
 .pack-lastup {
   font-size: 14px;
   display: inline-block;

services/package.js

@@ -75,7 +75,7 @@ exports.getModuleByTag = function* (name, tag) {
 };
 
 exports.getModuleByRange = function* (name, range) {
-  var rows = yield exports.listModulesByName(name);
+  var rows = yield exports.listModulesByName(name, [ 'id', 'version' ]);
   var versionMap = {};
   var versions = rows.map(function(row) {
     versionMap[row.version] = row;
@@ -85,7 +85,12 @@ exports.getModuleByRange = function* (name, range) {
   });
 
   var version = semver.maxSatisfying(versions, range);
-  return versionMap[version];
+  if (!versionMap[version]) {
+    return null;
+  }
+
+  var id = versionMap[version].id;
+  return yield exports.getModuleById(id);
 };
 
 exports.getLatestModule = function* (name) {
@@ -241,12 +246,13 @@ exports.listAllPublicModuleNames = function* () {
   });
 };
 
-exports.listModulesByName = function* (moduleName) {
+exports.listModulesByName = function* (moduleName, attributes) {
   var mods = yield Module.findAll({
     where: {
       name: moduleName
     },
-    order: [ ['id', 'DESC'] ]
+    order: [ ['id', 'DESC'] ],
+    attributes,
   });
 
   for (var mod of mods) {

sync/changes_stream_syncer.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const ChangesStream = require('changes-stream');
+const path = require('path');
+const fs = require('mz/fs');
+const urllib = require('urllib');
+const streamAwait = require('await-event');
+const logger = require('../common/logger');
+const config = require('../config');
+
+const db = 'https://replicate.npmjs.com';
+const lastSeqFile = path.join(config.dataDir, '.cnpmjs.org.last_seq.txt');
+let _STREAM_ID = 0;
+
+module.exports = function* sync() {
+  const since = yield getLastSequence();
+  const streamId = _STREAM_ID++;
+  let changesCount = 0;
+  logger.syncInfo('start changes stream#%d, since: %s', streamId, since);
+  const changes = new ChangesStream({
+    db,
+    since,
+    include_docs: false,
+  });
+  changes.await = streamAwait;
+  changes.on('data', change => {
+    changesCount++;
+    logger.syncInfo('stream#%d get change#%d: %j', streamId, changesCount, change);
+    syncPackage(change);
+  });
+
+  try {
+    yield changes.await('error');
+  } catch (err) {
+    // make sure changes steam is destroy
+    changes.destroy();
+    err.message += `, stream#${streamId}, changesCount#${changesCount}`;
+    throw err;
+  }
+};
+
+function syncPackage(change) {
+  const url = `${config.handleSyncRegistry}/${change.id}/sync`;
+  urllib.request(url, {
+    method: 'PUT',
+    dataType: 'json',
+    timeout: 10000,
+  }, (err, data) => {
+    if (err) {
+      logger.syncInfo('%s:%s PUT %s error: %s, retry after 5s',
+        change.seq, change.id, url, err);
+      logger.syncError(err);
+      syncPackage(change);
+      setTimeout(() => syncPackage(change), 5000);
+    } else {
+      saveLastSequence(change.seq);
+      logger.syncInfo('%s:%s sync request sent, log: %s/log/%s',
+        change.seq, change.id, url, data.logId);
+    }
+  });
+}
+
+function* getLastSequence() {
+  let lastSeq;
+  if (yield fs.exists(lastSeqFile)) {
+    lastSeq = yield fs.readFile(lastSeqFile, 'utf8');
+    lastSeq = Number(lastSeq);
+  }
+  if (!lastSeq) {
+    lastSeq = 2649694;
+  }
+  // const r = yield urllib.request(db, {
+  //   dataType: 'json',
+  //   timeout: 15000,
+  // });
+  // logger.syncInfo('get registry info: %j', r.data);
+  // if (lastSeq < r.data.update_seq) {
+  //   lastSeq = r.data.update_seq;
+  // }
+  return lastSeq;
+}
+
+function saveLastSequence(seq) {
+  fs.writeFile(lastSeqFile, String(seq), () => {});
+}

sync/index.js

@@ -1,36 +1,23 @@
-/**
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+const debug = require('debug')('cnpmjs.org:sync:index');
+const co = require('co');
+const ms = require('humanize-ms');
+const util = require('util');
+const config = require('../config');
+const mail = require('../common/mail');
+const logger = require('../common/logger');
+const totalService = require('../services/total');
 
-var debug = require('debug')('cnpmjs.org:sync:index');
-var co = require('co');
-var ms = require('humanize-ms');
-var util = require('util');
-var config = require('../config');
-var mail = require('../common/mail');
-var logger = require('../common/logger');
-var totalService = require('../services/total');
-
-var sync = null;
+let sync = null;
 
 switch (config.syncModel) {
-case 'all':
-  sync = require('./sync_all');
-  break;
-case 'exist':
-  sync = require('./sync_exist');
-  break;
+  case 'all':
+    sync = require('./sync_all');
+    break;
+  case 'exist':
+    sync = require('./sync_exist');
+    break;
 }
 
 if (!sync && config.enableCluster) {
@@ -38,34 +25,72 @@ if (!sync && config.enableCluster) {
   process.exit(0);
 }
 
-console.log('[%s] [sync_worker:%s] syncing with %s mode',
-  Date(), process.pid, config.syncModel);
+console.log('[%s] [sync_worker:%s] syncing with %s mode, changesStreamingSync: %s',
+  Date(), process.pid, config.syncModel, config.changesStreamingSync);
 
 function onerror(err) {
   logger.error(err);
 }
 
-//set sync_status = 0 at first
+// set sync_status = 0 at first
 co(function* () {
   yield totalService.updateSyncStatus(0);
   yield checkSyncStatus();
 }).catch(onerror);
 
-var syncInterval = ms(config.syncInterval);
-var minSyncInterval = ms('5m');
+let syncInterval = ms(config.syncInterval);
+const minSyncInterval = ms('5m');
 if (!syncInterval || syncInterval < minSyncInterval) {
   syncInterval = minSyncInterval;
 }
 
-// the same time only sync once
-var syncing = false;
-var syncFn = co.wrap(function* () {
-  debug('mode: %s, syncing: %s', config.syncModel, syncing);
-  if (!syncing) {
+if (sync) {
+  // the same time only sync once
+  let syncing = false;
+  const syncFn = co.wrap(function*() {
+    debug('mode: %s, syncing: %s', config.syncModel, syncing);
+    if (!syncing) {
+      syncing = true;
+      debug('start syncing');
+      let data;
+      let error;
+      try {
+        data = yield sync();
+      } catch (err) {
+        error = err;
+        error.message += ' (sync package error)';
+        logger.syncError(error);
+      }
+      data && logger.syncInfo(data);
+      if (!config.debug) {
+        sendMailToAdmin(error, data, new Date());
+      }
+      syncing = false;
+    }
+
+    // check last_sync_time and last_exist_sync_time
+    yield checkSyncStatus();
+  });
+
+  syncFn().catch(onerror);
+  setInterval(() => syncFn().catch(onerror), syncInterval);
+}
+
+/**
+ * sync popular modules
+ */
+
+if (config.syncPopular) {
+  const sync = require('./sync_popular');
+  let syncing = false;
+  const syncFn = co.wrap(function*() {
+    if (syncing) {
+      return;
+    }
     syncing = true;
-    debug('start syncing');
-    var data;
-    var error;
+    logger.syncInfo('Start syncing popular modules...');
+    let data;
+    let error;
     try {
       data = yield sync();
     } catch (err) {
@@ -73,76 +98,53 @@ var syncFn = co.wrap(function* () {
       error.message += ' (sync package error)';
       logger.syncError(error);
     }
-    data && logger.syncInfo(data);
+
+    if (data) {
+      logger.syncInfo(data);
+    }
     if (!config.debug) {
       sendMailToAdmin(error, data, new Date());
     }
     syncing = false;
-  }
+  });
 
-  // check last_sync_time and last_exist_sync_time
-  yield checkSyncStatus();
-});
-
-if (sync) {
   syncFn().catch(onerror);
-  setInterval(function () {
-    syncFn().catch(onerror);
-  }, syncInterval);
+  setInterval(() => syncFn().catch(onerror), ms(config.syncPopularInterval));
 }
 
-/**
- * sync popular modules
- */
+if (config.syncChangesStream) {
+  const sync = require('./changes_stream_syncer');
+  let syncing = false;
+  const syncFn = co.wrap(function*() {
+    if (syncing) {
+      return;
+    }
+    syncing = true;
+    logger.syncInfo('Start changes stream syncing...');
+    try {
+      yield sync();
+    } catch (err) {
+      err.message += ' (sync changes stream error)';
+      logger.syncError(err);
+    }
+    syncing = false;
+  });
 
-var startSyncPopular = require('./sync_popular');
-var syncingPopular = false;
-var syncPopularFn = co.wrap(function* syncPopular() {
-  if (syncingPopular) {
-    return;
-  }
-  syncingPopular = true;
-  logger.syncInfo('Start syncing popular modules...');
-  var data;
-  var error;
-  try {
-    data = yield startSyncPopular();
-  } catch (err) {
-    error = err;
-    error.message += ' (sync package error)';
-    logger.syncError(error);
-  }
-
-  if (data) {
-    logger.syncInfo(data);
-  }
-  if (!config.debug) {
-    sendMailToAdmin(error, data, new Date());
-  }
-
-  syncingPopular = false;
-});
-
-if (config.syncPopular) {
-  syncPopularFn().catch(onerror);
-  setInterval(function () {
-    syncPopularFn().catch(onerror);
-  }, ms(config.syncPopularInterval));
-} else {
-  logger.syncInfo('sync popular module disable');
+  syncFn().catch(onerror);
+  setInterval(() => syncFn().catch(onerror), ms('1m'));
 }
 
 function sendMailToAdmin(err, result, syncTime) {
   result = result || {};
-  var to = [];
+  const to = [];
   for (var name in config.admins) {
     to.push(config.admins[name]);
   }
   debug('Send email to all admins: %j, with err message: %s, result: %j, start sync time: %s.',
     to, err ? err.message : '', result, syncTime);
-  var subject;
-  var type;
-  var html;
+  let subject;
+  let type;
+  let html;
   if (err) {
     // ignore 503 error
     if (err.status === 503) {
@@ -169,7 +171,7 @@ function sendMailToAdmin(err, result, syncTime) {
   debug('send email with type: %s, subject: %s, html: %s', type, subject, html);
   logger.syncInfo('send email with type: %s, subject: %s, html: %s', type, subject, html);
   if (type && type !== 'log') {
-    mail[type](to, subject, html, function (err) {
+    mail[type](to, subject, html, err => {
       if (err) {
         logger.error(err);
       }
@@ -182,8 +184,8 @@ function* checkSyncStatus() {
   if (!config.sourceNpmRegistryIsCNpm) {
     return;
   }
-  var total = yield totalService.getTotalInfo();
-  var lastSyncTime;
+  const total = yield totalService.getTotalInfo();
+  let lastSyncTime;
   if (config.syncModel === 'all') {
     lastSyncTime = total.last_sync_time;
   } else if (config.syncModel === 'exist') {
@@ -193,11 +195,11 @@ function* checkSyncStatus() {
   if (!lastSyncTime) {
     return;
   }
-  var diff = Date.now() - lastSyncTime;
-  var oneDay = 3600000 * 24;
-  var maxTime = Math.max(oneDay, syncInterval * 2);
+  const diff = Date.now() - lastSyncTime;
+  const oneDay = 3600000 * 24;
+  const maxTime = Math.max(oneDay, syncInterval * 2);
   if (diff > maxTime) {
-    var err = new Error('Last sync time is expired in ' + diff + ' ms, lastSyncTime: ' +
+    const err = new Error('Last sync time is expired in ' + diff + ' ms, lastSyncTime: ' +
       new Date(lastSyncTime) + ', maxTime: ' + maxTime + ' ms');
     err.name = 'SyncExpiredError';
     sendMailToAdmin(err, null, new Date());

sync/status.js

@@ -1,19 +1,5 @@
-/*!
- * cnpmjs.org - sync/status.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:sync:status');
 var co = require('co');
 var Total = require('../services/total');
@@ -36,7 +22,7 @@ Status.prototype.log = function (syncDone) {
     lastSyncModule: this.lastSyncModule,
   };
   co(function* () {
-    yield* Total.updateSyncNum(params);
+    yield Total.updateSyncNum(params);
   }).catch(function () {});
 };
 

sync/sync_all.js

@@ -1,20 +1,5 @@
-/**!
- * cnpmjs.org - sync/sync_all.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <m@fengmk2.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var thunkify = require('thunkify-wrap');
 var config = require('../config');
 var Status = require('./status');
@@ -72,19 +57,22 @@ module.exports = function* sync() {
     concurrency: config.syncConcurrency,
     syncUpstreamFirst: false,
   });
-  Status.init({need: packages.length}, worker);
+  Status.init({
+    need: packages.length,
+  }, worker);
   worker.start();
   var end = thunkify.event(worker);
   yield end();
 
-  logger.syncInfo('All packages sync done, successes %d, fails %d',
-      worker.successes.length, worker.fails.length);
+  logger.syncInfo('All packages sync done, successes %d, fails %d, updates %d',
+      worker.successes.length, worker.fails.length, worker.updates.length);
   //only when all succss, set last sync time
   if (!worker.fails.length) {
-    yield* totalService.setLastSyncTime(syncTime);
+    yield totalService.setLastSyncTime(syncTime);
   }
   return {
     successes: worker.successes,
-    fails: worker.fails
+    fails: worker.fails,
+    updates: worker.updates,
   };
 };

sync/sync_exist.js

@@ -1,17 +1,5 @@
-/*!
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:sync:sync_exist');
 var Status = require('./status');
 var thunkify = require('thunkify-wrap');

sync/sync_popular.js

@@ -1,19 +1,5 @@
-/*!
- * cnpmjs.org - sync/sync_popular.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var debug = require('debug')('cnpmjs.org:sync:sync_popular');
 var thunkify = require('thunkify-wrap');
 var config = require('../config');

sync/sync_since.js

@@ -1,19 +1,5 @@
-/**!
- * sync packages since by some days ago
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 const thunkify = require('thunkify-wrap');
 const co = require('co');
 const ms = require('humanize-ms');

test/controllers/web/package/show.test.js

@@ -1,28 +1,14 @@
-/*!
- * cnpmjs.org - test/controllers/web/package/show.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
-var request = require('supertest');
+var request = require('supertest-as-promised');
 var mm = require('mm');
 var config = require('../../../../config');
 var app = require('../../../../servers/web');
 var registry = require('../../../../servers/registry');
 var utils = require('../../../utils');
 
-describe('controllers/web/package/show.test.js', function () {
+describe('controllers/web/package/show.test.js', () => {
   before(function (done) {
     var pkg = utils.getPackage('@cnpmtest/testmodule-web-show', '0.0.1', utils.admin);
     pkg.versions['0.0.1'].dependencies = {
@@ -113,17 +99,17 @@ describe('controllers/web/package/show.test.js', function () {
     });
   });
 
-  describe('unpublished package', function () {
-    before(function (done) {
-      utils.sync('tfs', done);
+  describe('unpublished package', () => {
+    before(done => {
+      utils.sync('mk2testmodule', done);
     });
 
-    it('should display unpublished info', function (done) {
+    it('should display unpublished info', () => {
       mm(config, 'syncModel', 'all');
-      request(app)
-      .get('/package/moduletest1')
-      .expect(200)
-      .expect(/This package has been unpublished\./, done);
+      return request(app)
+        .get('/package/mk2testmodule')
+        .expect(200)
+        .expect(/This package has been unpublished\./);
     });
   });
 
@@ -149,20 +135,22 @@ describe('controllers/web/package/show.test.js', function () {
     });
   });
 
-  describe('show npm package', function () {
-    before(function (done) {
+  describe('show npm package', () => {
+    before(done => {
       mm(config, 'syncModel', 'exists');
       utils.sync('pedding', done);
     });
 
-    it('should show pedding package info and contributors', function (done) {
+    it('should show pedding package info and contributors', () => {
       mm(config, 'syncModel', 'exists');
-      request(app)
-      .get('/package/pedding')
-      .expect(200)
-      // https://github.com/cnpm/cnpmjs.org/issues/497
-      .expect(/by <a href="\/\~fengmk2">fengmk2<\/a>/)
-      .expect(/pedding/, done);
+      return request(app)
+        .get('/package/pedding')
+        .expect(200)
+        // https://github.com/cnpm/cnpmjs.org/issues/497
+        .expect(/by <a href="\/\~fengmk2">fengmk2<\/a>/)
+        // snyk link
+        .expect(/<a class="badge-link" href="https:\/\/snyk\.io\/test\/npm\/pedding" target="_blank"><img title="Known Vulnerabilities" src="https:\/\/snyk\.io\/test\/npm\/pedding\/badge\.svg\?style=flat-square"><\/a>/)
+        .expect(/pedding/);
     });
   });
 });

view/web/package.html

@@ -15,13 +15,16 @@
       <% } %>
       <div class="pack-ver">
         <img title="<%= package.version %>" src="/badge/v/<%= package.name %>.svg?style=flat-square">
-        <% if (package.engines) {%>
+        <% if (package.engines) { %>
           <% for (var k in package.engines) {
             var engine = package.engines[k];
           %>
             <img title="<%= engine.title %>" src="<%- engine.badgeURL %>">
         <% }
-        }%>
+        } %>
+        <% if (package.snyk) { %>
+        <a class="badge-link" href="<%= package.snyk.url %>" target="_blank"><img title="Known Vulnerabilities" src="<%- package.snyk.badge %>"></a>
+        <% } %>
       </div>
       <div class="pack-repo">
         <% if (package.fromNow) { %>
@@ -52,7 +55,7 @@
 
         <% if (package.license || package.repository || package.bugs.url) {%> &middot; <% } %>
 
-        <% if (!package._publish_on_cnpm) { %>
+        <% if (!package.isPrivate) { %>
         <a href="https://www.npmjs.org/package/<%= package.name %>" target="_blank">Original npm</a>
         <% } %>
 
@@ -64,7 +67,7 @@
       <div class="pack-install">
         <pre class="sh">$ <%- config.npmClientName %> install <%= package.name %> <% if (package.preferGlobal) { %>-g<% } %></pre>
         <div class="pack-sync">
-          <% if (package._publish_on_cnpm) { %>
+          <% if (package.isPrivate) { %>
             Private package
           <% } else { %>
             <a href="/sync/<%= package.name %>" target="_blank">SYNC</a> missed versions from