Release 2.6.0

feat(sync): monitor sync status

History.md

@@ -1,4 +1,30 @@
 
+2.6.0 / 2016-01-12
+==================
+
+  * feat(sync): monitor sync status
+  * chore(package): update agentkeepalive to version 2.0.3
+  * fix SequelizeDatabaseError: ER_NO_SUCH_TABLE: Table 'qnpm.total' doesn't exist\nreproduce this bug:\nthe first startup of cnpmjs.org
+  * chore(package): update moment to version 2.11.0
+  * chore(package): update xss to version 0.2.10
+  * chore(package): update pg-hstore to version 2.3.2
+  * chore(package): update mini-logger to version 1.1.0
+  * chore(package): update urllib to version 2.6.0
+  * fix: row.package will json parse error
+  * remove bluebird
+  * chore(package): update utility to version 1.6.0
+
+2.5.1 / 2015-12-02
+==================
+
+ * chore(package): update bluebird to version 3.0.6
+ * fix: SequelizeDatabaseError
+ * fix(dist_tag): disable delete latest tag
+ * feat: count total private pkgs
+ * fix: use isoweek. a week start from monday
+ * chore(package): update xss to version 0.2.8
+ * chore(package): update semver to version 5.1.0
+
 2.5.0 / 2015-11-17
 ==================
 

Makefile

@@ -112,7 +112,7 @@ autod: install
 	@node_modules/.bin/autod -w \
 		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
-		--dep bluebird,mysql \
+		--dep mysql \
 		--keep should,supertest,should-http,chunkstream,mm,pedding
 	@$(MAKE) install
 

bin/cli.js

@@ -83,9 +83,10 @@ function start(options) {
   debug('save config %s to %s', configJSON, configfile);
 
   // if sqlite db file not exists, init first
-  initDatabase();
+  initDatabase(function() {
+    require('../dispatch');
+  });
 
-  require('../dispatch');
   fs.writeFileSync(path.join(dataDir, 'pid'), process.pid + '');
 }
 
@@ -105,7 +106,7 @@ function stop(options) {
   }
 }
 
-function initDatabase() {
+function initDatabase(callback) {
   var models = require('../models');
 
   models.sequelize.sync({ force: false })
@@ -117,6 +118,7 @@ function initDatabase() {
           throw err;
         } else {
           console.log('[models/init_script.js] `sqlite` sequelize sync and init success');
+          callback();
         }
       });
     })

controllers/registry/package/dist_tag.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - controllers/registry/package/dist_tag.js
- *
- * Copyright(c) fengmk2 and other contributors.
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -25,7 +23,7 @@ function ok() {
 // GET /-/package/:pkg/dist-tags -- returns the package's dist-tags
 exports.index = function* () {
   var name = this.params.name || this.params[0];
-  var rows = yield* packageService.listModuleTags(name);
+  var rows = yield packageService.listModuleTags(name);
   var tags = {};
   for (var i = 0; i < rows.length; i++) {
     var row = rows[i];
@@ -79,6 +77,14 @@ exports.set = function* () {
 exports.destroy = function* () {
   var name = this.params.name || this.params[0];
   var tag = this.params.tag || this.params[1];
+  if (tag === 'latest') {
+    this.status = 400;
+    this.body = {
+      error: 'dist_tag_error',
+      reason: 'Can\'t not delete latest tag',
+    };
+    return;
+  }
   yield packageService.removeModuleTagsByNames(name, tag);
   this.body = ok();
 };

controllers/registry/package/list.js

@@ -1,11 +1,9 @@
-/**!
- * cnpmjs.org - controllers/registry/package/list.js
- *
- * Copyright(c) fengmk2 and other contributors.
+/**
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -130,6 +128,10 @@ module.exports = function* list() {
   for (var i = 0; i < rows.length; i++) {
     var row = rows[i];
     var pkg = row.package;
+    // pkg is string ... ignore it
+    if (typeof pkg === 'string') {
+      continue;
+    }
     common.setDownloadURL(pkg, this);
     pkg._cnpm_publish_time = row.publish_time;
     versions[pkg.version] = pkg;

controllers/sync.js

@@ -60,8 +60,13 @@ exports.sync = function* () {
 };
 
 exports.getSyncLog = function* (next) {
-  var logId = this.params.id || this.params[1];
+  var logId = Number(this.params.id || this.params[1]);
   var offset = Number(this.query.offset) || 0;
+
+  if (!logId) { // NaN
+    this.status = 404;
+    return;
+  }
   var row = yield* Log.get(logId);
   if (!row) {
     return yield* next;

controllers/utils.js

@@ -56,11 +56,11 @@ exports.getDownloadTotal = function* (name) {
   var end = moment();
   var start = end.clone().subtract(1, 'months').startOf('month');
   var lastday = end.clone().subtract(1, 'days').format('YYYY-MM-DD');
-  var lastweekStart = end.clone().subtract(1, 'weeks').startOf('week');
-  var lastweekEnd = lastweekStart.clone().endOf('week').format('YYYY-MM-DD');
+  var lastweekStart = end.clone().subtract(1, 'weeks').startOf('isoweek');
+  var lastweekEnd = lastweekStart.clone().endOf('isoweek').format('YYYY-MM-DD');
   var lastmonthEnd = start.clone().endOf('month').format('YYYY-MM-DD');
   var thismonthStart = end.clone().startOf('month').format('YYYY-MM-DD');
-  var thisweekStart = end.clone().startOf('week').format('YYYY-MM-DD');
+  var thisweekStart = end.clone().startOf('isoweek').format('YYYY-MM-DD');
   start = start.format('YYYY-MM-DD');
   end = end.format('YYYY-MM-DD');
   lastweekStart = lastweekStart.format('YYYY-MM-DD');

middleware/exists_package.js

@@ -1,11 +1,9 @@
 /**!
- * cnpmjs.org - middleware/exists_package.js
- *
  * Copyright(c) fengmk2 and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -13,8 +13,7 @@
     "cnpmjs.org": "bin/cli.js"
   },
   "dependencies": {
-    "agentkeepalive": "~1.2.0",
-    "bluebird": "~3.0.5",
+    "agentkeepalive": "~2.0.3",
     "bytes": "~2.2.0",
     "cfork": "~1.4.0",
     "co": "~4.6.0",
@@ -41,19 +40,19 @@
     "koa-safe-jsonp": "~0.3.0",
     "markdown-it": "~3.0.6",
     "mime": "~1.3.4",
-    "mini-logger": "~1.0.0",
+    "mini-logger": "~1.1.0",
     "mkdirp": "~0.5.0",
-    "moment": "~2.10.6",
+    "moment": "~2.11.0",
     "mysql": "~2.5.4",
     "nodemailer": "~1.3.0",
-    "semver": "~5.0.3",
+    "semver": "~5.1.0",
     "sequelize": "~2.0.1",
     "thunkify-wrap": "~1.0.4",
     "treekill": "~1.0.0",
     "tunnel-agent": "^0.4.0",
-    "urllib": "~2.5.0",
-    "utility": "~1.5.0",
-    "xss": "~0.1.20"
+    "urllib": "~2.6.0",
+    "utility": "~1.6.0",
+    "xss": "~0.2.10"
   },
   "devDependencies": {
     "autod": "*",
@@ -67,7 +66,7 @@
     "node-dev": "*",
     "pedding": "*",
     "pg": "~4.4.2",
-    "pg-hstore": "~2.3.1",
+    "pg-hstore": "~2.3.2",
     "should": "~7.1.0",
     "should-http": "*",
     "sqlite3": "*",

servers/registry.js

@@ -15,7 +15,6 @@
  * Module dependencies.
  */
 
-global.Promise = require('bluebird');
 var koa = require('koa');
 var app = module.exports = koa();
 var http = require('http');

servers/web.js

@@ -15,7 +15,6 @@
  * Module dependencies.
  */
 
-global.Promise = require('bluebird');
 var opensearch = require('../middleware/opensearch');
 var notFound = require('../middleware/web_not_found');
 var staticCache = require('../middleware/static');

services/module_log.js

@@ -50,5 +50,5 @@ exports.append = function* (id, log) {
 };
 
 exports.get = function* (id) {
-  return yield ModuleLog.find(Number(id));
+  return yield ModuleLog.find(id);
 };

services/package.js

@@ -1,4 +1,4 @@
-/**!
+/**
  * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
@@ -25,7 +25,6 @@ var ModuleUnpublished = models.ModuleUnpublished;
 var NpmModuleMaintainer = models.NpmModuleMaintainer;
 
 // module
-
 var _parseRow = function (row) {
   if (row.package.indexOf('%7B%22') === 0) {
     // now store package will encodeURIComponent() after JSON.stringify
@@ -234,10 +233,11 @@ exports.listModulesByName = function* (moduleName) {
     },
     order: [ ['id', 'DESC'] ]
   });
-  return mods.map(function (mod) {
+
+  for (var mod of mods) {
     parseRow(mod);
-    return mod;
-  });
+  }
+  return mods;
 };
 
 exports.getModuleLastModified = function* (name) {

services/total.js

@@ -1,11 +1,9 @@
-/**!
- * cnpmjs.org - services/total.js
- *
- * Copyright(c) fengmk2 and other contributors.
+/**
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';

sync/index.js

@@ -1,11 +1,10 @@
-/**!
- * cnpmjs.org - sync/index.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
+/**
+ * Copyright(c) cnpm and other contributors.
  * MIT Licensed
  *
  * Authors:
  *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -14,7 +13,6 @@
  * Module dependencies.
  */
 
-global.Promise = require('bluebird');
 var debug = require('debug')('cnpmjs.org:sync:index');
 var co = require('co');
 var ms = require('humanize-ms');
@@ -49,7 +47,8 @@ function onerror(err) {
 
 //set sync_status = 0 at first
 co(function* () {
-  yield* totalService.updateSyncStatus(0);
+  yield totalService.updateSyncStatus(0);
+  yield checkSyncStatus();
 }).catch(onerror);
 
 var syncInterval = ms(config.syncInterval);
@@ -68,7 +67,7 @@ var syncFn = co.wrap(function* () {
     var data;
     var error;
     try {
-      data = yield* sync();
+      data = yield sync();
     } catch (err) {
       error = err;
       error.message += ' (sync package error)';
@@ -80,6 +79,9 @@ var syncFn = co.wrap(function* () {
     }
     syncing = false;
   }
+
+  // check last_sync_time and last_exist_sync_time
+  yield checkSyncStatus();
 });
 
 if (sync) {
@@ -104,7 +106,7 @@ var syncPopularFn = co.wrap(function* syncPopular() {
   var data;
   var error;
   try {
-    data = yield* startSyncPopular();
+    data = yield startSyncPopular();
   } catch (err) {
     error = err;
     error.message += ' (sync package error)';
@@ -174,3 +176,23 @@ function sendMailToAdmin(err, result, syncTime) {
     });
   }
 }
+
+function* checkSyncStatus() {
+  var total = yield totalService.getTotalInfo();
+  var lastSyncTime;
+  if (config.syncModel === 'all') {
+    lastSyncTime = total.last_sync_time;
+  } else if (config.syncModel === 'exist') {
+    lastSyncTime = total.last_exist_sync_time;
+  }
+  debug('checkSyncStatus start, lastSyncTime: %s, syncInterval: %s', lastSyncTime, syncInterval);
+  if (!lastSyncTime) {
+    return;
+  }
+  var diff = Date.now() - lastSyncTime;
+  if (diff > syncInterval * 2) {
+    var err = new Error('Last sync time is expired in ' + diff + ' ms, lastSyncTime: ' + new Date(lastSyncTime));
+    err.name = 'SyncExpriedError';
+    sendMailToAdmin(err, null, new Date());
+  }
+}

test/controllers/registry/package/dist_tag.test.js

@@ -170,6 +170,18 @@ describe('controllers/registry/package/dist_tag.test.js', function () {
       });
     });
 
+    it('should not destroy latest tag', function(done) {
+      request(app.listen())
+      .delete('/-/package/@cnpmtest/dist_tag_test_module_destroy/dist-tags/latest')
+      .set('authorization', utils.otherUserAuth)
+      .set('content-type', 'application/json')
+      .expect({
+        error: 'dist_tag_error',
+        reason: 'Can\'t not delete latest tag',
+      })
+      .expect(400, done);
+    });
+
     it('should 404 destroy not exists tag', function (done) {
       request(app.listen())
       .delete('/-/package/@cnpmtest/dist_tag_test_module_destroy/dist-tags/not-exists')

test/controllers/sync.test.js

@@ -123,6 +123,12 @@ describe('controllers/sync.test.js', function () {
       .get('/sync/pedding/log/123123123')
       .expect(404, done);
     });
+
+    it('should 404 when log id not number', function (done) {
+      request(webApp.listen())
+      .get('/sync/pedding/log/info.php')
+      .expect(404, done);
+    });
   });
 
   describe('scope package', function () {

view/web/private.html

@@ -15,7 +15,7 @@
     var packages = scopes[scope];
   %>
     <h1>
-      Private packages in <%= scope %>
+      Private packages in <%= scope %>(total: <%= packages.length %>)
     </h1>
     <hr />
     <% if (!packages.length) { %>