fix: don't update __all__ downloads every times (#1417)

controllers/registry/package/download.js

@@ -76,19 +76,30 @@ module.exports = function* download(next) {
   this.body = yield downloadAsReadStream(dist.key);
 };
 
+var saving = false;
 defer.setInterval(function* () {
-  // save download count
-  var totals = [];
-  for (var name in _downloads) {
-    var count = _downloads[name];
-    totals.push([name, count]);
-  }
-  _downloads = {};
-
-  if (totals.length === 0) {
+  if (saving) {
     return;
   }
 
+  // save download count
+  var totals = [];
+  var allCount = 0;
+  for (var name in _downloads) {
+    var count = _downloads[name];
+    if (name !== '__all__') {
+      totals.push([name, count]);
+    }
+    allCount += count;
+  }
+  _downloads = {};
+
+  if (allCount === 0) {
+    return;
+  }
+
+  saving = true;
+  totals.push([ '__all__', allCount ]);
   debug('save download total: %j', totals);
 
   var date = utility.YYYYMMDD();
@@ -107,4 +118,5 @@ defer.setInterval(function* () {
       _downloads[name] = (_downloads[name] || 0) + count;
     }
   }
+  saving = false;
 }, 5000 + Math.ceil(Math.random() * 1000));

services/download_total.js

@@ -1,6 +1,8 @@
 'use strict';
 
 var utility = require('utility');
+var config = require('../config');
+var models = require('../models');
 var DownloadTotal = require('../models').DownloadTotal;
 
 exports.getModuleTotal = function* (name, start, end) {
@@ -41,50 +43,36 @@ exports.getTotalByName = function* (name) {
 
 exports.plusModuleTotal = function* (data) {
   var yearMonth = parseYearMonth(data.date);
-  // all module download total
   var row = yield DownloadTotal.find({
     where: {
-      name: '__all__',
-      date: yearMonth
+      name: data.name,
+      date: yearMonth,
     }
   });
   if (!row) {
     row = DownloadTotal.build({
-      name: '__all__',
+      name: data.name,
       date: yearMonth,
     });
-  }
-  var field = 'd' + data.date.substring(8, 10);
-  if (typeof row[field] === 'string') {
-    // pg bigint is string...
-    row[field] = utility.toSafeNumber(row[field]);
-  }
-  row[field] += data.count;
-  if (row.changed()) {
     yield row.save();
   }
-
-  row = yield DownloadTotal.find({
-    where: {
-      name: data.name,
-      date: yearMonth,
-    }
-  });
-  if (!row) {
-    row = DownloadTotal.build({
-      name: data.name,
-      date: yearMonth,
-    });
-  }
   var field = 'd' + data.date.substring(8, 10);
-  if (typeof row[field] === 'string') {
-    // pg bigint is string...
-    row[field] = utility.toSafeNumber(row[field]);
-  }
-  row[field] += data.count;
-  if (row.changed()) {
-    return yield row.save();
+  if (config.database.dialect === 'mysql') {
+    // mysql update set field = field + count
+    yield models.query(`UPDATE downloads SET ${field} = ${field} + ${data.count}, gmt_modified=? WHERE id = ?`,
+      [ new Date(), row.id ]);
+  } else {
+    // pg
+    if (typeof row[field] === 'string') {
+      // pg bigint is string...
+      row[field] = utility.toSafeNumber(row[field]);
+    }
+    row[field] += data.count;
+    if (row.changed()) {
+      return yield row.save();
+    }
   }
+
   return row;
 };
 

test/services/download_total.test.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - test/services/download_total.test.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var DownloadTotal = require('../../services/download_total');
 
 describe('test/services/download_total.test.js', function () {
@@ -25,6 +11,12 @@ describe('test/services/download_total.test.js', function () {
         count: 1000
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-10-21',
+        name: '__all__',
+        count: 1000
+      };
+      yield DownloadTotal.plusModuleTotal(data);
 
       data = {
         date: '2014-10-22',
@@ -32,6 +24,12 @@ describe('test/services/download_total.test.js', function () {
         count: 2
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-10-22',
+        name: '__all__',
+        count: 2
+      };
+      yield DownloadTotal.plusModuleTotal(data);
 
       var rows = yield DownloadTotal.getModuleTotal(
         'plusModuleTotal-module', '2014-10-21', '2014-10-21');
@@ -54,6 +52,12 @@ describe('test/services/download_total.test.js', function () {
         count: 3
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-10-21',
+        name: '__all__',
+        count: 3
+      };
+      yield DownloadTotal.plusModuleTotal(data);
       rows = yield DownloadTotal.getModuleTotal(
         'plusModuleTotal-module', '2014-10-21', '2014-10-21');
       rows.should.length(1);
@@ -66,18 +70,38 @@ describe('test/services/download_total.test.js', function () {
         count: 3
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-10-22',
+        name: '__all__',
+        count: 3
+      };
+      yield DownloadTotal.plusModuleTotal(data);
+
       data = {
         date: '2014-12-22',
         name: 'plusModuleTotal-module2',
         count: 3
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-12-22',
+        name: '__all__',
+        count: 3
+      };
+      yield DownloadTotal.plusModuleTotal(data);
+
       data = {
         date: '2014-12-21',
         name: 'plusModuleTotal-module2',
         count: 3
       };
       yield DownloadTotal.plusModuleTotal(data);
+      data = {
+        date: '2014-12-21',
+        name: '__all__',
+        count: 3
+      };
+      yield DownloadTotal.plusModuleTotal(data);
 
       rows = yield DownloadTotal.getTotal('2014-10-21', '2014-12-21');
       rows.should.length(3);