open-git/cnpmjs.org
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(download_total): optimize download total

Browse Source 
Use downloads table instead of download_total

Fixes #544
This commit is contained in:
fengmk2
2014-12-22 16:44:07 +08:00
parent 6c98c1f07f
commit 0689cc93ce
7 changed files with 597 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
controllers/registry/package/download_total.js
View File

@@ -14,10 +14,7 @@
* Module dependencies.
*/
var debug = require('debug')('cnpmjs.org:controllers:registry:package:download_total');
var DownloadTotal = require('../../../services/download_total');
var moment = require('moment');
var DATE_REG = /^\d{4}-\d{2}-\d{2}$/;
module.exports = function* downloadTotal () {
@@ -39,7 +36,7 @@ module.exports = function* downloadTotal () {
this.body = name
? yield getPackageTotal(name, range[0], range[1])
: yield getTotal(range[0], range[1]);
}
};
function* getPackageTotal(name, start, end) {
var res = yield DownloadTotal.getModuleTotal(name, start, end);

143
docs/Migrating-from-1.x-to-2.x.md Normal file
View File

@@ -0,0 +1,143 @@
# Migrating from 1.x to 2.x
2.x using [Sequelize] ORM to supports MySQL, MariaDB, SQLite or PostgreSQL databases.
## New download total table structure
### Create `downloads` table SQL
You should create `downloads` table first:
```sql
CREATE TABLE IF NOT EXISTS `downloads` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`date` int unsigned NOT NULL COMMENT 'YYYYMM format',
`d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
`d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
`d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
`d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
`d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
`d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
`d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
`d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
`d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
`d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
`d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
`d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
`d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
`d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
`d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
`d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
`d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
`d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
`d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
`d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
`d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
`d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
`d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
`d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
`d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
`d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
`d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
`d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
`d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
`d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
`d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
PRIMARY KEY (`id`),
UNIQUE KEY `name_date` (`name`, `date`),
KEY `date` (`date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
```
### Sync `download_total` to `downloads`
Then use [sync_download_total.js](../tools/sync_download_total.js) scrpt to sync datas from `download_total`:
```bash
$ node --harmony tools/sync_download_total.js
```
# `config.js` changes in 2.x
## New database config
```js
/**
* database config
*/
database: {
db: 'cnpmjs_test',
username: 'root',
password: '',
// the sql dialect of the database
// - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
dialect: 'sqlite',
// custom host; default: 127.0.0.1
host: '127.0.0.1',
// custom port; default: 3306
port: 3306,
// use pooling in order to reduce db connection overload and to increase speed
// currently only for mysql and postgresql (since v1.5.0)
pool: {
maxConnections: 10,
minConnections: 0,
maxIdleTime: 30000
},
// the storage engine for 'sqlite'
// default store into ~/cnpmjs.org.sqlite
storage: path.join(process.env.HOME || root, 'cnpmjs.org.sqlite'),
logging: !!process.env.SQL_DEBUG,
},
```
If you're still using MySQL and old config.js `mysqlServers: []` from 1.x:
```js
mysqlServers: [
{
host: '127.0.0.1',
port: 3306,
user: 'root',
password: ''
}
],
mysqlDatabase: 'cnpmjs_test',
mysqlMaxConnections: 4,
mysqlQueryTimeout: 5000,
```
We will do forward compat, and auto change old style config.js to:
```js
database: {
db: 'cnpmjs_test',
username: 'root',
password: '',
dialect: 'mysql',
host: '127.0.0.1',
port: 3306,
pool: {
maxConnections: 10,
minConnections: 0,
maxIdleTime: 30000
},
logging: !!process.env.SQL_DEBUG,
},
```
## remove `adaptScope`
`adaptScope: true | false` feature was removed.
[Sequelize]: http://sequelizejs.com/

62
docs/db.sql
View File

@@ -151,19 +151,61 @@ INSERT INTO total(name, gmt_modified) VALUES('total', now())
-- ALTER TABLE `total` ADD `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync'
-- ALTER TABLE `total` ADD `last_sync_module` varchar(100) NOT NULL COMMENT 'last sync success module name';
CREATE TABLE IF NOT EXISTS `download_total` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`date` datetime NOT NULL COMMENT 'YYYY-MM-DD format',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
PRIMARY KEY (`id`),
UNIQUE KEY `date_name` (`date`, `name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
-- CREATE TABLE IF NOT EXISTS `download_total` (
-- `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
-- `gmt_create` datetime NOT NULL COMMENT 'create time',
-- `gmt_modified` datetime NOT NULL COMMENT 'modified time',
-- `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format',
-- `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
-- `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
-- PRIMARY KEY (`id`),
-- UNIQUE KEY `date_name` (`date`, `name`)
-- ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
-- ALTER TABLE `download_total` CHANGE `name` `name` VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
-- ALTER TABLE `download_total` CHANGE `date` `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format';
CREATE TABLE IF NOT EXISTS `downloads` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',
`gmt_modified` datetime NOT NULL COMMENT 'modified time',
`name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
`date` int unsigned NOT NULL COMMENT 'YYYYMM format',
`d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
`d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
`d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
`d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
`d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
`d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
`d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
`d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
`d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
`d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
`d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
`d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
`d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
`d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
`d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
`d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
`d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
`d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
`d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
`d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
`d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
`d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
`d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
`d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
`d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
`d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
`d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
`d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
`d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
`d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
`d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
PRIMARY KEY (`id`),
UNIQUE KEY `name_date` (`name`, `date`),
KEY `date` (`date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
CREATE TABLE IF NOT EXISTS `module_deps` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
`gmt_create` datetime NOT NULL COMMENT 'create time',

242
models/download_total.js
View File

@@ -14,42 +14,256 @@
* Module dependencies.
*/
// CREATE TABLE IF NOT EXISTS `download_total` (
// CREATE TABLE IF NOT EXISTS `downloads` (
// `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
// `gmt_create` datetime NOT NULL COMMENT 'create time',
// `gmt_modified` datetime NOT NULL COMMENT 'modified time',
// `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format',
// `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
// `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
// `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
// `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
// `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
// `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
// `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
// `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
// `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
// `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
// `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
// `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
// `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
// `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
// `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
// `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
// `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
// `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
// `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
// `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
// `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
// `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
// `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
// `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
// `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
// `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
// `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
// `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
// `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
// `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
// `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
// `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
// `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
// `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
// PRIMARY KEY (`id`),
// UNIQUE KEY `download_total_date_name` (`date`, `name`)
// UNIQUE KEY `name_date` (`name`, `date`)
// KEY `date` (`date`)
// ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
module.exports = function (sequelize, DataTypes) {
return sequelize.define('DownloadTotal', {
date: {
type: DataTypes.DATE,
allowNull: false,
comment: 'YYYY-MM-DD format',
},
name: {
type: DataTypes.STRING(100),
allowNull: false,
comment: 'module name',
},
count: {
date: {
type: DataTypes.INTEGER.UNSIGNED,
allowNull: false,
comment: 'YYYYMM format',
},
d01: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: 'download count',
}
comment: '01 download count',
},
d02: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '02 download count',
},
d03: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '03 download count',
},
d04: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '04 download count',
},
d05: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '05 download count',
},
d06: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '06 download count',
},
d07: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '07 download count',
},
d08: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '08 download count',
},
d09: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '09 download count',
},
d10: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '10 download count',
},
d11: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '11 download count',
},
d12: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '12 download count',
},
d13: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '13 download count',
},
d14: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '14 download count',
},
d15: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '15 download count',
},
d16: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '16 download count',
},
d17: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '17 download count',
},
d18: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '18 download count',
},
d19: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '19 download count',
},
d20: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '20 download count',
},
d21: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '21 download count',
},
d22: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '22 download count',
},
d23: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '23 download count',
},
d24: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '24 download count',
},
d25: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '25 download count',
},
d26: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '26 download count',
},
d27: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '27 download count',
},
d28: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '28 download count',
},
d29: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '29 download count',
},
d30: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '30 download count',
},
d31: {
type: DataTypes.BIGINT(20).UNSIGNED,
allowNull: false,
defaultValue: 0,
comment: '31 download count',
},
}, {
tableName: 'download_total',
tableName: 'downloads',
comment: 'module download total info',
indexes: [
{
unique: true,
fields: ['date', 'name']
fields: ['name', 'date']
},
{
fields: ['date']
}
],
classMethods: {

132
services/download_total.js
View File

@@ -15,39 +15,39 @@
*/
var moment = require('moment');
var models = require('../models');
var DownloadTotal = models.DownloadTotal;
var DownloadTotal = require('../models').DownloadTotal;
exports.getModuleTotal = function* (name, start, end) {
start += ' 00:00:00';
end += ' 23:59:59';
var startMonth = parseYearMonth(start);
var endMonth = parseYearMonth(end);
var rows = yield DownloadTotal.findAll({
where: {
date: {
gte: start,
lte: end
gte: startMonth,
lte: endMonth
},
name: name
}
});
return formatRows(rows);
return formatRows(rows, start, end);
};
exports.plusModuleTotal = function* (data) {
data.date = new Date(data.date);
var yearMonth = parseYearMonth(data.date);
var row = yield DownloadTotal.find({
where: {
date: data.date,
name: data.name
name: data.name,
date: yearMonth,
}
});
if (!row) {
row = DownloadTotal.build({
date: data.date,
name: data.name
name: data.name,
date: yearMonth,
});
}
row.count += data.count;
var field = 'd' + data.date.substring(8, 10);
row[field] += data.count;
if (row.isDirty) {
return yield row.save();
}
@@ -55,28 +55,90 @@ exports.plusModuleTotal = function* (data) {
};
exports.getTotal = function* (start, end) {
start += ' 00:00:00';
end += ' 23:59:59';
var sql = 'SELECT date, sum(count) AS count FROM download_total \
WHERE date >= ? AND date <= ? GROUP BY date;';
var rows = yield models.query(sql, [start, end]);
return formatRows(rows);
var startMonth = parseYearMonth(start);
var endMonth = parseYearMonth(end);
var rows = yield DownloadTotal.findAll({
where: {
date: {
gte: startMonth,
lte: endMonth
},
}
});
var map = {};
var downloads = [];
rows.forEach(function (row) {
var date = String(row.date);
var month = date.substring(4, 6);
var year = date.substring(0, 4);
var yearMonth = year + '-' + month;
var days = MONTHS[month];
if (month === '02' && moment([Number(year)]).isLeapYear()) {
days += 1;
}
for (var i = 1; i <= days; i++) {
var day = i < 10 ? '0' + i : String(i);
var field = 'd' + day;
var d = yearMonth + '-' + day;
var count = row[field];
if (d >= start && d <= end && count > 0) {
var item = map[d];
if (!item) {
map[d] = item = {
count: 0,
date: d
};
downloads.push(item);
}
item.count += row[field];
}
}
});
return downloads;
};
function formatRows(rows) {
return rows.map(function (row) {
var date = row.date;
if (typeof date === 'string') {
// sqlite raw datetime is string format ...
date = date.substring(0, 10);
} else {
// mysql return DateTime
date = moment(row.date).format('YYYY-MM-DD');
}
return {
name: row.name,
count: row.count,
date: date,
};
});
var MONTHS = {
'01': 31,
'02': 28, // leap year: 29
'03': 31,
'04': 30,
'05': 31,
'06': 30,
'07': 31,
'08': 31,
'09': 30,
'10': 31,
'11': 30,
'12': 31
};
function parseYearMonth(date) {
return Number(date.substring(0, 7).replace('-', ''));
}
function formatRows(rows, startDate, endDate) {
var dates = [];
rows.forEach(function (row) {
var date = String(row.date);
var month = date.substring(4, 6);
var year = date.substring(0, 4);
var yearMonth = year + '-' + month;
var days = MONTHS[month];
if (month === '02' && moment([Number(year)]).isLeapYear()) {
days += 1;
}
for (var i = 1; i <= days; i++) {
var day = i < 10 ? '0' + i : String(i);
var field = 'd' + day;
var d = yearMonth + '-' + day;
if (d >= startDate && d <= endDate) {
dates.push({
name: row.name,
count: row[field],
date: d
});
}
}
});
return dates;
}

16
test/services/download_total.test.js
View File

@@ -66,13 +66,27 @@ describe('services/download_total.test.js', function () {
count: 3
};
yield* DownloadTotal.plusModuleTotal(data);
data = {
date: '2014-12-22',
name: 'plusModuleTotal-module2',
count: 3
};
yield* DownloadTotal.plusModuleTotal(data);
data = {
date: '2014-12-21',
name: 'plusModuleTotal-module2',
count: 3
};
yield* DownloadTotal.plusModuleTotal(data);
rows = yield DownloadTotal.getTotal('2014-10-21', '2014-12-21');
rows.length.should.above(1);
rows.should.length(3);
rows[0].date.should.equal('2014-10-21');
rows[0].count.should.equal(1003);
rows[1].date.should.equal('2014-10-22');
rows[1].count.should.equal(5);
rows[2].date.should.equal('2014-12-21');
rows[2].count.should.equal(3);
});
});
});

61
tools/sync_download_total.js Normal file
View File

@@ -0,0 +1,61 @@
var co = require('co');
var moment = require('moment');
var models = require('../models');
var DownloadTotal = require('../services/download_total');
co(function* () {
var result = yield models.query('select count(*) as count from downloads;');
if (result[0].count > 0) {
console.log('downloads has %d rows, no need to sync', result[0].count);
return;
}
var lastId = 0;
var count = 0;
while (true) {
var rows = yield models.query('select id, name, date, count from download_total where id > ? limit 10000;', [lastId]);
count += rows.length;
console.log('[%s] last id: %s, got %d rows, total %d', Date(), lastId, rows.length, count);
if (rows.length === 0) {
break;
}
console.log('%j', rows[0]);
var tasks = [];
var currentDate = null;
for (var i = 0; i < rows.length; i++) {
var row = rows[i];
lastId = row.id;
var date = row.date;
if (typeof date !== 'string') {
date = moment(date).format('YYYY-MM-DD');
}
if (!currentDate) {
currentDate = date;
}
if (currentDate === date) {
tasks.push(DownloadTotal.plusModuleTotal({
date: date,
name: row.name,
count: row.count,
}));
if (tasks.length >= 100) {
yield tasks;
tasks = [];
}
} else {
// date change, flush tasks
yield tasks;
currentDate = null;
}
}
if (tasks.length > 0) {
yield tasks;
}
}
})(function (err) {
if (err) {
console.error(err);
throw err;
}
console.log('sync done, you can upgrade to 2.x now.');
process.exit(0);
});