diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index cd75d8e..6524f95 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [10, 12, 14, 16] + node-version: [14, 16] os: [ubuntu-latest] steps: diff --git a/config/index.js b/config/index.js index 1b98373..fd7c40c 100644 --- a/config/index.js +++ b/config/index.js @@ -201,6 +201,11 @@ var config = { officialNpmReplicate: 'https://replicate.npmjs.com', cnpmRegistry: 'https://r.cnpmjs.com', + // /-/all/changes + // since different changes are aggregated through many tables + // prevent changesStream changes collisions + changesDelay: 5000, + // sync source, upstream registry // If you want to directly sync from official npm's registry // please drop them an email first diff --git a/services/package.js b/services/package.js index c61b752..bff4aaf 100644 --- a/services/package.js +++ b/services/package.js @@ -203,16 +203,23 @@ exports.listPublicModuleNamesByUser = function* (username) { }; exports.listModelSince = function(Model, attributes, mapper) { + return function*(since, limit) { + var start = ensureSinceIsDate(since); var findCondition = { attributes: attributes, where: { gmt_modified: { - gte: start + gte: start, + // 添加延时,防止同一时间多个数据未同步 + lte: new Date(Date.now() - config.changesDelay || 5000), }, }, - order: [['gmt_modified', 'ASC'], ['id', 'ASC']], + order: [ + ["gmt_modified", "ASC"], + ["id", "ASC"], + ], }; if (limit) { findCondition.limit = limit; diff --git a/test/controllers/registry/package/changes.test.js b/test/controllers/registry/package/changes.test.js index 174ead2..fde27e4 100644 --- a/test/controllers/registry/package/changes.test.js +++ b/test/controllers/registry/package/changes.test.js @@ -3,6 +3,7 @@ var should = require('should'); var request = require('supertest'); var mm = require('mm'); +var config = require('../../../../config'); var app = require('../../../../servers/registry'); var utils = require('../../../utils'); var CHANGE_TYPE = require('../../../../services/common').CHANGE_TYPE; @@ -12,6 +13,7 @@ describe('test/controllers/registry/package/changes.test.js', function () { var since; before(function (done) { + mm(config, 'changesDelay', 0); setTimeout(() => { since = Date.now(); var pkg = utils.getPackage('@cnpmtest/test_changes', '0.0.1', utils.admin, 'alpha'); @@ -39,6 +41,7 @@ describe('test/controllers/registry/package/changes.test.js', function () { .expect(200, function (err, res) { should.not.exist(err); res.body.results.should.be.an.Array(); + res.body.results .filter(function (item) { return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED; @@ -53,6 +56,22 @@ describe('test/controllers/registry/package/changes.test.js', function () { }); }); + it('changes delay should work', function(done) { + mm(config, 'changesDelay', 10000); + request(app) + .get("/-/all/changes?since=" + since) + .expect(200, function (err, res) { + should.not.exist(err); + res.body.results.should.be.an.Array(); + res.body.results + .filter(function (item) { + return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED; + }) + .length.should.equal(0); + done(); + }); + }) + it('since should work', function (done) { var now = Date.now(); request(app) @@ -66,6 +85,7 @@ describe('test/controllers/registry/package/changes.test.js', function () { }); it('limit should work', function (done) { + mm(config, 'changesDelay', 0); request(app) .get('/-/all/changes?limit=1&since=' + since) .expect(200, function (err, res) { diff --git a/test/services/package.test.js b/test/services/package.test.js index e17a3cc..616b676 100644 --- a/test/services/package.test.js +++ b/test/services/package.test.js @@ -5,6 +5,8 @@ var sleep = require('co-sleep'); var Package = require('../../services/package'); var utils = require('../utils'); var common = require('../../services/common'); +var config = require('../../config'); +var mm = require('mm'); describe('test/services/package.test.js', function () { describe('addModuleTag()', function () { @@ -132,6 +134,7 @@ describe('test/services/package.test.js', function () { describe('listModelSince()', function () { it('list tags since', function* () { + mm(config, 'changesDelay', 0); yield utils.createModule('test-listModuleSince-module-0', '1.0.0'); yield sleep(2100); var start = Date.now() - 1000; @@ -162,6 +165,7 @@ describe('test/services/package.test.js', function () { ]); }); it('list package version since', function* () { + mm(config, 'changesDelay', 0); yield utils.createModule('test-listModuleSince-module-0', '1.0.0'); yield sleep(2100); var start = Date.now() - 1000;