diff --git a/controllers/registry/package/changes.js b/controllers/registry/package/changes.js new file mode 100644 index 0000000..43f2c5d --- /dev/null +++ b/controllers/registry/package/changes.js @@ -0,0 +1,55 @@ +'use strict'; + +var packageService = require('../../../services/package'); +var lodash = require('lodash'); +var gather = require('co-gather'); + +// GET /-/_changes?since={timestamp}&limit={number}&cursorId={number} +// List changes since the timestamp +// Similar with https://registry.npmmirror.com/_changes?since=1658974943840 +// Change types: +// 1. ✅ PACKAGE_VERSION_ADDED +// 2. ✅ PACKAGE_TAG_ADDED +// 3. 🆕 PACKAGE_UNPUBLISHED +// 4. 🆕 PACKAGE_VERSION_BLOCKED +// 5. ❎ PACKAGE_MAINTAINER_REMOVED +// 6. ❎ PACKAGE_MAINTAINER_CHANGED +// 7. ❎ PACKAGE_TAG_CHANGED +// +// Since we don't have the previous data, +// We can't compute the reliable seqId +// use gmt_modified cinstead of seqId +module.exports = function* listSince() { + var query = this.query; + var since = query.since; + var limit = Number(query.limit); + + // ensure limit + if (Number.isNaN(limit)) { + limit = 1000; + } + var queryResults = yield gather( + [ + "listVersionSince", + "listTagSince", + "listUnpublishedModuleSince", + "listBlockVersionSince", + ].map(function (method) { + return packageService[method](since, limit); + }) + ); + + var validResults = queryResults.map(function (result) { + if (!result.isError) { + return result.value; + } + return []; + }); + + var results = lodash.orderBy( + lodash.flatten(validResults).filter(Boolean), + "gmt_modified", + "asc" + ).slice(0, limit); + this.body = { results }; +}; diff --git a/controllers/utils.js b/controllers/utils.js index fa15575..eb101ae 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -159,3 +159,10 @@ exports.getOssLicenseUrlFromName = function (name) { return licenseMap[name.toLowerCase()] ? base + licenseMap[name.toLowerCase()] : base + name; }; + +exports.ensureSinceIsDate = function(since) { + if (!(since instanceof Date)) { + return new Date(Number(since)); + } + return since; +} diff --git a/package.json b/package.json index 2b17a09..2e5d0de 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "koa-rewrite": "^1.1.2", "koa-rt": "^1.0.0", "koa-safe-jsonp": "^0.3.1", + "lodash": "^4.17.21", "markdown-it": "^8.3.2", "mime": "^1.3.6", "mini-logger": "^1.1.3", diff --git a/routes/registry.js b/routes/registry.js index f6fcd61..9a9fc81 100644 --- a/routes/registry.js +++ b/routes/registry.js @@ -14,6 +14,7 @@ var listAll = require('../controllers/registry/package/list_all'); var listAllPackageVersions = require('../controllers/registry/package/list_versions'); var listShorts = require('../controllers/registry/package/list_shorts'); var listSince = require('../controllers/registry/package/list_since'); +var changes = require('../controllers/registry/package/changes'); var listAllVersions = require('../controllers/registry/package/list'); var listDependents = require('../controllers/registry/package/list_dependents'); var getOneVersion = require('../controllers/registry/package/show'); @@ -50,10 +51,10 @@ function routes(app) { } app.get('/', jsonp, showTotal); - // before /:name/:version // get all modules, for npm search app.get('/-/all', listAll); + app.get('/-/all/changes', changes); app.get('/-/all/since', listSince); // get all module names, for auto completion app.get('/-/short', listShorts); diff --git a/services/common.js b/services/common.js index 061c958..e056ac4 100644 --- a/services/common.js +++ b/services/common.js @@ -14,3 +14,10 @@ exports.isPrivatePackage = function (name) { } return false; }; + +exports.CHANGE_TYPE = { + PACKAGE_TAG_ADDED: 'PACKAGE_TAG_ADDED', + PACKAGE_VERSION_ADDED: 'PACKAGE_VERSION_ADDED', + PACKAGE_UNPUBLISHED: 'PACKAGE_UNPUBLISHED', + PACKAGE_VERSION_BLOCKED: 'PACKAGE_VERSION_BLOCKED' +}; diff --git a/services/package.js b/services/package.js index 6a6386b..c61b752 100644 --- a/services/package.js +++ b/services/package.js @@ -5,6 +5,8 @@ var models = require('../models'); var common = require('./common'); var libCommon = require('../lib/common'); var config = require('../config'); +var { ensureSinceIsDate } = require('../controllers/utils'); +var { BlockPackageVersion } = require('../models'); var Tag = models.Tag; var User = models.User; var Module = models.Module; @@ -15,6 +17,8 @@ var ModuleDependency = models.ModuleDependency; var ModuleUnpublished = models.ModuleUnpublished; var NpmModuleMaintainer = models.NpmModuleMaintainer; +var CHANGE_TYPE = common.CHANGE_TYPE; + // module var _parseRow = function (row) { if (row.package.indexOf('%7B%22') === 0) { @@ -198,6 +202,77 @@ exports.listPublicModuleNamesByUser = function* (username) { return names; }; +exports.listModelSince = function(Model, attributes, mapper) { + return function*(since, limit) { + var start = ensureSinceIsDate(since); + var findCondition = { + attributes: attributes, + where: { + gmt_modified: { + gte: start + }, + }, + order: [['gmt_modified', 'ASC'], ['id', 'ASC']], + }; + if (limit) { + findCondition.limit = limit; + } + var rows = yield Model.findAll(findCondition); + return rows.map(mapper); + } +} + +exports.listTagSince = this.listModelSince( + Tag, + ['id', 'name', 'tag', 'gmt_modified'], + function (row) { + return { + type: CHANGE_TYPE.PACKAGE_TAG_ADDED, + id: row.name, + changes: [{tag: row.tag}], + gmt_modified: row.gmt_modified, + }; + } +); + +exports.listVersionSince = this.listModelSince( + Module, + ['id', 'name', 'version', 'gmt_modified'], + function (row) { + return { + type: CHANGE_TYPE.PACKAGE_VERSION_ADDED, + id: row.name, + changes: [{version: row.version}], + gmt_modified: row.gmt_modified, + }; + } +); + +exports.listUnpublishedModuleSince = this.listModelSince( + ModuleUnpublished, + ['id', 'name', 'gmt_modified'], + function(row) { + return { + type: CHANGE_TYPE.PACKAGE_UNPUBLISHED, + id: row.name, + gmt_modified: row.gmt_modified, + }; + } +); + +exports.listBlockVersionSince = this.listModelSince( + BlockPackageVersion, + ['id', 'name', 'version', 'gmt_modified'], + function(row) { + return { + type: CHANGE_TYPE.PACKAGE_VERSION_BLOCKED, + id: row.name, + gmt_modified: row.gmt_modified, + }; + } +); + + // start must be a date or timestamp exports.listPublicModuleNamesSince = function* listPublicModuleNamesSince(start) { if (!(start instanceof Date)) { diff --git a/test/controllers/registry/package/changes.test.js b/test/controllers/registry/package/changes.test.js new file mode 100644 index 0000000..174ead2 --- /dev/null +++ b/test/controllers/registry/package/changes.test.js @@ -0,0 +1,80 @@ +'use strict'; + +var should = require('should'); +var request = require('supertest'); +var mm = require('mm'); +var app = require('../../../../servers/registry'); +var utils = require('../../../utils'); +var CHANGE_TYPE = require('../../../../services/common').CHANGE_TYPE; + +describe('test/controllers/registry/package/changes.test.js', function () { + afterEach(mm.restore); + + var since; + before(function (done) { + setTimeout(() => { + since = Date.now(); + var pkg = utils.getPackage('@cnpmtest/test_changes', '0.0.1', utils.admin, 'alpha'); + request(app) + .put('/' + pkg.name) + .set('authorization', utils.adminAuth) + .send(pkg) + .expect(201, function() { + setTimeout(function() { + pkg = utils.getPackage('@cnpmtest/test_changes_gogo', '0.0.2', utils.admin, 'beta'); + request(app) + .put('/' + pkg.name) + .set('authorization', utils.adminAuth) + .send(pkg) + .expect(201, done); + }, 2000); + }); + }, 1000); + }); + + describe('GET /-/all/changes', function () { + it('should 200', function (done) { + request(app) + .get("/-/all/changes?since=" + since) + .expect(200, function (err, res) { + should.not.exist(err); + res.body.results.should.be.an.Array(); + res.body.results + .filter(function (item) { + return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED; + }) + .length.should.equal(2); + res.body.results + .filter(function (item) { + return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED; + }) + .length.should.equal(2); + done(); + }); + }); + + it('since should work', function (done) { + var now = Date.now(); + request(app) + .get("/-/all/changes?since=" + now + 5000) + .expect(200, function (err, res) { + should.not.exist(err); + res.body.results.should.be.an.Array(); + res.body.results.length.should.equal(0); + done(); + }); + }); + + it('limit should work', function (done) { + request(app) + .get('/-/all/changes?limit=1&since=' + since) + .expect(200, function (err, res) { + should.not.exist(err); + res.body.results.should.be.an.Array(); + res.body.results.length.should.equal(1); + done(); + }); + }); + + }); +}); diff --git a/test/services/package.test.js b/test/services/package.test.js index b038ad1..e17a3cc 100644 --- a/test/services/package.test.js +++ b/test/services/package.test.js @@ -130,6 +130,69 @@ describe('test/services/package.test.js', function () { }); }); + describe('listModelSince()', function () { + it('list tags since', function* () { + yield utils.createModule('test-listModuleSince-module-0', '1.0.0'); + yield sleep(2100); + var start = Date.now() - 1000; + yield utils.createModule('test-listModuleSince-module-1', '1.0.0'); + yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta'); + yield utils.createModule('test-listModuleSince-module-2', '1.0.0'); + var tags = yield Package.listTagSince(start); + + var modules = tags.map(function (item) { + return { name: item.id, tag: item.changes[0].tag }; + }); + + modules + .should.eql([ + { name: "test-listModuleSince-module-1", tag: "latest" }, + { name: "test-listModuleSince-module-1", tag: "beta" }, + { name: "test-listModuleSince-module-2", tag: "latest" }, + ]); + + tags = yield Package.listTagSince(start, 2); + modules = tags.map(function (item) { + return { name: item.id, tag: item.changes[0].tag}; + }); + modules + .should.eql([ + { name: "test-listModuleSince-module-1", tag: "latest" }, + { name: "test-listModuleSince-module-1", tag: "beta" }, + ]); + }); + it('list package version since', function* () { + yield utils.createModule('test-listModuleSince-module-0', '1.0.0'); + yield sleep(2100); + var start = Date.now() - 1000; + yield utils.createModule('test-listModuleSince-module-1', '1.0.0'); + yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta'); + yield utils.createModule('test-listModuleSince-module-2', '1.0.0'); + var tags = yield Package.listVersionSince(start); + + var modules = tags.map(function (item) { + return { name: item.id, version: item.changes[0].version }; + }); + + modules + .should.eql([ + { name: "test-listModuleSince-module-1", version: "1.0.0" }, + { name: "test-listModuleSince-module-1", version: "1.0.1" }, + { name: "test-listModuleSince-module-2", version: "1.0.0" }, + ]); + + tags = yield Package.listVersionSince(start, 2); + modules = tags.map(function (item) { + return { name: item.id, version: item.changes[0].version}; + }); + modules + .should.eql([ + { name: "test-listModuleSince-module-1", version: "1.0.0" }, + { name: "test-listModuleSince-module-1", version: "1.0.1" }, + ]); + }); + }); + describe('listPublicModuleNamesSince(), listAllPublicModuleNames()', function () { it('should got those module names', function* () { yield utils.createModule('test-listPublicModuleNamesSince-module-0', '1.0.0');