feat: add changes api (#1734)

controllers/registry/package/changes.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+var lodash = require('lodash');
+var gather = require('co-gather');
+
+// GET /-/_changes?since={timestamp}&limit={number}&cursorId={number}
+// List changes since the timestamp
+// Similar with https://registry.npmmirror.com/_changes?since=1658974943840
+// Change types:
+// 1. ✅ PACKAGE_VERSION_ADDED
+// 2. ✅ PACKAGE_TAG_ADDED
+// 3. 🆕 PACKAGE_UNPUBLISHED
+// 4. 🆕 PACKAGE_VERSION_BLOCKED
+// 5. ❎ PACKAGE_MAINTAINER_REMOVED
+// 6. ❎ PACKAGE_MAINTAINER_CHANGED
+// 7. ❎ PACKAGE_TAG_CHANGED
+//
+// Since we don't have the previous data,
+// We can't compute the reliable seqId
+// use gmt_modified cinstead of seqId
+module.exports = function* listSince() {
+  var query = this.query;
+  var since = query.since;
+  var limit = Number(query.limit);
+
+  // ensure limit
+  if (Number.isNaN(limit)) {
+    limit = 1000;
+  }
+  var queryResults = yield gather(
+    [
+      "listVersionSince",
+      "listTagSince",
+      "listUnpublishedModuleSince",
+      "listBlockVersionSince",
+    ].map(function (method) {
+      return packageService[method](since, limit);
+    })
+  );
+
+  var validResults = queryResults.map(function (result) {
+    if (!result.isError) {
+      return result.value;
+    }
+    return [];
+  });
+
+  var results = lodash.orderBy(
+    lodash.flatten(validResults).filter(Boolean),
+    "gmt_modified",
+    "asc"
+  ).slice(0, limit);
+  this.body = { results };
+};

controllers/utils.js

@@ -159,3 +159,10 @@ exports.getOssLicenseUrlFromName = function (name) {
   return licenseMap[name.toLowerCase()] ?
     base + licenseMap[name.toLowerCase()] : base + name;
 };
+
+exports.ensureSinceIsDate = function(since) {
+  if (!(since instanceof Date)) {
+    return new Date(Number(since));
+  }
+  return since;
+}

package.json

@@ -54,6 +54,7 @@
     "koa-rewrite": "^1.1.2",
     "koa-rt": "^1.0.0",
     "koa-safe-jsonp": "^0.3.1",
+    "lodash": "^4.17.21",
     "markdown-it": "^8.3.2",
     "mime": "^1.3.6",
     "mini-logger": "^1.1.3",

routes/registry.js

@@ -14,6 +14,7 @@ var listAll = require('../controllers/registry/package/list_all');
 var listAllPackageVersions = require('../controllers/registry/package/list_versions');
 var listShorts = require('../controllers/registry/package/list_shorts');
 var listSince = require('../controllers/registry/package/list_since');
+var changes = require('../controllers/registry/package/changes');
 var listAllVersions = require('../controllers/registry/package/list');
 var listDependents = require('../controllers/registry/package/list_dependents');
 var getOneVersion = require('../controllers/registry/package/show');
@@ -50,10 +51,10 @@ function routes(app) {
   }
 
   app.get('/', jsonp, showTotal);
-
   // before /:name/:version
   // get all modules, for npm search
   app.get('/-/all', listAll);
+  app.get('/-/all/changes', changes);
   app.get('/-/all/since', listSince);
   // get all module names, for auto completion
   app.get('/-/short', listShorts);

services/common.js

@@ -14,3 +14,10 @@ exports.isPrivatePackage = function (name) {
   }
   return false;
 };
+
+exports.CHANGE_TYPE = {
+  PACKAGE_TAG_ADDED: 'PACKAGE_TAG_ADDED',
+  PACKAGE_VERSION_ADDED: 'PACKAGE_VERSION_ADDED',
+  PACKAGE_UNPUBLISHED: 'PACKAGE_UNPUBLISHED',
+  PACKAGE_VERSION_BLOCKED: 'PACKAGE_VERSION_BLOCKED'
+};

services/package.js

@@ -5,6 +5,8 @@ var models = require('../models');
 var common = require('./common');
 var libCommon = require('../lib/common');
 var config = require('../config');
+var { ensureSinceIsDate } = require('../controllers/utils');
+var { BlockPackageVersion } = require('../models');
 var Tag = models.Tag;
 var User = models.User;
 var Module = models.Module;
@@ -15,6 +17,8 @@ var ModuleDependency = models.ModuleDependency;
 var ModuleUnpublished = models.ModuleUnpublished;
 var NpmModuleMaintainer = models.NpmModuleMaintainer;
 
+var CHANGE_TYPE = common.CHANGE_TYPE;
+
 // module
 var _parseRow = function (row) {
   if (row.package.indexOf('%7B%22') === 0) {
@@ -198,6 +202,77 @@ exports.listPublicModuleNamesByUser = function* (username) {
   return names;
 };
 
+exports.listModelSince = function(Model, attributes, mapper) {
+  return function*(since, limit) {
+    var start = ensureSinceIsDate(since);
+    var findCondition = {
+      attributes: attributes,
+      where: {
+        gmt_modified: {
+          gte: start
+        },
+      },
+      order: [['gmt_modified', 'ASC'], ['id', 'ASC']],
+    };
+    if (limit) {
+      findCondition.limit = limit;
+    }
+    var rows = yield Model.findAll(findCondition);
+    return rows.map(mapper);
+  }
+}
+
+exports.listTagSince = this.listModelSince(
+  Tag,
+  ['id', 'name', 'tag', 'gmt_modified'],
+  function (row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_TAG_ADDED,
+      id: row.name,
+      changes: [{tag: row.tag}],
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listVersionSince = this.listModelSince(
+  Module,
+  ['id', 'name', 'version', 'gmt_modified'],
+  function (row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_VERSION_ADDED,
+      id: row.name,
+      changes: [{version: row.version}],
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listUnpublishedModuleSince = this.listModelSince(
+  ModuleUnpublished,
+  ['id', 'name', 'gmt_modified'],
+  function(row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_UNPUBLISHED,
+      id: row.name,
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listBlockVersionSince = this.listModelSince(
+  BlockPackageVersion,
+  ['id', 'name', 'version', 'gmt_modified'],
+  function(row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_VERSION_BLOCKED,
+      id: row.name,
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+
 // start must be a date or timestamp
 exports.listPublicModuleNamesSince = function* listPublicModuleNamesSince(start) {
   if (!(start instanceof Date)) {

test/controllers/registry/package/changes.test.js

@@ -0,0 +1,80 @@
+'use strict';
+
+var should = require('should');
+var request = require('supertest');
+var mm = require('mm');
+var app = require('../../../../servers/registry');
+var utils = require('../../../utils');
+var CHANGE_TYPE = require('../../../../services/common').CHANGE_TYPE;
+
+describe('test/controllers/registry/package/changes.test.js', function () {
+  afterEach(mm.restore);
+
+  var since;
+  before(function (done) {
+    setTimeout(() => {
+      since = Date.now();
+      var pkg = utils.getPackage('@cnpmtest/test_changes', '0.0.1', utils.admin, 'alpha');
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, function() {
+        setTimeout(function() {
+          pkg = utils.getPackage('@cnpmtest/test_changes_gogo', '0.0.2', utils.admin, 'beta');
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', utils.adminAuth)
+          .send(pkg)
+          .expect(201, done);
+        }, 2000);
+      });
+    }, 1000);
+  });
+
+  describe('GET /-/all/changes', function () {
+    it('should 200', function (done) {
+      request(app)
+        .get("/-/all/changes?since=" + since)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(2);
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(2);
+          done();
+        });
+    });
+
+    it('since should work', function (done) {
+      var now = Date.now();
+      request(app)
+        .get("/-/all/changes?since=" + now + 5000)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+          res.body.results.length.should.equal(0);
+          done();
+        });
+    });
+
+    it('limit should work', function (done) {
+      request(app)
+      .get('/-/all/changes?limit=1&since=' + since)
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.results.should.be.an.Array();
+        res.body.results.length.should.equal(1);
+        done();
+      });
+    });
+
+  });
+});

test/services/package.test.js

@@ -130,6 +130,69 @@ describe('test/services/package.test.js', function () {
     });
   });
 
+  describe('listModelSince()', function () {
+    it('list tags since', function* () {
+      yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
+      yield sleep(2100);
+      var start = Date.now() - 1000;
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.0');
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta');
+      yield utils.createModule('test-listModuleSince-module-2', '1.0.0');
+      var tags = yield Package.listTagSince(start);
+
+      var modules = tags.map(function (item) {
+        return { name: item.id, tag: item.changes[0].tag };
+      });
+
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", tag: "latest" },
+          { name: "test-listModuleSince-module-1", tag: "beta" },
+          { name: "test-listModuleSince-module-2", tag: "latest" },
+        ]);
+
+      tags = yield Package.listTagSince(start, 2);
+      modules = tags.map(function (item) {
+        return { name: item.id, tag: item.changes[0].tag};
+      });
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", tag: "latest" },
+          { name: "test-listModuleSince-module-1", tag: "beta" },
+        ]);
+    });
+    it('list package version since', function* () {
+      yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
+      yield sleep(2100);
+      var start = Date.now() - 1000;
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.0');
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta');
+      yield utils.createModule('test-listModuleSince-module-2', '1.0.0');
+      var tags = yield Package.listVersionSince(start);
+
+      var modules = tags.map(function (item) {
+        return { name: item.id, version: item.changes[0].version };
+      });
+
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", version: "1.0.0" },
+          { name: "test-listModuleSince-module-1", version: "1.0.1" },
+          { name: "test-listModuleSince-module-2", version: "1.0.0" },
+        ]);
+
+      tags = yield Package.listVersionSince(start, 2);
+      modules = tags.map(function (item) {
+        return { name: item.id, version: item.changes[0].version};
+      });
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", version: "1.0.0" },
+          { name: "test-listModuleSince-module-1", version: "1.0.1" },
+        ]);
+    });
+  });
+
   describe('listPublicModuleNamesSince(), listAllPublicModuleNames()', function () {
     it('should got those module names', function* () {
       yield utils.createModule('test-listPublicModuleNamesSince-module-0', '1.0.0');