Release 3.0.0-rc.66

> 包名超长时，更新下载记录接口会无限重试
* 更新下载记录前，先判断当前包名是否存在

-------------

> When the package name is too long, the update download record
interface will retry indefinitely.

* Check pkg exists, before updating the download record.

.github/workflows/nodejs.yml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [10, 12, 14, 16]
+        node-version: [14, 16]
         os: [ubuntu-latest]
 
     steps:

History.md

@@ -1,4 +1,14 @@
 
+3.0.0-rc.66 / 2023-03-25
+==================
+
+**features**
+  * [[`a289d7d`](http://github.com/cnpm/cnpmjs.org/commit/a289d7dd5ec4881e86c6f500c72cab134388782b)] - feat: ignore sync private pkg (#1747) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`af9f5f7`](http://github.com/cnpm/cnpmjs.org/commit/af9f5f749979d1f389d61e98eab7dfbf639c995b)] - feat: add changes delay (#1739) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+**fixes**
+  * [[`6769a46`](http://github.com/cnpm/cnpmjs.org/commit/6769a46d71508a24f4e2d4dc14aa4aebd826bf38)] - fix: download row create error (#1749) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
 3.0.0-rc.63 / 2022-08-31
 ==================
 

config/index.js

@@ -201,6 +201,11 @@ var config = {
   officialNpmReplicate: 'https://replicate.npmjs.com',
   cnpmRegistry: 'https://r.cnpmjs.com',
 
+  // /-/all/changes
+  // since different changes are aggregated through many tables
+  // prevent changesStream changes collisions
+  changesDelay: 5000,
+
   // sync source, upstream registry
   // If you want to directly sync from official npm's registry
   // please drop them an email first

controllers/registry/package/download.js

@@ -130,9 +130,12 @@ defer.setInterval(function* () {
         err.message += '; name: ' + name + ', count: ' + count + ', date: ' + date;
         logger.error(err);
       }
-      // save back to globalDownloads, try again next time
-      count = (globalDownloads.get(name) || 0) + count;
-      globalDownloads.set(name, count);
+      var pkgExist = yield packageService.getModuleLastModified(name);
+      if (pkgExist) {
+        // save back to globalDownloads, try again next time
+        count = (globalDownloads.get(name) || 0) + count;
+        globalDownloads.set(name, count);
+      }
     }
   }
   saving = false;

lib/common.js

@@ -97,6 +97,9 @@ exports.isMaintainer = function (user, maintainers) {
 exports.isLocalModule = function (mods) {
   for (var i = 0; i < mods.length; i++) {
     var r = mods[i];
+    if (config.privatePackages.includes(r.name)) {
+      return true;
+    }
     if (r.package && r.package._publish_on_cnpm) {
       return true;
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "3.0.0-rc.63",
+  "version": "3.0.0-rc.66",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {

services/package.js

@@ -203,16 +203,23 @@ exports.listPublicModuleNamesByUser = function* (username) {
 };
 
 exports.listModelSince = function(Model, attributes, mapper) {
+
   return function*(since, limit) {
+
     var start = ensureSinceIsDate(since);
     var findCondition = {
       attributes: attributes,
       where: {
         gmt_modified: {
-          gte: start
+          gte: start,
+          // 添加延时，防止同一时间多个数据未同步
+          lte: new Date(Date.now() - config.changesDelay || 5000),
         },
       },
-      order: [['gmt_modified', 'ASC'], ['id', 'ASC']],
+      order: [
+        ["gmt_modified", "ASC"],
+        ["id", "ASC"],
+      ],
     };
     if (limit) {
       findCondition.limit = limit;

test/controllers/registry/package/changes.test.js

@@ -3,6 +3,7 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var config = require('../../../../config');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 var CHANGE_TYPE = require('../../../../services/common').CHANGE_TYPE;
@@ -12,6 +13,7 @@ describe('test/controllers/registry/package/changes.test.js', function () {
 
   var since;
   before(function (done) {
+    mm(config, 'changesDelay', 0);
     setTimeout(() => {
       since = Date.now();
       var pkg = utils.getPackage('@cnpmtest/test_changes', '0.0.1', utils.admin, 'alpha');
@@ -39,6 +41,7 @@ describe('test/controllers/registry/package/changes.test.js', function () {
         .expect(200, function (err, res) {
           should.not.exist(err);
           res.body.results.should.be.an.Array();
+
           res.body.results
             .filter(function (item) {
               return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
@@ -53,6 +56,22 @@ describe('test/controllers/registry/package/changes.test.js', function () {
         });
     });
 
+    it('changes delay should work', function(done) {
+      mm(config, 'changesDelay', 10000);
+      request(app)
+        .get("/-/all/changes?since=" + since)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(0);
+          done();
+        });
+    })
+
     it('since should work', function (done) {
       var now = Date.now();
       request(app)
@@ -66,6 +85,7 @@ describe('test/controllers/registry/package/changes.test.js', function () {
     });
 
     it('limit should work', function (done) {
+      mm(config, 'changesDelay', 0);
       request(app)
       .get('/-/all/changes?limit=1&since=' + since)
       .expect(200, function (err, res) {

test/lib/common.test.js

@@ -11,28 +11,50 @@
 
 'use strict';
 
+const { co } = require('co');
+const mm = require('mm');
+const config = require('../../config');
+const createModule = require('../utils').createModule;
+const packageService = require('../../services/package');
 /**
  * Module dependencies.
  */
 
 var common = require('../../lib/common');
 
-describe('test/lib/common.test.js', function () {
-  describe('isAdmin()', function () {
-    it('should admin is admin', function () {
-      common.isAdmin('admin').should.equal(true);
-      common.isAdmin('fengmk2').should.equal(true);
-      common.isAdmin('constructor').should.equal(false);
-      common.isAdmin('toString').should.equal(false);
+describe("test/lib/common.test.js", function () {
+  describe("isAdmin()", function () {
+    it("should admin is admin", function () {
+      common.isAdmin("admin").should.equal(true);
+      common.isAdmin("fengmk2").should.equal(true);
+      common.isAdmin("constructor").should.equal(false);
+      common.isAdmin("toString").should.equal(false);
     });
   });
 
-  describe('getCDNKey()', function () {
-    it('should auto fix scope filename', function () {
-      common.getCDNKey('foo', 'foo-1.0.0.tgz').should.equal('/foo/-/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', 'foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', '@bar/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', '@bar1/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar1/foo-1.0.0.tgz');
+  describe("getCDNKey()", function () {
+    it("should auto fix scope filename", function () {
+      common
+        .getCDNKey("foo", "foo-1.0.0.tgz")
+        .should.equal("/foo/-/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "@bar/foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "@bar1/foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar1/foo-1.0.0.tgz");
+    });
+  });
+
+  describe("isLocalModule", function () {
+    it("should ignore private packages", function * () {
+      yield createModule("banana", "1.0.0");
+      const modules = yield packageService.listModulesByName('banana');
+      mm(config, "privatePackages", ["banana"]);
+      common.isLocalModule(modules).should.equal(true);
     });
   });
 });

test/services/package.test.js

@@ -5,6 +5,8 @@ var sleep = require('co-sleep');
 var Package = require('../../services/package');
 var utils = require('../utils');
 var common = require('../../services/common');
+var config = require('../../config');
+var mm = require('mm');
 
 describe('test/services/package.test.js', function () {
   describe('addModuleTag()', function () {
@@ -132,6 +134,7 @@ describe('test/services/package.test.js', function () {
 
   describe('listModelSince()', function () {
     it('list tags since', function* () {
+      mm(config, 'changesDelay', 0);
       yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
       yield sleep(2100);
       var start = Date.now() - 1000;
@@ -162,6 +165,7 @@ describe('test/services/package.test.js', function () {
         ]);
     });
     it('list package version since', function* () {
+      mm(config, 'changesDelay', 0);
       yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
       yield sleep(2100);
       var start = Date.now() - 1000;