Release 2.19.4

make sure package data be update event replicate down.

https://github.com/npm/registry/issues/129

History.md

@@ -1,4 +1,15 @@
 
+2.19.4 / 2017-03-26
+==================
+
+  * feat: need to sync sourceNpmRegistry also (#1153)
+  * docs: change user.json to utf8mb4
+
+2.19.3 / 2017-02-22
+==================
+
+  * fix: should get package from orginal registry when package is unpublished (#1130)
+
 2.19.2 / 2017-01-05
 ==================
 

Makefile

@@ -23,7 +23,6 @@ test: init-database
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -48,7 +47,6 @@ test-cov cov: init-database
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -69,7 +67,6 @@ test-travis: init-database
 		--reporter dot \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -99,6 +96,6 @@ autod:
 		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
 		--dep mysql \
-		--keep should,supertest,should-http,chunkstream,mm,pedding
+		--keep should,supertest,chunkstream,mm,pedding
 
 .PHONY: test

controllers/sync_module_worker.js

@@ -292,6 +292,19 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return setImmediate(this.finish.bind(this));
   }
 
+  // try to sync from official replicate when source npm registry is not cnpmjs.org
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+
+  const versions = yield this.syncByName(concurrencyId, name, registry);
+  if (versions && versions.length === 0 && registry === config.officialNpmReplicate) {
+    // need to sync sourceNpmRegistry also
+    // make sure package data be update event replicate down.
+    // https://github.com/npm/registry/issues/129
+    yield this.syncByName(concurrencyId, name, config.officialNpmRegistry);
+  }
+};
+
+SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry) {
   var that = this;
   that.syncingNames[name] = true;
   var pkg = null;
@@ -309,8 +322,6 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
 
   // get from npm
   const packageUrl = '/' + name.replace('/', '%2f');
-  // try to sync from official replicate when source npm registry is not cnpmjs.org
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
   try {
     var result = yield npmSerivce.request(packageUrl, { registry: registry });
     pkg = result.data;
@@ -326,6 +337,26 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     }
   }
 
+  if (status === 404 && pkg && pkg.reason === 'deleted' && registry === config.officialNpmReplicate) {
+    // unpublished package on replicate.npmjs.com
+    // 404 { error: 'not_found', reason: 'deleted' }
+    // try to read from registry.npmjs.com and get the whole unpublished info
+    try {
+      var result = yield npmSerivce.request(packageUrl, { registry: config.sourceNpmRegistry });
+      pkg = result.data;
+      status = result.status;
+    } catch (err) {
+      // if 404
+      if (!err.res || err.res.statusCode !== 404) {
+        var errMessage = err.name + ': ' + err.message;
+        that.log('[c#%s] [error] [%s] get package(%s%s) error: %s, status: %s',
+          concurrencyId, name, config.sourceNpmRegistry, packageUrl, errMessage, status);
+        yield that._doneOne(concurrencyId, name, false);
+        return;
+      }
+    }
+  }
+
   var unpublishedInfo = null;
   if (status === 404) {
     // check if it's unpublished
@@ -335,6 +366,11 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     } else {
       pkg = null;
     }
+  } else {
+    // unpublished package status become to 200
+    if (name.length < 80 && pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+      unpublishedInfo = pkg.time.unpublished;
+    }
   }
 
   if (!pkg) {
@@ -344,8 +380,10 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return;
   }
 
-  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, start...',
-    concurrencyId, name, registry, packageUrl, status, pkg['dist-tags'], pkg.time && pkg.time.modified);
+  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, unpublished: %j, start...',
+    concurrencyId, name, registry, packageUrl, status,
+    pkg['dist-tags'], pkg.time && pkg.time.modified,
+    unpublishedInfo);
 
   if (unpublishedInfo) {
     try {
@@ -375,6 +413,8 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
   this.log('[c#%d] [%s] synced success, %d versions: %s',
     concurrencyId, name, versions.length, versions.join(', '));
   yield this._doneOne(concurrencyId, name, true);
+
+  return versions;
 };
 
 function* _listStarUsers(modName) {

docs/db.sql

@@ -18,6 +18,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 -- ALTER TABLE `user`
 --   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
 --   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
+-- ALTER TABLE `user` CHANGE `json` `json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details';
 
 CREATE TABLE IF NOT EXISTS `module_keyword` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.19.2",
+  "version": "2.19.4",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -73,7 +73,6 @@
     "pg": "5",
     "pg-hstore": "2",
     "should": "8",
-    "should-http": "*",
     "sqlite3": "*",
     "supertest": "2",
     "thunk-mocha": "1"

test/controllers/sync_module_worker.test.js

@@ -181,7 +181,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
 
   it('should sync unpublished info', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['tnpm'],
+      name: ['afp'],
       username: 'fengmk2'
     });
 
@@ -189,7 +189,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['tnpm']);
+      names.should.eql([ 'afp' ]);
       done();
     });
   });

test/controllers/web/package/show.test.js

@@ -36,7 +36,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -52,7 +52,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -181,7 +181,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -213,7 +213,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -245,7 +245,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -277,7 +277,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org\.git/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -309,7 +309,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -341,10 +341,10 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/http:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
     });
   });
-});
+});

test/sync/sync_exist.test.js

@@ -36,8 +36,8 @@ describe('test/sync/sync_exist.test.js', function () {
     it('should sync first time ok', function *() {
       mm.data(npmService, 'getShort', ['byte']);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: 0});
-      var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      var data = yield sync();
+      data.successes[0].should.equal('byte');
     });
 
     it('should sync common ok', function *() {
@@ -47,7 +47,7 @@ describe('test/sync/sync_exist.test.js', function () {
       });
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      data.successes[0].should.equal('byte');
 
       mm.data(npmService, 'getAllSince', []);
       var data = yield* sync();
@@ -62,7 +62,7 @@ describe('test/sync/sync_exist.test.js', function () {
       ]);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield* sync();
-      data.successes.should.eql(['byte']);
+      data.successes[0].should.equal('byte');
 
       mm.data(npmService, 'getAllSince', []);
       var data = yield* sync();