fix: use koa-bodyparser instead of koa-middlewares

https://snyk.io/vuln/npm:qs:20170213

History.md

@@ -1,4 +1,9 @@
 
+2.19.3 / 2017-02-22
+==================
+
+  * fix: should get package from orginal registry when package is unpublished (#1130)
+
 2.19.2 / 2017-01-05
 ==================
 

Makefile

@@ -23,7 +23,6 @@ test: init-database
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -48,7 +47,6 @@ test-cov cov: init-database
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -69,7 +67,6 @@ test-travis: init-database
 		--reporter dot \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require should-http \
 		--require thunk-mocha \
 		--require ./test/init.js \
 		$(MOCHA_OPTS) \
@@ -99,6 +96,6 @@ autod:
 		--prefix "~" \
 		--exclude public,view,docs,backup,coverage \
 		--dep mysql \
-		--keep should,supertest,should-http,chunkstream,mm,pedding
+		--keep should,supertest,chunkstream,mm,pedding
 
 .PHONY: test

controllers/sync_module_worker.js

@@ -326,6 +326,26 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     }
   }
 
+  if (status === 404 && pkg && pkg.reason === 'deleted' && registry === config.officialNpmReplicate) {
+    // unpublished package on replicate.npmjs.com
+    // 404 { error: 'not_found', reason: 'deleted' }
+    // try to read from registry.npmjs.com and get the whole unpublished info
+    try {
+      var result = yield npmSerivce.request(packageUrl, { registry: config.sourceNpmRegistry });
+      pkg = result.data;
+      status = result.status;
+    } catch (err) {
+      // if 404
+      if (!err.res || err.res.statusCode !== 404) {
+        var errMessage = err.name + ': ' + err.message;
+        that.log('[c#%s] [error] [%s] get package(%s%s) error: %s, status: %s',
+          concurrencyId, name, config.sourceNpmRegistry, packageUrl, errMessage, status);
+        yield that._doneOne(concurrencyId, name, false);
+        return;
+      }
+    }
+  }
+
   var unpublishedInfo = null;
   if (status === 404) {
     // check if it's unpublished
@@ -335,6 +355,11 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     } else {
       pkg = null;
     }
+  } else {
+    // unpublished package status become to 200
+    if (name.length < 80 && pkg && pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+      unpublishedInfo = pkg.time.unpublished;
+    }
   }
 
   if (!pkg) {
@@ -344,8 +369,10 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return;
   }
 
-  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, start...',
-    concurrencyId, name, registry, packageUrl, status, pkg['dist-tags'], pkg.time && pkg.time.modified);
+  that.log('[c#%d] [%s] package(%s%s) status: %s, dist-tags: %j, time.modified: %s, unpublished: %j, start...',
+    concurrencyId, name, registry, packageUrl, status,
+    pkg['dist-tags'], pkg.time && pkg.time.modified,
+    unpublishedInfo);
 
   if (unpublishedInfo) {
     try {
@@ -367,6 +394,13 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return;
   }
 
+  // if (versions.length === 0 && registry === config.officialNpmReplicate) {
+  //   // TODO:
+  //   // need to sync sourceNpmRegistry also
+  //   // make sure package data be update event replicate down.
+  //   // https://github.com/npm/registry/issues/129
+  // }
+
   // has new version
   if (versions.length > 0) {
     that.updates.push(name);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "2.19.2",
+  "version": "2.19.3",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -36,6 +36,7 @@
     "is-type-of": "^1.0.0",
     "kcors": "^1.2.1",
     "koa": "^1.2.0",
+    "koa-bodyparser": "^2.4.0",
     "koa-limit": "^1.0.2",
     "koa-markdown": "^2.0.1",
     "koa-maxrequests": "^1.0.0",
@@ -73,7 +74,6 @@
     "pg": "5",
     "pg-hstore": "2",
     "should": "8",
-    "should-http": "*",
     "sqlite3": "*",
     "supertest": "2",
     "thunk-mocha": "1"

servers/registry.js

@@ -14,6 +14,7 @@ var notFound = require('../middleware/registry_not_found');
 var cors = require('kcors');
 var proxyToNpm = require('../middleware/proxy_to_npm');
 var maxrequests = require('koa-maxrequests');
+var bodyParser = require('koa-bodyparser');
 
 app.use(maxrequests());
 app.use(block());
@@ -24,7 +25,7 @@ app.use(staticCache);
 
 app.keys = ['todokey', config.sessionSecret];
 app.proxy = true;
-app.use(middlewares.bodyParser({ jsonLimit: config.jsonLimit }));
+app.use(bodyParser({ jsonLimit: config.jsonLimit }));
 app.use(cors({
   allowMethods: 'GET,HEAD',
 }));

servers/web.js

@@ -18,6 +18,7 @@ var http = require('http');
 var koa = require('koa');
 var fs = require('fs');
 var maxrequests = require('koa-maxrequests');
+var bodyParser = require('koa-bodyparser');
 
 var app = koa();
 
@@ -43,7 +44,7 @@ app.proxy = true;
 app.use(proxyToNpm({
   isWeb: true
 }));
-app.use(middlewares.bodyParser());
+app.use(bodyParser({ jsonLimit: config.jsonLimit }));
 app.use(auth());
 app.use(notFound);
 

test/controllers/sync_module_worker.test.js

@@ -181,7 +181,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
 
   it('should sync unpublished info', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['tnpm'],
+      name: ['afp'],
       username: 'fengmk2'
     });
 
@@ -189,7 +189,7 @@ describe('test/controllers/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['tnpm']);
+      names.should.eql([ 'afp' ]);
       done();
     });
   });

test/controllers/web/package/show.test.js

@@ -36,7 +36,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -52,7 +52,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/Dependencies/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -181,7 +181,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -213,7 +213,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -245,7 +245,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -277,7 +277,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org\.git/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -309,7 +309,7 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/https:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
@@ -341,10 +341,10 @@ describe('test/controllers/web/package/show.test.js', () => {
       .expect(/http:\/\/github\.com\/cnpm\/cnpmjs\.org/)
       .expect(/Downloads/, function (err, res) {
         should.not.exist(err);
-        res.should.have.header('etag');
+        should.exist(res.headers.etag);
         res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
     });
   });
-});
+});