open-git/cnpmjs.org
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support dist.integrity (#1677)

Browse Source 
https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#dist
This commit is contained in:
fengmk2
2021-11-09 14:43:39 +08:00
committed by GitHub
parent a49deec3b2
commit abc1723bef
7 changed files with 232 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/nodejs.yml vendored
View File

@@ -28,7 +28,7 @@ jobs:
strategy:
fail-fast: false
matrix:
node-version: [10, 12]
node-version: [10, 12, 14]
os: [ubuntu-latest]
steps:

101
controllers/registry/package/save.js
View File

@@ -1,9 +1,10 @@
'use strict';
var debug = require('debug')('cnpmjs.org:controllers:registry:package:save');
var crypto = require('crypto');
var ssri = require('ssri');
var deprecateVersions = require('./deprecate');
var packageService = require('../../../services/package');
var logger = require('../../../common/logger');
var common = require('../../../lib/common');
var nfs = require('../../../common/nfs');
var config = require('../../../config');
@@ -16,14 +17,42 @@ var hook = require('../../../services/hook');
//
// new flows: only one request
// PUT /:name
// https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
// old publish: https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
// new publish: https://github.com/npm/libnpmpublish/blob/main/publish.js#L91
module.exports = function* save(next) {
// 'dist-tags': { latest: '0.0.2' },
// _attachments:
// { 'nae-sandbox-0.0.2.tgz':
// { content_type: 'application/octet-stream',
// data: 'H4sIAAAAA
// length: 9883
// {
// "_id": "@cnpm/foo",
// "name": "@cnpm/foo",
// "dist-tags": {
// "latest": "1.0.0"
// },
// "versions": {
// "1.0.0": {
// "name": "@cnpm/foo",
// "version": "1.0.0",
// "dependencies": {
// "xprofiler": "^1.2.6"
// },
// "readme": "ERROR: No README data found!",
// "_id": "@cnpm/foo@1.0.0",
// "_nodeVersion": "16.13.0",
// "_npmVersion": "8.1.0",
// "dist": {
// "integrity": "sha512-7nm0vpDEWs7y+tTwlxd7YnGaBc+9Gk5KaPsx2cqQz6H84ndBXlw5nMxGtL4Uy0bCQIknPAZAVe+KNheInmmJrQ==",
// "shasum": "afd05dcfb8759b9b1c7151492a04f2254365c602",
// "tarball": "http://127.0.0.1:7001/@cnpm/foo/-/@cnpm/foo-1.0.0.tgz"
// }
// }
// },
// "access": null,
// "_attachments": {
// "@cnpm/foo-1.0.0.tgz": {
// "content_type": "application/octet-stream",
// "data": "H4sIAAAAAA...",
// "length": 208
// }
// }
// }
var pkg = this.request.body;
var username = this.user.name;
var name = this.params.name || this.params[0];
@@ -150,7 +179,6 @@ module.exports = function* save(next) {
username, name, version, attachment.length, versionPackage.maintainers, distTags);
var exists = yield packageService.getModule(name, version);
var shasum;
if (exists) {
this.status = 403;
const error = '[forbidden] cannot modify pre-existing version: ' + version;
@@ -186,21 +214,60 @@ module.exports = function* save(next) {
}
}
shasum = crypto.createHash('sha1');
shasum.update(tarballBuffer);
shasum = shasum.digest('hex');
var originDist = versionPackage.dist || {};
var shasum;
var integrity = originDist.integrity;
// for content security reason
// check integrity
if (integrity) {
var algorithm = ssri.checkData(tarballBuffer, integrity);
if (!algorithm) {
logger.error('[registry:save:integrity:invalid] %s@%s, dist:%j', name, version, originDist);
this.status = 400;
const error = '[invalid] dist.integrity invalid';
this.body = {
error,
reason: error,
};
return;
}
var integrityObj = ssri.fromData(tarballBuffer, {
algorithms: ['sha1'],
});
shasum = integrityObj.sha1[0].hexDigest();
} else {
var integrityObj = ssri.fromData(tarballBuffer, {
algorithms: ['sha512', 'sha1'],
});
integrity = integrityObj.sha512[0].toString();
shasum = integrityObj.sha1[0].hexDigest();
if (originDist.shasum && originDist.shasum !== shasum) {
// if integrity not exists, check shasum
logger.error('[registry:save:shasum:invalid] %s@%s, dist:%j', name, version, originDist);
this.status = 400;
const error = '[invalid] dist.shasum invalid';
this.body = {
error,
reason: error,
};
return;
}
}
var options = {
key: common.getCDNKey(name, filename),
shasum: shasum
shasum: shasum,
integrity: integrity,
};
var uploadResult = yield nfs.uploadBuffer(tarballBuffer, options);
debug('upload %j', uploadResult);
debug('upload %j, options: %j', uploadResult, options);
var dist = {
var dist = Object.assign({}, originDist, {
tarball: '',
integrity: integrity,
shasum: shasum,
size: attachment.length
};
size: attachment.length,
});
// if nfs upload return a key, record it
if (uploadResult.url) {

6
controllers/sync_module_worker.js
View File

@@ -1717,11 +1717,13 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
mod.package._publish_on_cnpm = true;
}
var dist = {
// keep orgin dist fields, like `integrity` and so on
var dist = Object.assign({}, sourcePackage.dist, {
tarball: '',
shasum: shasum,
size: dataSize,
noattachment: dataSize === 0,
};
});
if (result.url) {
dist.tarball = result.url;

1
package.json
View File

@@ -66,6 +66,7 @@
"rimraf": "^2.6.2",
"semver": "^5.4.1",
"sequelize": "^3.23.4",
"ssri": "^8.0.1",
"thunkify-wrap": "^1.0.4",
"treekill": "^1.0.0",
"tunnel-agent": "^0.6.0",

22
test/controllers/registry/package/list.test.js
View File

@@ -20,7 +20,10 @@ describe('test/controllers/registry/package/list.test.js', () => {
var pkg = utils.getPackage('@cnpmtest/testmodule-list-1', '0.0.1', utils.otherUser);
pkg.versions['0.0.1'].dependencies = {
bytetest: '~0.0.1',
mocha: '~1.0.0'
mocha: '~1.0.0',
};
pkg.versions['0.0.1'].scripts = {
install: 'node -v',
};
request(app)
.put('/' + pkg.name)
@@ -351,6 +354,23 @@ describe('test/controllers/registry/package/list.test.js', () => {
});
});
it('should return abbreviated meta on private package and has hasInstallScript field', () => {
mm(config, 'enableAbbreviatedMetadata', true);
return request(app)
.get('/@cnpmtest/testmodule-list-1')
.set('Accept', 'application/vnd.npm.install-v1+json')
.expect(200)
.expect(res => {
const data = res.body;
// console.log(JSON.stringify(data, null, 2));
assert(data.name === '@cnpmtest/testmodule-list-1');
assert(data.modified);
assert(data['dist-tags'].latest);
assert(data.versions['0.0.1'].hasInstallScript === true);
assert(data.versions['0.0.1'].scripts === undefined);
});
});
it('should return abbreviated meta with cache-control', () => {
mm(config, 'registryCacheControlHeader', 'max-age=0, s-maxage=10, must-revalidate');
mm(config, 'syncModel', 'all');

102
test/controllers/registry/package/save.test.js
View File

@@ -11,7 +11,7 @@ var app = require('../../../../servers/registry');
var config = require('../../../../config');
var utils = require('../../../utils');
describe('test/controllers/registry/package/save.test.js', function () {
describe('test/controllers/registry/package/save.test.js', () => {
afterEach(mm.restore);
describe('no @scoped package', function () {
@@ -46,22 +46,112 @@ describe('test/controllers/registry/package/save.test.js', function () {
});
});
it('should publish new version package and save dependencies', function (done) {
it('should publish new version package and save dependencies', done => {
request(app)
.get('/testmodule-new-1')
.expect(200, function (err, res) {
should.not.exist(err);
assert(!err);
var data = res.body;
data.name.should.equal('testmodule-new-1');
Object.keys(data.versions).should.eql(['0.0.1']);
data.versions['0.0.1'].dependencies.should.eql({
assert(data.name === 'testmodule-new-1');
assert.deepStrictEqual(Object.keys(data.versions), ['0.0.1']);
assert.deepStrictEqual(data.versions['0.0.1'].dependencies, {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
});
// should has integrity with sha512
assert(data.versions['0.0.1'].dist.integrity === 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==');
assert(data.versions['0.0.1'].dist.shasum === 'fa475605f88bab9b1127833633ca3ae0a477224c');
done();
});
});
it('should publish error on shasum invaild', done => {
mm(config, 'privatePackages', ['testmodule-new-1']);
var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
pkg.versions['0.0.88'].dependencies = {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
};
pkg.versions['0.0.88'].dist.shasum = 'fa475605f88bab9b1127833633ca3ae0a47wrong';
request(app)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(400, function (err, res) {
assert(!err);
assert(res.body.error === '[invalid] dist.shasum invalid');
assert(res.body.reason === '[invalid] dist.shasum invalid');
done();
});
});
it('should publish error on integrity invaild', done => {
mm(config, 'privatePackages', ['testmodule-new-1']);
var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
pkg.versions['0.0.88'].dependencies = {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
};
pkg.versions['0.0.88'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfBBB==';
request(app)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(400, function (err, res) {
assert(!err);
assert(res.body.error === '[invalid] dist.integrity invalid');
assert(res.body.reason === '[invalid] dist.integrity invalid');
done();
});
});
it('should publish success with integrity and shasum', done => {
mm(config, 'privatePackages', ['testmodule-new-1']);
var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
pkg.versions['0.0.88'].dependencies = {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
};
pkg.versions['0.0.88'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==';
request(app)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, done);
});
it('should publish success with integrity and without shasum', done => {
mm(config, 'privatePackages', ['testmodule-new-1']);
var pkg = utils.getPackage('testmodule-new-1', '0.0.881', utils.admin);
pkg.versions['0.0.881'].dependencies = {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
};
pkg.versions['0.0.881'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==';
delete pkg.versions['0.0.881'].dist;
request(app)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, done);
});
it('should publish success without integrity and without shasum', done => {
mm(config, 'privatePackages', ['testmodule-new-1']);
var pkg = utils.getPackage('testmodule-new-1', '0.0.882', utils.admin);
pkg.versions['0.0.882'].dependencies = {
'bytetest-1': '~0.0.1',
mocha: '~1.0.0'
};
delete pkg.versions['0.0.882'].dist.integrity;
delete pkg.versions['0.0.882'].dist;
request(app)
.put('/' + pkg.name)
.set('authorization', utils.adminAuth)
.send(pkg)
.expect(201, done);
});
it('should publish new package and fire globalHook', done => {
done = pedding(done, 2);
mm(config, 'globalHook', function* (envelope) {

25
test/controllers/sync_module_worker.test.js
View File

@@ -298,6 +298,31 @@ describe('test/controllers/sync_module_worker.test.js', () => {
};
}
yield checkResult();
function checkFullResult() {
return function (done) {
request(app)
.get('/mk2test-module-cnpmsync-issue-1667')
.set('accept', 'application/json')
.expect(function (res) {
lastResHeaders = res.headers;
console.log('%j', res.body);
pkg = res.body.versions['3.0.0'];
assert(pkg.hasInstallScript === true);
// has scripts
assert(pkg.scripts);
// console.log(pkg.dist);
assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz');
assert(pkg.dist.integrity === 'sha512-pwnnZyjvr29UxwFAIx7xHvVCkFpGVAYgaFllr/m5AZoD1CR2uHHPw16ISEO/A2rZ0WM3UoAghwd5bAZ4pYzD2Q==');
assert(pkg.dist.shasum === 'c31af371a6cdc10dd5b9ad26625a4c863249198d');
assert(pkg.dist.fileCount === 2);
assert(pkg.dist.unpackedSize === 232);
assert(pkg.dist.size === 271);
})
.expect(200, done);
};
}
yield checkFullResult();
});
it('should sync upstream first', function* () {