From 3c5bc9dc5ef46c567e718705d18edc1f57009b5e Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Wed, 24 Nov 2021 00:39:48 +0800 Subject: [PATCH] feat: support package version block list (#1683) --- config/index.js | 5 ++ controllers/registry/package/list.js | 25 +++++++- controllers/web/package/show.js | 11 ++++ docs/db.sql | 11 ++++ models/block_package_version.js | 29 ++++++++++ models/download_total.js | 14 ----- models/index.js | 11 ++++ models/total.js | 14 ----- services/blocklist.js | 28 +++++++++ .../controllers/registry/package/list.test.js | 58 +++++++++++++++++++ test/controllers/web/package/show.test.js | 37 ++++++++++++ test/services/blocklist.test.js | 28 +++++++++ test/services/common.test.js | 14 ----- 13 files changed, 242 insertions(+), 43 deletions(-) create mode 100644 models/block_package_version.js create mode 100644 services/blocklist.js create mode 100644 test/services/blocklist.test.js diff --git a/config/index.js b/config/index.js index 25f85da..cc8a438 100644 --- a/config/index.js +++ b/config/index.js @@ -287,6 +287,9 @@ var config = { // if enable this option, must create module_abbreviated and package_readme table in database enableAbbreviatedMetadata: false, + // enable package or package version block list, must create package_version_blocklist table in database + enableBlockPackageVersion: false, + // global hook function: function* (envelope) {} // envelope format please see https://github.com/npm/registry/blob/master/docs/hooks/hooks-payload.md#payload globalHook: null, @@ -334,6 +337,8 @@ if (process.env.NODE_ENV === 'test') { yield next; }; }); + + config.enableBlockPackageVersion = true; } if (process.env.NODE_ENV !== 'test') { diff --git a/controllers/registry/package/list.js b/controllers/registry/package/list.js index a1b1687..a43e27a 100644 --- a/controllers/registry/package/list.js +++ b/controllers/registry/package/list.js @@ -3,6 +3,7 @@ var debug = require('debug')('cnpmjs.org:controllers:registry:package:list'); var utility = require('utility'); var packageService = require('../../../services/package'); +var blocklistService = require('../../../services/blocklist'); var common = require('../../../lib/common'); var SyncModuleWorker = require('../../sync_module_worker'); var config = require('../../../config'); @@ -15,6 +16,13 @@ function etag(objs) { return 'W/"' + utility.md5(JSON.stringify(objs)) + '"'; } +function filterBlockVerions(rows, blocks) { + if (!blocks) { + return rows; + } + return rows.filter(row => !blocks[row.version]); +} + /** * list all version of a module * GET /:name @@ -63,9 +71,22 @@ module.exports = function* list() { var rs = yield [ packageService.getModuleLastModified(name), packageService.listModuleTags(name), + blocklistService.findBlockPackageVersions(name), ]; var modifiedTime = rs[0]; var tags = rs[1]; + var blocks = rs[2]; + + if (blocks && blocks['*']) { + this.status = 451; + const error = `[block] package was blocked, reason: ${blocks['*'].reason}`; + this.jsonp = { + name, + error, + reason: error, + }; + return; + } debug('show %s, last modified: %s, tags: %j', name, modifiedTime, tags); if (modifiedTime) { @@ -89,11 +110,13 @@ module.exports = function* list() { if (needAbbreviatedMeta) { var rows = yield packageService.listModuleAbbreviatedsByName(name); + rows = filterBlockVerions(rows, blocks); if (rows.length > 0) { yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows, cacheKey); return; } var fullRows = yield packageService.listModulesByName(name); + fullRows = filterBlockVerions(fullRows, blocks); if (fullRows.length > 0) { // no abbreviated meta rows, use the full meta convert to abbreviated meta yield handleAbbreviatedMetaRequestWithFullMeta(this, name, modifiedTime, tags, fullRows); @@ -106,7 +129,7 @@ module.exports = function* list() { packageService.listStarUserNames(name), packageService.listMaintainers(name), ]; - var rows = r[0]; + var rows = filterBlockVerions(r[0], blocks); var starUsers = r[1]; var maintainers = r[2]; diff --git a/controllers/web/package/show.js b/controllers/web/package/show.js index 7f7f13d..d9fa36f 100644 --- a/controllers/web/package/show.js +++ b/controllers/web/package/show.js @@ -12,6 +12,7 @@ var utils = require('../../utils'); var setDownloadURL = require('../../../lib/common').setDownloadURL; var renderMarkdown = require('../../../common/markdown').render; var packageService = require('../../../services/package'); +var blocklistService = require('../../../services/blocklist'); var downloadTotalService = require('../../../services/download_total'); module.exports = function* show(next) { @@ -71,6 +72,16 @@ module.exports = function* show(next) { return yield next; } + var blocks = yield blocklistService.findBlockPackageVersions(name); + if (blocks) { + var block = blocks['*'] || blocks[pkg.version]; + if (block) { + this.status = 451; + this.body = `[block] package@${pkg.version} was blocked, reason: ${block.reason}`; + return; + } + } + var r = yield [ utils.getDownloadTotal(name), packageService.listDependents(name), diff --git a/docs/db.sql b/docs/db.sql index f4d8536..ebf691b 100644 --- a/docs/db.sql +++ b/docs/db.sql @@ -333,3 +333,14 @@ CREATE TABLE IF NOT EXISTS `token` ( UNIQUE KEY `uk_token` (`token`), KEY `idx_user` (`user_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='token info'; + +CREATE TABLE IF NOT EXISTS `package_version_blocklist` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `gmt_create` datetime(3) NOT NULL COMMENT 'create time', + `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time', + `name` varchar(214) NOT NULL COMMENT 'package name', + `version` varchar(30) NOT NULL COMMENT 'package version, "*" meaning all versions', + `reason` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL COMMENT 'block reason', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_name_version` (`name`, `version`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='package version block list'; diff --git a/models/block_package_version.js b/models/block_package_version.js new file mode 100644 index 0000000..095fbe5 --- /dev/null +++ b/models/block_package_version.js @@ -0,0 +1,29 @@ +'use strict'; + +module.exports = function (sequelize, DataTypes) { + return sequelize.define('BlockPackageVersion', { + name: { + type: DataTypes.STRING(214), + allowNull: false, + comment: 'package name' + }, + version: { + type: DataTypes.STRING(30), + allowNull: false, + comment: 'package version' + }, + reason: { + type: DataTypes.LONGTEXT, + comment: 'block reason', + }, + }, { + tableName: 'package_version_blocklist', + comment: 'package version block list', + indexes: [ + { + unique: true, + fields: ['name', 'version'], + }, + ], + }); +}; diff --git a/models/download_total.js b/models/download_total.js index c6cfd81..3dec67d 100644 --- a/models/download_total.js +++ b/models/download_total.js @@ -1,19 +1,5 @@ -/**! - * cnpmjs.org - models/download_total.js - * - * Copyright(c) fengmk2 and other contributors. - * MIT Licensed - * - * Authors: - * fengmk2 (http://fengmk2.github.com) - */ - 'use strict'; -/** - * Module dependencies. - */ - // CREATE TABLE IF NOT EXISTS `downloads` ( // `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', // `gmt_create` datetime NOT NULL COMMENT 'create time', diff --git a/models/index.js b/models/index.js index 81fb4e6..c236010 100644 --- a/models/index.js +++ b/models/index.js @@ -10,6 +10,7 @@ function load(name) { var _ModuleAbbreviated = config.enableAbbreviatedMetadata ? load('module_abbreviated') : null; var _PackageReadme = config.enableAbbreviatedMetadata ? load('package_readme') : null; +var _BlockPackageVersion = config.enableBlockPackageVersion ? load('block_package_version') : null; module.exports = { sequelize: sequelize, @@ -60,4 +61,14 @@ module.exports = { } return _PackageReadme; }, + + get BlockPackageVersion() { + if (!config.enableBlockPackageVersion) { + return null; + } + if (!_BlockPackageVersion) { + _BlockPackageVersion = load('block_package_version'); + } + return _BlockPackageVersion; + } }; diff --git a/models/total.js b/models/total.js index f6ef598..bead410 100644 --- a/models/total.js +++ b/models/total.js @@ -1,19 +1,5 @@ -/**! - * cnpmjs.org - models/total.js - * - * Copyright(c) fengmk2 and other contributors. - * MIT Licensed - * - * Authors: - * fengmk2 (http://fengmk2.github.com) - */ - 'use strict'; -/** - * Module dependencies. - */ - // CREATE TABLE IF NOT EXISTS `total` ( // `name` varchar(214) NOT NULL COMMENT 'total name', // `gmt_modified` datetime NOT NULL COMMENT 'modified time', diff --git a/services/blocklist.js b/services/blocklist.js new file mode 100644 index 0000000..bb1342f --- /dev/null +++ b/services/blocklist.js @@ -0,0 +1,28 @@ +'use strict'; + +const BlockPackageVersion = require('../models').BlockPackageVersion; + +exports.blockPackageVersion = function* (name, version, reason) { + const row = yield BlockPackageVersion.findOne({ where: { name, version } }); + if (row) { + row.reason = reason; + yield row.save(); + } else { + yield BlockPackageVersion.create({ name, version, reason }); + } +}; + +exports.findBlockPackageVersions = function* (name) { + if (!BlockPackageVersion) { + return null; + } + const rows = yield BlockPackageVersion.findAll({ where: { name } }); + if (rows.length === 0) { + return null; + } + const blocks = {}; + for (const row of rows) { + blocks[row.version] = row; + } + return blocks; +}; diff --git a/test/controllers/registry/package/list.test.js b/test/controllers/registry/package/list.test.js index f6cce50..dfc5b5e 100644 --- a/test/controllers/registry/package/list.test.js +++ b/test/controllers/registry/package/list.test.js @@ -6,6 +6,7 @@ var request = require('supertest'); var mm = require('mm'); var pedding = require('pedding'); var packageService = require('../../../../services/package'); +var blocklistService = require('../../../../services/blocklist'); var app = require('../../../../servers/registry'); var utils = require('../../../utils'); var config = require('../../../../config'); @@ -44,6 +45,63 @@ describe('test/controllers/registry/package/list.test.js', () => { }); }); + describe('block versions', () => { + before(function* () { + var pkg = utils.getPackage('@cnpmtest/testmodule-list-block', '0.0.1', utils.otherUser); + pkg.versions['0.0.1'].dependencies = { + bytetest: '~0.0.1', + mocha: '~1.0.0', + }; + pkg.versions['0.0.1'].scripts = { + install: 'node -v', + }; + yield request(app) + .put('/' + pkg.name) + .set('authorization', utils.otherUserAuth) + .send(pkg) + .expect(201); + + pkg = utils.getPackage('@cnpmtest/testmodule-list-block', '1.0.0', utils.otherUser); + pkg.versions['1.0.0'].dependencies = { + bytetest: '~0.0.1', + mocha: '~1.0.0' + }; + yield request(app) + .put('/' + pkg.name) + .set('authorization', utils.otherUserAuth) + .send(pkg) + .expect(201); + }); + + it('should block one version and all versions', function* () { + yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-list-block', '0.0.1', 'unittest'); + let res = yield request(app) + .get('/@cnpmtest/testmodule-list-block') + .expect(200); + let data = res.body; + assert(Object.keys(data.versions).length === 1); + assert(data.versions['1.0.0']); + assert(!data.versions['0.0.1']); + + res = yield request(app) + .get('/@cnpmtest/testmodule-list-block') + .set('Accept', 'application/vnd.npm.install-v1+json') + .expect(200); + data = res.body; + assert(Object.keys(data.versions).length === 1); + assert(data.versions['1.0.0']); + assert(!data.versions['0.0.1']); + + yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-list-block', '*', 'unittest'); + res = yield request(app) + .get('/@cnpmtest/testmodule-list-block') + .expect(451); + data = res.body; + console.log(data); + assert(data.error === '[block] package was blocked, reason: unittest'); + }); + }); + it('should use costomized registry middleware', done => { request(app) .get('/@cnpmtest/testmodule-list-1') diff --git a/test/controllers/web/package/show.test.js b/test/controllers/web/package/show.test.js index d7a4afb..aa5854d 100644 --- a/test/controllers/web/package/show.test.js +++ b/test/controllers/web/package/show.test.js @@ -1,11 +1,13 @@ 'use strict'; var should = require('should'); +var assert = require('assert'); var request = require('supertest'); var mm = require('mm'); var config = require('../../../../config'); var app = require('../../../../servers/web'); var registry = require('../../../../servers/registry'); +var blocklistService = require('../../../../services/blocklist'); var utils = require('../../../utils'); describe('test/controllers/web/package/show.test.js', () => { @@ -55,6 +57,41 @@ describe('test/controllers/web/package/show.test.js', () => { }); }); + it('should get block package', function* () { + var pkg = utils.getPackage('@cnpmtest/testmodule-web-show-block', '0.0.1', utils.admin); + pkg.versions['0.0.1'].dependencies = { + bytetest: '~0.0.1', + mocha: '~1.0.0', + 'testmodule-web-show': '0.0.1' + }; + yield request(registry) + .put('/' + pkg.name) + .set('authorization', utils.adminAuth) + .send(pkg) + .expect(201); + + yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '0.0.1', 'unittest'); + let res = yield request(app) + .get('/package/@cnpmtest/testmodule-web-show-block') + .expect(451) + .expect('content-type', 'text/plain; charset=utf-8'); + assert(res.text === '[block] package@0.0.1 was blocked, reason: unittest'); + + yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '0.0.1', 'unittest'); + res = yield request(app) + .get('/package/@cnpmtest/testmodule-web-show-block/0.0.1') + .expect(451) + .expect('content-type', 'text/plain; charset=utf-8'); + assert(res.text === '[block] package@0.0.1 was blocked, reason: unittest'); + + yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '*', 'block all'); + res = yield request(app) + .get('/package/@cnpmtest/testmodule-web-show-block') + .expect(451) + .expect('content-type', 'text/plain; charset=utf-8'); + assert(res.text === '[block] package@0.0.1 was blocked, reason: block all'); + }); + it('should get scoped package', function (done) { request(app) .get('/package/@cnpmtest/testmodule-web-show') diff --git a/test/services/blocklist.test.js b/test/services/blocklist.test.js new file mode 100644 index 0000000..22ffc9d --- /dev/null +++ b/test/services/blocklist.test.js @@ -0,0 +1,28 @@ +'use strict'; + +const assert = require('assert'); +const blocklist = require('../../services/blocklist'); + +describe('test/services/blocklist.test.js', () => { + describe('blockPackageVersion()', () => { + it('should block one package version ', function* () { + yield blocklist.blockPackageVersion('test-block-name-other', '1.0.0', 'only for test'); + + yield blocklist.blockPackageVersion('test-block-name', '1.0.0', 'only for test'); + const blocks1 = yield blocklist.findBlockPackageVersions('test-block-name'); + assert(Object.keys(blocks1).length === 1); + assert(blocks1['1.0.0'].reason === 'only for test'); + // block again + yield blocklist.blockPackageVersion('test-block-name', '1.0.0', 'only for test new'); + const blocks2 = yield blocklist.findBlockPackageVersions('test-block-name'); + assert(Object.keys(blocks2).length === 1); + assert(blocks2['1.0.0'].reason === 'only for test new'); + + // block all versions + yield blocklist.blockPackageVersion('test-block-name', '*', 'only for test all'); + const blocks3 = yield blocklist.findBlockPackageVersions('test-block-name'); + assert(Object.keys(blocks3).length === 2); + assert(blocks3['*'].reason === 'only for test all'); + }); + }); +}); diff --git a/test/services/common.test.js b/test/services/common.test.js index f574fa5..54888d6 100644 --- a/test/services/common.test.js +++ b/test/services/common.test.js @@ -1,19 +1,5 @@ -/**! - * cnpmjs.org - test/services/common.test.js - * - * Copyright(c) fengmk2 and other contributors. - * MIT Licensed - * - * Authors: - * fengmk2 (http://fengmk2.github.com) - */ - 'use strict'; -/** - * Module dependencies. - */ - var mm = require('mm'); var config = require('../../config'); var common = require('../../services/common');