From 39cf77ae0f676584d93ada77c6ea61c3d044b267 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Sun, 7 Nov 2021 23:38:32 +0800 Subject: [PATCH] refactor: use remote abbreviated version data (#1675) avoid npm abbreviated version fields change closes https://github.com/cnpm/cnpmjs.org/issues/1667 and starting use the new domain: npmmirror.com --- Dockerfile | 2 +- README.md | 4 +- config/index.js | 4 +- controllers/registry/package/list.js | 8 ++++ controllers/sync_module_worker.js | 27 +++++++------ docs/dockerize/config.js | 2 +- docs/network.puml | 24 ++++++------ docs/web/readme.md | 16 ++++---- services/package.js | 23 ++++++++--- test/controllers/sync_module_worker.test.js | 43 +++++++++++++++++++++ test/fixtures/scope-package/Makefile | 2 +- test/mocks/package/koa/default.js | 2 +- 12 files changed, 111 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e35325..2f5b100 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ WORKDIR ${CNPM_DIR} COPY package.json ${CNPM_DIR} -RUN npm set registry https://registry.npm.taobao.org +RUN npm set registry https://registry.npmmirror.com RUN npm install --production diff --git a/README.md b/README.md index 6635c20..de88b14 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Our goal is to provide a low cost maintenance, easy to use, and easy to scale so ## What can you do with `cnpmjs.org`? * Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now) -* Build a npm mirror. (we use it to build a mirror in China: [https://npm.taobao.org/](https://npm.taobao.org/)) +* Build a npm mirror. (we use it to build a mirror in China: [https://npmmirror.com/](https://npmmirror.com/)) * Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github) ## Features @@ -66,7 +66,7 @@ you only need to change the registry in client config. * [node](http://nodejs.org) >= 8.0.0 * Databases: only required one type - * [sqlite3](https://npm.taobao.org/package/sqlite3) >= 3.0.2, we use `sqlite3` by default + * [sqlite3](https://npmmirror.com/package/sqlite3) >= 3.0.2, we use `sqlite3` by default * [MySQL](http://dev.mysql.com/downloads/) >= 5.6.16, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`. * MariaDB * PostgreSQL diff --git a/config/index.js b/config/index.js index f1fe6f2..bcecf62 100644 --- a/config/index.js +++ b/config/index.js @@ -207,8 +207,8 @@ var config = { // sync source, upstream registry // If you want to directly sync from official npm's registry // please drop them an email first - sourceNpmRegistry: 'https://registry.npm.taobao.org', - sourceNpmWeb: 'https://npm.taobao.org', + sourceNpmRegistry: 'https://registry.npmmirror.com', + sourceNpmWeb: 'https://npmmirror.com', // upstream registry is base on cnpm/cnpmjs.org or not // if your upstream is official npm registry, please turn it off diff --git a/controllers/registry/package/list.js b/controllers/registry/package/list.js index 35af206..a1b1687 100644 --- a/controllers/registry/package/list.js +++ b/controllers/registry/package/list.js @@ -386,6 +386,13 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags continue; } // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object + var hasInstallScript; + if (row.package.scripts) { + // https://www.npmjs.com/package/fix-has-install-script + if (row.package.scripts.install || row.package.scripts.preinstall || row.package.scripts.postinstall) { + hasInstallScript = true; + } + } var pkg = { name: row.package.name, version: row.package.version, @@ -404,6 +411,7 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags engines: row.package.engines, workspaces: row.package.workspaces, _hasShrinkwrap: row.package._hasShrinkwrap, + hasInstallScript: hasInstallScript, publish_time: row.package.publish_time || row.publish_time, }; common.setDownloadURL(pkg, ctx); diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js index fcea02e..d4d980a 100644 --- a/controllers/sync_module_worker.js +++ b/controllers/sync_module_worker.js @@ -841,6 +841,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { // get package AbbreviatedMetadata var remoteAbbreviatedMetadatas = {}; + // store remote abbreviated versions + var remoteAbbreviatedVersionsMap = {}; if (config.enableAbbreviatedMetadata) { // use ?cache=0 tell registry dont use cache result var packageUrl = '/' + name.replace('/', '%2f') + '?cache=0&sync_timestamp=' + Date.now(); @@ -860,9 +862,9 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { name, err, result.headers, result.data); } if (data) { - var versions = data && data.versions || {}; - for (var version in versions) { - const item = versions[version]; + remoteAbbreviatedVersionsMap = data && data.versions || {}; + for (var version in remoteAbbreviatedVersionsMap) { + const item = remoteAbbreviatedVersionsMap[version]; if (!item) { continue; } @@ -874,7 +876,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { metaData._hasShrinkwrap = item._hasShrinkwrap; } - const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces' ]; + // https://github.com/cnpm/cnpmjs.org/issues/1667 + const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces', 'hasInstallScript' ]; for (const key of metaDataKeys) { if (key in item) { hasMetaData = true; @@ -1079,8 +1082,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { if (abbreviatedMetadata) { for (var key in abbreviatedMetadata) { const value = abbreviatedMetadata[key]; - // boolean: _hasShrinkwrap - if (key === '_hasShrinkwrap' && typeof value === 'boolean') { + // boolean: _hasShrinkwrap, hasInstallScript + if ((key === '_hasShrinkwrap' || key === 'hasInstallScript') && typeof value === 'boolean') { if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) { missingAbbreviatedMetadatas.push(Object.assign({ id: exists.id, @@ -1171,7 +1174,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { var tries = 3; while (true) { try { - yield that._syncOneVersion(index, syncModule); + yield that._syncOneVersion(index, syncModule, remoteAbbreviatedVersionsMap[syncModule.version]); syncedVersionNames.push(syncModule.version); break; } catch (err) { @@ -1322,7 +1325,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { that.log(' [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length); var res = yield gather(missingModuleAbbreviateds.map(function (item) { - return packageService.saveModuleAbbreviated(item); + return packageService.saveModuleAbbreviated(item, remoteAbbreviatedVersionsMap[item.version]); })); for (var i = 0; i < res.length; i++) { @@ -1541,7 +1544,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) { return syncedVersionNames; }; -SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) { +SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage, remoteAbbreviatedVersion) { var delay = Date.now() - sourcePackage.publish_time; logger.syncInfo('[sync_module_worker] delay: %s ms, publish_time: %s, start sync %s@%s', delay, utility.logDate(new Date(sourcePackage.publish_time)), @@ -1678,7 +1681,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack throw err; } logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result); - var r = yield afterUpload(result); + var r = yield afterUpload(result, remoteAbbreviatedVersion); logger.syncInfo('[sync_module_worker] sync %s@%s done!', sourcePackage.name, sourcePackage.version); return r; @@ -1687,7 +1690,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack fs.unlink(filepath, utility.noop); } - function *afterUpload(result) { + function *afterUpload(result, remoteAbbreviatedVersion) { //make sure sync module have the correct author info //only if can not get maintainers, use the username var author = username; @@ -1731,7 +1734,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack var r = yield packageService.saveModule(mod); var moduleAbbreviatedId = null; if (config.enableAbbreviatedMetadata) { - var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod); + var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod, remoteAbbreviatedVersion); moduleAbbreviatedId = moduleAbbreviatedResult.id; } diff --git a/docs/dockerize/config.js b/docs/dockerize/config.js index c5cba3f..30f1f1b 100644 --- a/docs/dockerize/config.js +++ b/docs/dockerize/config.js @@ -168,7 +168,7 @@ var config = { // sync source, upstream registry // If you want to directly sync from official npm's registry // please drop them an email first - sourceNpmRegistry: 'https://registry.npm.taobao.org', + sourceNpmRegistry: 'https://registry.npmmirror.com', // upstream registry is base on cnpm/cnpmjs.org or not // if your upstream is official npm registry, please turn it off diff --git a/docs/network.puml b/docs/network.puml index 63518c5..9fd27cd 100644 --- a/docs/network.puml +++ b/docs/network.puml @@ -1,6 +1,6 @@ @startuml -title cnpmjs.org, npm.taobao.org Network +title cnpmjs.org, npmmirror.com Network node "China User" { [cnpm cli] @@ -11,8 +11,8 @@ node "OSS: aliyuncs.com" { } node "SLB: 114.55.80.225 Hangzhou" { - [npm.taobao.org] - [registry.npm.taobao.org] + [npmmirror.com] + [registry.npmmirror.com] } node "npmtaobao3: 121.41.*" { @@ -38,7 +38,7 @@ node "cnpm6: 47.88.189.193 Singapore" { } node "Aliyun CDN" { - [cdn.npm.taobao.org] + [oss.npmmirror.com] } node "qiniu CDN" { @@ -59,17 +59,17 @@ node "npmjs.com" { [www.npmjs.com] } -[cnpm cli] -down-> [registry.npm.taobao.org]: Read, Write -[cnpm cli] -down-> [cdn.npm.taobao.org]: Read tgz -[cnpm cli] -down-> [npm.taobao.org]: "Read /mirrors" +[cnpm cli] -down-> [registry.npmmirror.com]: Read, Write +[cnpm cli] -down-> [oss.npmmirror.com]: Read tgz +[cnpm cli] -down-> [npmmirror.com]: "Read /mirrors" -[registry.npm.taobao.org] -down-> [nginx 5] +[registry.npmmirror.com] -down-> [nginx 5] [nginx 5] -down-> [registry 5] -[npm.taobao.org] -down-> [nginx 5] +[npmmirror.com] -down-> [nginx 5] [nginx 5] -down-> [web 5] -[registry.npm.taobao.org] -down-> [nginx 6] +[registry.npmmirror.com] -down-> [nginx 6] [nginx 6] -down-> [registry 6] -[npm.taobao.org] -down-> [nginx 6] +[npmmirror.com] -down-> [nginx 6] [nginx 6] -down-> [web 6] [registry 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write @@ -92,7 +92,7 @@ node "npmjs.com" { [syncer 7] -down-> [cnpmjs.up.qiniu.com]: Write [syncer 7] -down-> [registry.npmjs.com]: Read -[cdn.npm.taobao.org] -down-> [tnpm-hz.oss-cn-hangzhou]: Read +[oss.npmmirror.com] -down-> [tnpm-hz.oss-cn-hangzhou]: Read [dn-cnpm.qbox.me] -down-> [cnpmjs.up.qiniu.com]: Read @enduml diff --git a/docs/web/readme.md b/docs/web/readme.md index 411d786..20aa8d4 100644 --- a/docs/web/readme.md +++ b/docs/web/readme.md @@ -7,7 +7,7 @@ So `cnpm` is meaning: **Company npm**. - Our public registry: [r.cnpmjs.org](https://r.cnpmjs.org), syncing from [registry.npmjs.com](https://registry.npmjs.com) - [cnpmjs.org](/) version: - [Node.js](https://nodejs.org) version: -- For developers in China, please visit [the China mirror](https://npm.taobao.org). 中国用户请访问[国内镜像站点](https://npm.taobao.org)。 +- For developers in China, please visit [the China mirror](https://npmmirror.com). 中国用户请访问[国内镜像站点](https://npmmirror.com)。 - Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
@@ -81,21 +81,21 @@ Badge URL: `https://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-bad use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm: ```bash -$ npm install -g cnpm --registry=https://registry.npm.taobao.org +$ npm install -g cnpm --registry=https://registry.npmmirror.com ``` Or you can alias NPM to use it: ```bash -alias cnpm="npm --registry=https://registry.npm.taobao.org \ +alias cnpm="npm --registry=https://registry.npmmirror.com \ --cache=$HOME/.npm/.cache/cnpm \ ---disturl=https://npm.taobao.org/mirrors/node \ +--disturl=https://npmmirror.com/mirrors/node \ --userconfig=$HOME/.cnpmrc" #Or alias it in .bashrc or .zshrc -$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npm.taobao.org \ +$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npmmirror.com \ --cache=$HOME/.npm/.cache/cnpm \ - --disturl=https://npm.taobao.org/mirrors/node \ + --disturl=https://npmmirror.com/mirrors/node \ --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc ``` @@ -118,7 +118,7 @@ $ cnpm sync connect sync package on web: [sync/connect](/sync/connect) ```bash -$ open http://registry.npm.taobao.org/sync/connect +$ open http://registry.npmmirror.com/sync/connect ``` ### publish / unpublish @@ -146,7 +146,7 @@ $ cnpm info cnpm Release [History](/history). -## npmjs.org, cnpmjs.org and npm.taobao.org relation +## npmjs.org, cnpmjs.org and npmmirror.com relation ![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png) diff --git a/services/package.js b/services/package.js index 40846ec..1b84c16 100644 --- a/services/package.js +++ b/services/package.js @@ -355,11 +355,22 @@ exports.findAllModuleAbbreviateds = function* (where, order, limit, offset) { }; // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object -exports.saveModuleAbbreviated = function* (mod) { - var pkg = JSON.stringify(Object.assign({}, mod.package, { - // ignore readme force - readme: undefined, - })); +exports.saveModuleAbbreviated = function* (mod, remoteAbbreviatedVersion) { + // try to use remoteAbbreviatedVersion first + var pkg; + if (remoteAbbreviatedVersion) { + pkg = Object.assign({}, remoteAbbreviatedVersion, { + // override remote tarball + dist: Object.assign({}, remoteAbbreviatedVersion.dist, mod.package.dist, { + noattachment: undefined, + }), + }); + } else { + pkg = Object.assign({}, mod.package, { + // ignore readme force + readme: undefined, + }); + } var publish_time = mod.publish_time || Date.now(); var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version); if (!item) { @@ -369,7 +380,7 @@ exports.saveModuleAbbreviated = function* (mod) { }); } item.publish_time = publish_time; - item.package = pkg; + item.package = JSON.stringify(pkg); if (item.changed()) { item = yield item.save(); diff --git a/test/controllers/sync_module_worker.test.js b/test/controllers/sync_module_worker.test.js index 3ca9881..3d31a1d 100644 --- a/test/controllers/sync_module_worker.test.js +++ b/test/controllers/sync_module_worker.test.js @@ -257,6 +257,49 @@ describe('test/controllers/sync_module_worker.test.js', () => { assert(newEtag == lastResHeaders.etag); }); + it('should sync mk2test-module-cnpmsync-issue-1667 with remoteAbbreviatedVersion success', function* () { + mm(config, 'enableAbbreviatedMetadata', true); + mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.com'); + var log = yield logService.create({ + name: 'mk2test-module-cnpmsync-issue-1667', + username: 'fengmk2', + }); + log.id.should.above(0); + var worker = new SyncModuleWorker({ + logId: log.id, + name: 'mk2test-module-cnpmsync-issue-1667', + username: 'fengmk2', + noDep: true, + }); + worker.start(); + var end = thunkify.event(worker, 'end'); + yield end(); + + let pkg; + let pkgV2; + let pkgV3; + let lastResHeaders; + function checkResult() { + return function (done) { + request(app) + .get('/mk2test-module-cnpmsync-issue-1667') + .set('accept', 'application/vnd.npm.install-v1+json') + .expect(function (res) { + lastResHeaders = res.headers; + console.log('%j', res.body); + pkg = res.body.versions['3.0.0']; + assert(pkg.hasInstallScript === true); + // no scripts + assert(!pkg.scripts); + assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz'); + assert(!('noattachment' in pkg.dist)); + }) + .expect(200, done); + }; + } + yield checkResult(); + }); + it('should sync upstream first', function* () { mm(config, 'sourceNpmRegistryIsCNpm', true); var log = yield logService.create({ diff --git a/test/fixtures/scope-package/Makefile b/test/fixtures/scope-package/Makefile index ca847ed..19b094a 100644 --- a/test/fixtures/scope-package/Makefile +++ b/test/fixtures/scope-package/Makefile @@ -4,7 +4,7 @@ TIMEOUT = 1000 MOCHA_OPTS = install: - @npm install --registry=https://registry.npm.taobao.org --disturl=https://npm.taobao.org/dist + @npm install --registry=https://registry.npmmirror.com --disturl=https://npmmirror.com/dist jshint: install @./node_modules/.bin/jshint . diff --git a/test/mocks/package/koa/default.js b/test/mocks/package/koa/default.js index fc85f31..d2e3ce3 100644 --- a/test/mocks/package/koa/default.js +++ b/test/mocks/package/koa/default.js @@ -5,7 +5,7 @@ module.exports = { name: 'koa', version: '0.13.0', description: 'description: Koa web app framework', - registryUrl: 'https://registry.npm.taobao.org/koa', + registryUrl: 'https://registry.npmmirror.com/koa', engines: { node: { version: '>= 0.11.13',