refactor: use remote abbreviated version data (#1675)

avoid npm abbreviated version fields change closes https://github.com/cnpm/cnpmjs.org/issues/1667 and starting use the new domain: npmmirror.com
2021-11-07 23:38:32 +08:00
parent 37ba629028
commit 39cf77ae0f
12 changed files with 111 additions and 46 deletions
--- a/2
+++ b/2
@@ -12,7 +12,7 @@ WORKDIR ${CNPM_DIR}

 COPY package.json ${CNPM_DIR}

-RUN npm set registry https://registry.npm.taobao.org
+RUN npm set registry https://registry.npmmirror.com

 RUN npm install --production

--- a/README.md
+++ b/README.md
@@ -30,7 +30,7 @@ Our goal is to provide a low cost maintenance, easy to use, and easy to scale so
 ## What can you do with `cnpmjs.org`?

 * Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
-* Build a npm mirror. (we use it to build a mirror in China: [https://npm.taobao.org/](https://npm.taobao.org/))
+* Build a npm mirror. (we use it to build a mirror in China: [https://npmmirror.com/](https://npmmirror.com/))
 * Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)

 ## Features
@@ -66,7 +66,7 @@ you only need to change the registry in client config.

 * [node](http://nodejs.org) >= 8.0.0
 * Databases: only required one type
-  * [sqlite3](https://npm.taobao.org/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
+  * [sqlite3](https://npmmirror.com/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
  * [MySQL](http://dev.mysql.com/downloads/) >= 5.6.16, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
  * MariaDB
  * PostgreSQL
--- a/config/index.js
+++ b/config/index.js
@@ -207,8 +207,8 @@ var config = {
  // sync source, upstream registry
  // If you want to directly sync from official npm's registry
  // please drop them an email first
-  sourceNpmRegistry: 'https://registry.npm.taobao.org',
-  sourceNpmWeb: 'https://npm.taobao.org',
+  sourceNpmRegistry: 'https://registry.npmmirror.com',
+  sourceNpmWeb: 'https://npmmirror.com',

  // upstream registry is base on cnpm/cnpmjs.org or not
  // if your upstream is official npm registry, please turn it off
--- a/controllers/registry/package/list.js
+++ b/controllers/registry/package/list.js
@@ -386,6 +386,13 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
      continue;
    }
    // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+    var hasInstallScript;
+    if (row.package.scripts) {
+      // https://www.npmjs.com/package/fix-has-install-script
+      if (row.package.scripts.install || row.package.scripts.preinstall || row.package.scripts.postinstall) {
+        hasInstallScript = true;
+      }
+    }
    var pkg = {
      name: row.package.name,
      version: row.package.version,
@@ -404,6 +411,7 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
      engines: row.package.engines,
      workspaces: row.package.workspaces,
      _hasShrinkwrap: row.package._hasShrinkwrap,
+      hasInstallScript: hasInstallScript,
      publish_time: row.package.publish_time || row.publish_time,
    };
    common.setDownloadURL(pkg, ctx);
--- a/controllers/sync_module_worker.js
+++ b/controllers/sync_module_worker.js
@@ -841,6 +841,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {

  // get package AbbreviatedMetadata
  var remoteAbbreviatedMetadatas = {};
+  // store remote abbreviated versions
+  var remoteAbbreviatedVersionsMap = {};
  if (config.enableAbbreviatedMetadata) {
    // use ?cache=0 tell registry dont use cache result
    var packageUrl = '/' + name.replace('/', '%2f') + '?cache=0&sync_timestamp=' + Date.now();
@@ -860,9 +862,9 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
          name, err, result.headers, result.data);
      }
      if (data) {
-        var versions = data && data.versions || {};
-        for (var version in versions) {
-          const item = versions[version];
+        remoteAbbreviatedVersionsMap = data && data.versions || {};
+        for (var version in remoteAbbreviatedVersionsMap) {
+          const item = remoteAbbreviatedVersionsMap[version];
          if (!item) {
            continue;
          }
@@ -874,7 +876,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
            metaData._hasShrinkwrap = item._hasShrinkwrap;
          }

-          const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces' ];
+          // https://github.com/cnpm/cnpmjs.org/issues/1667
+          const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'workspaces', 'hasInstallScript' ];
          for (const key of metaDataKeys) {
            if (key in item) {
              hasMetaData = true;
@@ -1079,8 +1082,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
        if (abbreviatedMetadata) {
          for (var key in abbreviatedMetadata) {
            const value = abbreviatedMetadata[key];
-            // boolean: _hasShrinkwrap
-            if (key === '_hasShrinkwrap' && typeof value === 'boolean') {
+            // boolean: _hasShrinkwrap, hasInstallScript
+            if ((key === '_hasShrinkwrap' || key === 'hasInstallScript') && typeof value === 'boolean') {
              if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
                missingAbbreviatedMetadatas.push(Object.assign({
                  id: exists.id,
@@ -1171,7 +1174,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
    var tries = 3;
    while (true) {
      try {
-        yield that._syncOneVersion(index, syncModule);
+        yield that._syncOneVersion(index, syncModule, remoteAbbreviatedVersionsMap[syncModule.version]);
        syncedVersionNames.push(syncModule.version);
        break;
      } catch (err) {
@@ -1322,7 +1325,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
    that.log('  [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length);

    var res = yield gather(missingModuleAbbreviateds.map(function (item) {
-      return packageService.saveModuleAbbreviated(item);
+      return packageService.saveModuleAbbreviated(item, remoteAbbreviatedVersionsMap[item.version]);
    }));

    for (var i = 0; i < res.length; i++) {
@@ -1541,7 +1544,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  return syncedVersionNames;
 };

-SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage, remoteAbbreviatedVersion) {
  var delay = Date.now() - sourcePackage.publish_time;
  logger.syncInfo('[sync_module_worker] delay: %s ms, publish_time: %s, start sync %s@%s',
    delay, utility.logDate(new Date(sourcePackage.publish_time)),
@@ -1678,7 +1681,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
      throw err;
    }
    logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result);
-    var r = yield afterUpload(result);
+    var r = yield afterUpload(result, remoteAbbreviatedVersion);
    logger.syncInfo('[sync_module_worker] sync %s@%s done!',
      sourcePackage.name, sourcePackage.version);
    return r;
@@ -1687,7 +1690,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
    fs.unlink(filepath, utility.noop);
  }

-  function *afterUpload(result) {
+  function *afterUpload(result, remoteAbbreviatedVersion) {
    //make sure sync module have the correct author info
    //only if can not get maintainers, use the username
    var author = username;
@@ -1731,7 +1734,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
    var r = yield packageService.saveModule(mod);
    var moduleAbbreviatedId = null;
    if (config.enableAbbreviatedMetadata) {
-      var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod);
+      var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod, remoteAbbreviatedVersion);
      moduleAbbreviatedId = moduleAbbreviatedResult.id;
    }

--- a/docs/dockerize/config.js
+++ b/docs/dockerize/config.js
@@ -168,7 +168,7 @@ var config = {
  // sync source, upstream registry
  // If you want to directly sync from official npm's registry
  // please drop them an email first
-  sourceNpmRegistry: 'https://registry.npm.taobao.org',
+  sourceNpmRegistry: 'https://registry.npmmirror.com',

  // upstream registry is base on cnpm/cnpmjs.org or not
  // if your upstream is official npm registry, please turn it off
--- a/docs/network.puml
+++ b/docs/network.puml
@@ -1,6 +1,6 @@
@startuml

-title cnpmjs.org, npm.taobao.org Network
+title cnpmjs.org, npmmirror.com Network

 node "China User" {
  [cnpm cli]
@@ -11,8 +11,8 @@ node "OSS: aliyuncs.com" {
 }

 node "SLB: 114.55.80.225 Hangzhou" {
-  [npm.taobao.org]
-  [registry.npm.taobao.org]
+  [npmmirror.com]
+  [registry.npmmirror.com]
 }

 node "npmtaobao3: 121.41.*" {
@@ -38,7 +38,7 @@ node "cnpm6: 47.88.189.193 Singapore" {
 }

 node "Aliyun CDN" {
-  [cdn.npm.taobao.org]
+  [oss.npmmirror.com]
 }

 node "qiniu CDN" {
@@ -59,17 +59,17 @@ node "npmjs.com" {
  [www.npmjs.com]
 }

-[cnpm cli] -down-> [registry.npm.taobao.org]: Read, Write
-[cnpm cli] -down-> [cdn.npm.taobao.org]: Read tgz
-[cnpm cli] -down-> [npm.taobao.org]: "Read /mirrors"
+[cnpm cli] -down-> [registry.npmmirror.com]: Read, Write
+[cnpm cli] -down-> [oss.npmmirror.com]: Read tgz
+[cnpm cli] -down-> [npmmirror.com]: "Read /mirrors"

-[registry.npm.taobao.org] -down-> [nginx 5]
+[registry.npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [registry 5]
-[npm.taobao.org] -down-> [nginx 5]
+[npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [web 5]
-[registry.npm.taobao.org] -down-> [nginx 6]
+[registry.npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [registry 6]
-[npm.taobao.org] -down-> [nginx 6]
+[npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [web 6]

 [registry 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
@@ -92,7 +92,7 @@ node "npmjs.com" {
 [syncer 7] -down-> [cnpmjs.up.qiniu.com]: Write
 [syncer 7] -down-> [registry.npmjs.com]: Read

-[cdn.npm.taobao.org] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
+[oss.npmmirror.com] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
 [dn-cnpm.qbox.me] -down-> [cnpmjs.up.qiniu.com]: Read

@enduml
--- a/docs/web/readme.md
+++ b/docs/web/readme.md
@@ -7,7 +7,7 @@ So `cnpm` is meaning: **Company npm**.
 - Our public registry: [r.cnpmjs.org](https://r.cnpmjs.org), syncing from [registry.npmjs.com](https://registry.npmjs.com)
 - [cnpmjs.org](/) version: <span id="app-version"></span>
 - [Node.js](https://nodejs.org) version: <span id="node-version"></span>
- For developers in China, please visit [the China mirror](https://npm.taobao.org). 中国用户请访问[国内镜像站点](https://npm.taobao.org)。
+- For developers in China, please visit [the China mirror](https://npmmirror.com). 中国用户请访问[国内镜像站点](https://npmmirror.com)。
 - Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)

 <div class="ant-table">
@@ -81,21 +81,21 @@ Badge URL: `https://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-bad
 use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:

 ```bash
-$ npm install -g cnpm --registry=https://registry.npm.taobao.org
+$ npm install -g cnpm --registry=https://registry.npmmirror.com
 ```

 Or you can alias NPM to use it:

 ```bash
-alias cnpm="npm --registry=https://registry.npm.taobao.org \
+alias cnpm="npm --registry=https://registry.npmmirror.com \
 --cache=$HOME/.npm/.cache/cnpm \
--disturl=https://npm.taobao.org/mirrors/node \
+--disturl=https://npmmirror.com/mirrors/node \
 --userconfig=$HOME/.cnpmrc"

 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npm.taobao.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npmmirror.com \
  --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=https://npm.taobao.org/mirrors/node \
+  --disturl=https://npmmirror.com/mirrors/node \
  --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```

@@ -118,7 +118,7 @@ $ cnpm sync connect
 sync package on web: [sync/connect](/sync/connect)

 ```bash
-$ open http://registry.npm.taobao.org/sync/connect
+$ open http://registry.npmmirror.com/sync/connect
 ```

 ### publish / unpublish
@@ -146,7 +146,7 @@ $ cnpm info cnpm

 Release [History](/history).

-## npmjs.org, cnpmjs.org and npm.taobao.org relation
+## npmjs.org, cnpmjs.org and npmmirror.com relation

 ![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png)

--- a/services/package.js
+++ b/services/package.js
@@ -355,11 +355,22 @@ exports.findAllModuleAbbreviateds = function* (where, order, limit, offset) {
 };

 // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
-exports.saveModuleAbbreviated = function* (mod) {
-  var pkg = JSON.stringify(Object.assign({}, mod.package, {
-    // ignore readme force
-    readme: undefined,
-  }));
+exports.saveModuleAbbreviated = function* (mod, remoteAbbreviatedVersion) {
+  // try to use remoteAbbreviatedVersion first
+  var pkg;
+  if (remoteAbbreviatedVersion) {
+    pkg = Object.assign({}, remoteAbbreviatedVersion, {
+      // override remote tarball
+      dist: Object.assign({}, remoteAbbreviatedVersion.dist, mod.package.dist, {
+        noattachment: undefined,
+      }),
+    });
+  } else {
+    pkg = Object.assign({}, mod.package, {
+      // ignore readme force
+      readme: undefined,
+    });
+  }
  var publish_time = mod.publish_time || Date.now();
  var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version);
  if (!item) {
@@ -369,7 +380,7 @@ exports.saveModuleAbbreviated = function* (mod) {
    });
  }
  item.publish_time = publish_time;
-  item.package = pkg;
+  item.package = JSON.stringify(pkg);

  if (item.changed()) {
    item = yield item.save();
--- a/test/controllers/sync_module_worker.test.js
+++ b/test/controllers/sync_module_worker.test.js
@@ -257,6 +257,49 @@ describe('test/controllers/sync_module_worker.test.js', () => {
    assert(newEtag == lastResHeaders.etag);
  });

+  it('should sync mk2test-module-cnpmsync-issue-1667 with remoteAbbreviatedVersion success', function* () {
+    mm(config, 'enableAbbreviatedMetadata', true);
+    mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.com');
+    var log = yield logService.create({
+      name: 'mk2test-module-cnpmsync-issue-1667',
+      username: 'fengmk2',
+    });
+    log.id.should.above(0);
+    var worker = new SyncModuleWorker({
+      logId: log.id,
+      name: 'mk2test-module-cnpmsync-issue-1667',
+      username: 'fengmk2',
+      noDep: true,
+    });
+    worker.start();
+    var end = thunkify.event(worker, 'end');
+    yield end();
+
+    let pkg;
+    let pkgV2;
+    let pkgV3;
+    let lastResHeaders;
+    function checkResult() {
+      return function (done) {
+        request(app)
+        .get('/mk2test-module-cnpmsync-issue-1667')
+        .set('accept', 'application/vnd.npm.install-v1+json')
+        .expect(function (res) {
+          lastResHeaders = res.headers;
+          console.log('%j', res.body);
+          pkg = res.body.versions['3.0.0'];
+          assert(pkg.hasInstallScript === true);
+          // no scripts
+          assert(!pkg.scripts);
+          assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz');
+          assert(!('noattachment' in pkg.dist));
+        })
+        .expect(200, done);
+      };
+    }
+    yield checkResult();
+  });
+
  it('should sync upstream first', function* () {
    mm(config, 'sourceNpmRegistryIsCNpm', true);
    var log = yield logService.create({
--- a/test/fixtures/scope-package/Makefile
+++ b/test/fixtures/scope-package/Makefile
@@ -4,7 +4,7 @@ TIMEOUT = 1000
 MOCHA_OPTS =

 install:
-	@npm install --registry=https://registry.npm.taobao.org --disturl=https://npm.taobao.org/dist
+	@npm install --registry=https://registry.npmmirror.com --disturl=https://npmmirror.com/dist

 jshint: install
 	@./node_modules/.bin/jshint .
--- a/test/mocks/package/koa/default.js
+++ b/test/mocks/package/koa/default.js
@@ -5,7 +5,7 @@ module.exports = {
    name: 'koa',
    version: '0.13.0',
    description: 'description: Koa web app framework',
-    registryUrl: 'https://registry.npm.taobao.org/koa',
+    registryUrl: 'https://registry.npmmirror.com/koa',
    engines: {
      node: {
        version: '>= 0.11.13',