feat: support sync private package from define registry (#1701)

Co-authored-by: wangjx2018 <wangjx2018@chinaunicom.cn>
2022-02-14 00:16:00 +08:00
parent 19e5c3def2
commit f52e9c3382
9 changed files with 366 additions and 13 deletions
--- a/config/index.js
+++ b/config/index.js
@@ -248,6 +248,20 @@ var config = {
  syncDownloadOptions: {
    // formatRedirectUrl: function (url, location)
  },
+
+  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
+  syncScope: false,
+  syncScopeInterval: '12h',
+  // scope package sync config
+  /**
+ * sync scope package from assign registry
+ * @param {Array<scope>} scopes
+ * @param {String} scope.scope scope name
+ * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
+ * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
+ */
+  syncScopeConfig: [],
+
  handleSyncRegistry: 'http://127.0.0.1:7001',

  // default badge subject
--- a/controllers/sync.js
+++ b/controllers/sync.js
@@ -2,6 +2,7 @@

 var debug = require('debug')('cnpmjs.org:controllers:sync');
 var Log = require('../services/module_log');
+var npmService = require('../services/npm');
 var SyncModuleWorker = require('./sync_module_worker');
 var config = require('../config');

@@ -52,6 +53,55 @@ exports.sync = function* () {
  };
 };

+exports.scopeSync = function* () {
+  var scope = this.params.scope;
+
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    this.status = 404;
+    this.body = {
+      error: 'no_scope',
+      reason: 'only has syncScopeConfig config can use this feature'
+    };
+    return;
+  }
+
+  var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
+  var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
+  var packages = yield npmService.getScopePackagesShort(scope, scopeCnpmWeb)
+
+  debug('scopeSync %s with query: %j', scope, this.query);
+
+  var packageSyncWorkers = []
+
+  for (let i = 0; i < packages.length; i++) {
+    packageSyncWorkers.push(function* () {
+      var name = packages[i]
+      var logId = yield SyncModuleWorker.sync(name, 'admin', {
+        type: 'package',
+        publish: true,
+        noDep: true,
+        syncUpstreamFirst: false,
+        syncPrivatePackage: { [scope]: scopeCnpmRegistry }
+      })
+      return { name: name, logId: logId }
+    })
+  }
+
+  var logIds = yield packageSyncWorkers
+
+  debug('scopeSync %s got log id %j', scope, logIds);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logIds: logIds
+  };
+};
+
 exports.getSyncLog = function* (next) {
  var logId = Number(this.params.id || this.params[1]);
  var offset = Number(this.query.offset) || 0;
--- a/controllers/sync_module_worker.js
+++ b/controllers/sync_module_worker.js
@@ -49,6 +49,8 @@ function SyncModuleWorker(options) {
  this.names = options.name;
  this.startName = this.names[0];

+  this.syncPrivatePackage = options.syncPrivatePackage;
+
  this.username = options.username;
  this.concurrency = options.concurrency || 1;
  this._publish = options.publish === true; // _publish_on_cnpm
@@ -319,14 +321,21 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
    return setImmediate(this.finish.bind(this));
  }

-  if (config.syncModel === 'none') {
+  const defineRegistry = this.getPrivatePackageDefineRegistry(name)
+
+  if (!defineRegistry && config.syncModel === 'none') {
    this.log('[c#%d] [%s] syncModel is none, ignore',
-      concurrencyId, name);
+     concurrencyId, name);
    return this.finish();
  }

-  // try to sync from official replicate when source npm registry is not cnpmjs.org
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  // try to sync from official replicate when no defineRegistry or source npm registry is not cnpmjs.org
+  let registry
+  if (defineRegistry) {
+    registry = defineRegistry
+  } else {
+    registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  }

  yield this.syncByName(concurrencyId, name, registry);
 };
@@ -526,8 +535,9 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry

  this.log('----------------- Syncing %s -------------------', name);

+  const isNeedSyncPrivatePackage = this.getPrivatePackageDefineRegistry(name)
  // ignore private scoped package
-  if (common.isPrivateScopedPackage(name)) {
+  if (!isNeedSyncPrivatePackage && common.isPrivateScopedPackage(name)) {
    this.log('[c#%d] [%s] ignore sync private scoped %j package',
      concurrencyId, name, config.scopes);
    yield this._doneOne(concurrencyId, name, true);
@@ -693,6 +703,21 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
  return versions;
 };

+SyncModuleWorker.prototype.getPrivatePackageDefineRegistry = function (name) {
+  if (typeof name !== 'string') return false
+  return this.syncPrivatePackage && this.syncPrivatePackage[name.split('/')[0]]
+}
+
+SyncModuleWorker.prototype.isLocalModule = function (mods) {
+  var res = common.isLocalModule(mods)
+  if (!this.syncPrivatePackage) return res
+  if (!mods[0] || !mods[0].package || !mods[0].package.name) return res
+
+  if (this.getPrivatePackageDefineRegistry(mods[0].package.name)) return false
+
+  return res
+}
+
 function* _listStarUsers(modName) {
  var users = yield packageService.listStarUserNames(modName);
  var userMap = {};
@@ -733,7 +758,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
  var mods = yield packageService.listModulesByName(name);
  this.log('  [%s] start unpublished %d versions from local cnpm registry',
    name, mods.length);
-  if (common.isLocalModule(mods)) {
+  if (this.isLocalModule(mods)) {
    // publish on cnpm, dont sync this version package
    this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
    return [];
@@ -801,7 +826,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
  var existsNpmMaintainers = result[3];
  var existsModuleAbbreviateds = result[4];

-  if (common.isLocalModule(moduleRows)) {
+  if (this.isLocalModule(moduleRows)) {
    // publish on cnpm, dont sync this version package
    that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
    return [];
@@ -1913,6 +1938,7 @@ SyncModuleWorker.sync = function* (name, username, options) {
    publish: options.publish,
    syncUpstreamFirst: options.syncUpstreamFirst,
    syncFromBackupFile: options.syncFromBackupFile,
+    syncPrivatePackage: options.syncPrivatePackage
  });
  worker.start();
  return result.id;
--- a/controllers/web/show_scope_sync.js
+++ b/controllers/web/show_scope_sync.js
@@ -0,0 +1,23 @@
+'use strict';
+var config = require('../../config');
+var npmService = require('../../services/npm');
+
+module.exports = function* showScopeSync () {
+  var scope = this.params.scope;
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    return this.redirect('/');
+  }
+
+  var packages = yield npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
+
+  yield this.render('scope_sync', {
+    packages: packages,
+    scope: scopeConfig.scope,
+    sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
+    title: 'Sync Scope Packages',
+  });
+};
--- a/dispatch.js
+++ b/dispatch.js
@@ -7,20 +7,21 @@ var cfork = require('cfork');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
 var syncPath = path.join(__dirname, 'sync');
+var scopeSyncPath = path.join(__dirname, 'sync/sync_scope');

 console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s',
  config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel);

 if (config.enableCluster) {
  forkWorker();
-  if (config.syncModel !== 'none') {
-    forkSyncer();
-  }
+  config.syncModel !== 'none' && forkSyncer();
+  // scync assign pravate scope package
+  config.syncScope && forkScopeSyncer();
 } else {
  require(workerPath);
-  if (config.syncModel !== 'none') {
-    require(syncPath);
-  }
+  config.syncModel !== 'none' && require(syncPath);
+  // scync assign pravate scope package
+  config.syncScope && require(scopeSyncPath);
 }

 function forkWorker() {
@@ -52,3 +53,15 @@ function forkSyncer() {
    setTimeout(forkSyncer, 1000);
  });
 }
+
+function forkScopeSyncer() {
+  var syncer = childProcess.fork(scopeSyncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkScopeSyncer, 1000);
+  });
+}
--- a/routes/web.js
+++ b/routes/web.js
@@ -5,6 +5,7 @@ var searchPackage = require('../controllers/web/package/search');
 var searchRange = require('../controllers/web/package/search_range');
 var listPrivates = require('../controllers/web/package/list_privates');
 var showSync = require('../controllers/web/show_sync');
+var showScopeSync = require('../controllers/web/show_scope_sync');
 var showUser = require('../controllers/web/user/show');
 var sync = require('../controllers/sync');
 var showTotal = require('../controllers/total');
@@ -35,6 +36,9 @@ function routes(app) {
  app.put(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, sync.sync);
  app.put('/sync/:name', sync.sync);

+  app.get('/scopeSync/:scope', showScopeSync);
+  app.put('/scopeSync/:scope', sync.scopeSync);
+
  app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog);
  app.get('/sync/:name/log/:id', sync.getSyncLog);

--- a/services/npm.js
+++ b/services/npm.js
@@ -238,3 +238,13 @@ exports.listChanges = function* (updateSeq) {
  // {"results":[{"seq":1988653,"type":"PACKAGE_VERSION_ADDED","id":"dsr-package-mercy-magot-thorp-sward","changes":[{"version":"1.0.1"}]},
  return data.results || [];
 };
+
+exports.getScopePackagesShort = function* (scope, registry) {
+  var response = yield request('/browse/keyword/' + scope, {
+    timeout: 3000,
+    registry: registry,
+    dataType: 'text'
+  });
+  var res = response.data.match(/class="package-name">(\S*)<\/a>/g)
+  return res ? res.map(a => a.match(/class="package-name">(\S*)<\/a>/)[1]).filter(name => name.indexOf(`${scope}/`) === 0) : []
+};
--- a/sync/sync_scope.js
+++ b/sync/sync_scope.js
@@ -0,0 +1,89 @@
+'use strict';
+
+var thunkify = require('thunkify-wrap');
+const co = require('co');
+const ms = require('humanize-ms');
+var config = require('../config');
+var npmService = require('../services/npm');
+var SyncModuleWorker = require('../controllers/sync_module_worker');
+var logger = require('../common/logger');
+
+
+let syncing = false;
+const syncFn = co.wrap(function*() {
+  if (syncing) { return; }
+  syncing = true;
+  logger.syncInfo('Start syncing scope modules...');
+  let data;
+  let error;
+  try {
+    data = yield sync();
+  } catch (err) {
+    error = err;
+    error.message += ' (sync package error)';
+    logger.syncError(error);
+  }
+
+  if (data) {
+    logger.syncInfo(data);
+  }
+  if (!config.debug) {
+    sendMailToAdmin(error, data, new Date());
+  }
+  syncing = false;
+});
+
+syncFn().catch(onerror);
+setInterval(() => syncFn().catch(onerror), ms(config.syncScopeInterval));
+
+function onerror(err) {
+  logger.error('====================== scope sync error ========================');
+  logger.error(err);
+}
+
+function* getOtherCnpmDefineScopePackages(scopes) {
+  var arr = []
+  for (var i = 0; i < scopes.length; i++) {
+    var packageList = yield npmService.getScopePackagesShort(scopes[i].scope, scopes[i].sourceCnpmWeb)
+    arr = arr.concat(packageList)
+  }
+  return arr
+}
+
+function* sync() {
+  var scopeConfig = config.syncScopeConfig
+  if (!scopeConfig || scopeConfig.length === 0) {
+    process.exit(0);
+  }
+  var packages = yield getOtherCnpmDefineScopePackages(scopeConfig);
+
+  if (!packages.length) {
+    return;
+  }
+  logger.syncInfo('Total %d scope packages to sync: %j', packages.length, packages);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    syncUpstreamFirst: false,
+    publish: true,
+    concurrency: config.syncConcurrency,
+    syncPrivatePackage: scopeConfig.reduce((arr, cur) => {
+      arr[cur.scope] = cur.sourceCnpmRegistry
+      return arr
+    }, {})
+  });
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  logger.syncInfo('scope packages sync done, successes %d, fails %d, updates %d',
+      worker.successes.length, worker.fails.length, worker.updates.length);
+
+  return {
+    successes: worker.successes,
+    fails: worker.fails,
+    updates: worker.updates,
+  };
+};
--- a/view/web/scope_sync.html
+++ b/view/web/scope_sync.html
@@ -0,0 +1,124 @@
+<div id="scopeSync">
+  <h2>Scope Packages Sync</h2>
+  <div>
+    <div><%= scope %></div>
+    <div>source registry ：<%= sourceCnpmRegistry %></div>
+    <div>scope package number：<%= packages.length %> </div>
+    <div id="last-sync-time">Last Sync Time: </div>
+    <button id='start-button' onclick="startAllSync()">Start sync all scope packages</button>
+  </div>
+  <ul>
+    <% for(var i=0;i<packages.length;i++){ %>
+      <li id="sync-result-<%= packages[i] %>"><%= packages[i] %>： wait sync</li>
+    <% } %>
+  </ul>
+  <h2>Log</h2>
+  <div id="sync-logs">
+    <% for(var i=0;i<packages.length;i++){ %>
+      <h2>Sync <span style="color:#09f;"><%= packages[i] %></span></h2>
+      <div id="sync-notify-<%= packages[i] %>">
+        <div class="ant-alert ant-alert-success">Sync started, please wait patiently.</div>
+      </div>
+      <pre style="min-height: 400px;" id="sync-log-<%= packages[i] %>"></pre>
+    <% } %>
+  </div>
+</div>
+<script>
+  var scope = '<%= scope %>';
+  var packages = '<%= packages %>';
+  var excutePackages = packages.split(',').map(function (name) {
+    var logId = localStorage.getItem(name)
+    var obj = {
+      name: name,
+      logTimer: '',
+      logId: logId,
+      syncDone: false,
+      hasFail: false
+    }
+    logId && getSyncLog(obj)
+
+    return obj
+  })
+
+  var lastSyncStartTime = localStorage.getItem(`${scope}_last_sync_start_time`)
+  lastSyncStartTime && $(document.getElementById('last-sync-time')).html('<div class="ant-alert ant-alert-success">本浏览器上次同步时间为：' + lastSyncStartTime + '</div>')
+
+  function startAllSync () {
+    if (lastSyncStartTime) {
+      if (!window.confirm(`laste sync time is ${lastSyncStartTime}，are you sure restart ?`)) return
+    }
+
+    $.ajax({
+        url: `/scopeSync/${scope}`,
+        type: 'PUT',
+        dataType: 'json',
+        success: function (data) {
+          if (!data.ok) {
+            console.log(data)
+            alert('Sync request error.')
+            return
+          }
+          localStorage.setItem(`${scope}_last_sync_start_time`, new Date())
+
+          data.logIds.forEach(({ name, logId }) => {
+            localStorage.setItem(name, logId)
+            $(document.getElementById('sync-result-' + name)).html(name + '---------------- Syncing ----------------')
+            var obj = excutePackages.find(obj => obj.name === name)
+            obj.logId = logId
+            obj.logTimer = setInterval(function () { getSyncLog(obj) }, 2000);
+          })
+        },
+        error: function (err) {
+          console.log(err)
+          alert(JSON.stringify(err))
+        }
+      });
+  }
+
+  function getSyncLog(obj) {
+    var name = obj.name
+    var id = obj.logId
+    var timer = obj.logTimer
+    var $log = $(document.getElementById('sync-log-' + name));
+    var $notify = $(document.getElementById('sync-notify-' + name));
+    var $syncResult = $(document.getElementById('sync-result-' + name));
+
+    $.ajax({
+      url: '/sync/' + name + '/log/' + id,
+      type: 'GET',
+      dataType: 'json',
+      success: function (data) {
+        if (!data.ok) {
+          return;
+        }
+
+        var log = data.log || '';
+
+        obj.syncDone = data.syncDone;
+
+        if (log.indexOf('Fail: [') >= 0) {
+          var failInfo = log.match(/Fail: \[ (.*?) \]/);
+          if (failInfo && failInfo[1]) {
+            obj.hasFail = true;
+          }
+        }
+        if (obj.syncDone) {
+          log += '\nSync ' + name + ' complete!';
+          if (obj.hasFail) {
+            log += ' scome package sync faile，you can try cleae localStorage and restart sync';
+            location.hash = '';
+            $notify.html('<div class="ant-alert ant-alert-error">～～～～～～sync failed～～～～～～</div>');
+            $syncResult.html('<div style="color: red">' + name + '：Sync failed!!!!!!!!!!!!!' + '</div>')
+          } else {
+            $notify.html('<div class="ant-alert ant-alert-success">Sync success.</div>');
+            $syncResult.html('<div style="color: #4deb4d">' + name + '：Sync success' + '</div>')
+          }
+          clearInterval(timer);
+        } else {
+          $syncResult.html(name + '---------------- Syncing ----------------')
+        }
+        $log.html(log);
+      }
+    });
+  }
+</script>