feat: sync delete mode (#398)

> 为了避免部分 npm 包误封、误删，导致生产环境影响，新增 syncDeleteMode 配置，允许自定义同步策略 * 新增 `syncDeleteMode` : 'ignore' | 'block' | 'delete' * delete: 目前默认值，同步删包事件 * ignore: 忽略 upstream 所有删包事件 * block: 不做物理删除，只新增 block 记录，不允许访问，除非管理员手动恢复并更新 `syncPackageBlockList` * `npm-security-holder` 场景也判断为删包事件 * 更新原有删包流程，统一处理，调整部分日志输出 --------------- > New `syncDeleteMode` to allow custom syncing policy to avoid some npm packages being blocked or deleted by mistake. * Add `syncDeleteMode` : 'ignore' | 'block' | 'delete' * delete: by default, sync delete events * ignore: ignore all upstream delete events * block: only add block records, cant access unless the administrator manually restores and update `syncPackageBlockList`. * `npm-security-holder` event is also determined to be a delete event * Update the original packet deletion process, update log output by the way
2023-02-10 21:31:23 +08:00
parent 18cfb0d35a
commit 27af0beaad
6 changed files with 220 additions and 53 deletions
--- a/app/common/constants.ts
+++ b/app/common/constants.ts
@@ -1,3 +1,13 @@
 export const BUG_VERSIONS = 'bug-versions';
 export const LATEST_TAG = 'latest';
 export const GLOBAL_WORKER = 'GLOBAL_WORKER';
+export enum SyncMode {
+  none = 'none',
+  exist = 'exist',
+  all = 'all',
+}
+export enum SyncDeleteMode {
+  ignore = 'ignore',
+  block = 'block',
+  delete = 'delete',
+}
--- a/app/core/service/PackageSyncerService.ts
+++ b/app/core/service/PackageSyncerService.ts
@@ -10,7 +10,8 @@ import {
 } from 'egg';
 import { setTimeout } from 'timers/promises';
 import { rm } from 'fs/promises';
-import { NPMRegistry } from '../../common/adapter/NPMRegistry';
+import semver from 'semver';
+import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
 import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
 import { downloadToTempfile } from '../../common/FileUtil';
 import { TaskState, TaskType } from '../../common/enum/Task';
@@ -31,6 +32,16 @@ import { Registry } from '../entity/Registry';
 import { BadRequestError } from 'egg-errors';
 import { ScopeManagerService } from './ScopeManagerService';
 import { EventCorkAdvice } from './EventCorkerAdvice';
+import { SyncDeleteMode } from '../../common/constants';
+
+type syncDeletePkgOptions = {
+  task: Task,
+  pkg: Package | null,
+  logUrl: string,
+  url: string,
+  logs: string[],
+  data: any,
+};

 function isoNow() {
  return new Date().toISOString();
@@ -209,6 +220,89 @@ export class PackageSyncerService extends AbstractService {
    await this.taskService.appendTaskLog(task, logs.join('\n'));
  }

+  private isRemovedInRemote(remoteFetchResult: RegistryResponse) {
+    const { status, data } = remoteFetchResult;
+
+    // deleted or blocked
+    if (status === 404 || status === 451) {
+      return true;
+    }
+
+    const hasMaintainers = data?.maintainers && data?.maintainers.length !== 0;
+    if (hasMaintainers) {
+      return false;
+    }
+
+    // unpublished
+    const timeMap = data.time || {};
+    if (timeMap.unpublished) {
+      return true;
+    }
+
+    // security holder
+    // test/fixtures/registry.npmjs.org/security-holding-package.json
+    let isSecurityHolder = true;
+    for (const versionInfo of Object.entries<{ _npmUser?: { name: string } }>(data.versions || {})) {
+      const [ v, info ] = versionInfo;
+      // >=0.0.1-security <0.0.2-0
+      const isSecurityVersion = semver.satisfies(v, '^0.0.1-security');
+      const isNpmUser = info?._npmUser?.name === 'npm';
+      if (!isSecurityVersion || !isNpmUser) {
+        isSecurityHolder = false;
+        break;
+      }
+    }
+
+    return isSecurityHolder;
+  }
+
+  // sync deleted package, deps on the syncDeleteMode
+  // - ignore: do nothing, just finish the task
+  // - delete: remove the package from local registry
+  // - block: block the package, update the manifest.block, instead of delete versions
+  // 根据 syncDeleteMode 配置，处理删包场景
+  // - ignore: 不做任何处理，直接结束任务
+  // - delete: 删除包数据，包括 manifest 存储
+  // - block: 软删除 将包标记为 block，用户无法直接使用
+  private async syncDeletePkg({ task, pkg, logUrl, url, logs, data }: syncDeletePkgOptions) {
+    const fullname = task.targetName;
+    const failEnd = `❌❌❌❌❌ ${url || fullname} ❌❌❌❌❌`;
+    const syncDeleteMode: SyncDeleteMode = this.config.cnpmcore.syncDeleteMode;
+    logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was removed in remote registry, response data: ${JSON.stringify(data)}, config.syncDeleteMode = ${syncDeleteMode}`);
+
+    // pkg not exists in local registry
+    if (!pkg) {
+      task.error = `Package not exists, response data: ${JSON.stringify(data)}`;
+      logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ${failEnd}`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      return;
+    }
+
+    if (syncDeleteMode === SyncDeleteMode.ignore) {
+      // ignore deleted package
+      logs.push(`[${isoNow()}] 🟢 Skip remove since config.syncDeleteMode = ignore`);
+    } else if (syncDeleteMode === SyncDeleteMode.block) {
+      // block deleted package
+      await this.packageManagerService.blockPackage(pkg, 'Removed in remote registry');
+      logs.push(`[${isoNow()}] 🟢 Block the package since config.syncDeleteMode = block`);
+    } else if (syncDeleteMode === SyncDeleteMode.delete) {
+      // delete package
+      await this.packageManagerService.unpublishPackage(pkg);
+      logs.push(`[${isoNow()}] 🟢 Delete the package since config.syncDeleteMode = delete`);
+    }
+
+    // update log
+    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
+    this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
+      task.taskId, task.targetName);
+
+  }
+
  // 初始化对应的 Registry
  // 1. 优先从 pkg.registryId 获取 (registryId 一经设置 不应改变)
  // 1. 其次从 task.data.registryId (创建单包同步任务时传入)
@@ -314,11 +408,11 @@ export class PackageSyncerService extends AbstractService {
      return;
    }

-    let result: any;
+    let registryFetchResult: RegistryResponse;
    try {
-      result = await this.npmRegistry.getFullManifests(fullname);
+      registryFetchResult = await this.npmRegistry.getFullManifests(fullname);
    } catch (err: any) {
-      const status = err.status || 'unknow';
+      const status = err.status || 'unknown';
      task.error = `request manifests error: ${err}, status: ${status}`;
      logs.push(`[${isoNow()}] ❌ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
@@ -328,7 +422,7 @@ export class PackageSyncerService extends AbstractService {
      return;
    }

-    const { url, data, headers, res, status } = result;
+    const { url, data, headers, res, status } = registryFetchResult;
    let readme = data.readme || '';
    if (typeof readme !== 'string') {
      readme = JSON.stringify(readme);
@@ -342,33 +436,15 @@ export class PackageSyncerService extends AbstractService {
    const contentLength = headers['content-length'] || '-';
    logs.push(`[${isoNow()}] HTTP [${status}] content-length: ${contentLength}, timing: ${JSON.stringify(res.timing)}`);

-    // 404 unpublished
-    // 451 blocked
-    const shouldRemovePkg = status === 404 || status === 451;
-    if (shouldRemovePkg) {
-      if (pkg) {
-        await this.packageManagerService.unpublishPackage(pkg);
-        logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was unpublished caused by ${status} response: ${JSON.stringify(data)}`);
-        logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-        logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
-        await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
-          task.taskId, task.targetName);
-      } else {
-        task.error = `Package not exists, response data: ${JSON.stringify(data)}`;
-        logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
-        logs.push(`[${isoNow()}] ${failEnd}`);
-        await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s',
-          task.taskId, task.targetName, task.error);
-      }
+    if (this.isRemovedInRemote(registryFetchResult)) {
+      await this.syncDeletePkg({ task, pkg, logs, logUrl, url, data });
      return;
    }

    const versionMap = data.versions || {};
    const distTags = data['dist-tags'] || {};

-    // show latest infomations
+    // show latest information
    if (distTags.latest) {
      logs.push(`[${isoNow()}] 📖 ${fullname} latest version: ${distTags.latest ?? '-'}, published time: ${JSON.stringify(timeMap[distTags.latest])}`);
    }
@@ -432,20 +508,6 @@ export class PackageSyncerService extends AbstractService {
      //     }
      //   }
      // }
-      if (timeMap.unpublished) {
-        if (pkg) {
-          await this.packageManagerService.unpublishPackage(pkg);
-          logs.push(`[${isoNow()}] 🟢 Sync unpublished package: ${JSON.stringify(timeMap.unpublished)} success`);
-        } else {
-          logs.push(`[${isoNow()}] 📖 Ignore unpublished package: ${JSON.stringify(timeMap.unpublished)}`);
-        }
-        logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-        logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
-        await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
-        this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s',
-          task.taskId, task.targetName);
-        return;
-      }

      // invalid maintainers, sync fail
      task.error = `invalid maintainers: ${JSON.stringify(maintainers)}`;
--- a/app/infra/NFSClientAdapter.ts
+++ b/app/infra/NFSClientAdapter.ts
@@ -1,10 +1,10 @@
 import {
  AccessLevel,
  EggObjectLifecycle,
-  InitTypeQualifier,
  Inject,
-  ObjectInitType,
  SingletonProto,
+  EggQualifier,
+  EggType,
 } from '@eggjs/tegg';
 import { EggAppConfig, EggLogger } from 'egg';
 import FSClient from 'fs-cnpm';
@@ -17,7 +17,7 @@ import { Readable } from 'stream';
 })
 export class NFSClientAdapter implements EggObjectLifecycle, NFSClient {
  @Inject()
-  @InitTypeQualifier(ObjectInitType.SINGLETON)
+  @EggQualifier(EggType.APP)
  private logger: EggLogger;

  @Inject()
--- a/config/config.default.ts
+++ b/config/config.default.ts
@@ -3,6 +3,7 @@ import { join } from 'path';
 import { EggAppConfig, PowerPartial } from 'egg';
 import OSSClient from 'oss-cnpm';
 import { patchAjv } from '../app/port/typebox';
+import { SyncDeleteMode, SyncMode } from '../app/common/constants';

 export default (appInfo: EggAppConfig) => {
  const config = {} as PowerPartial<EggAppConfig>;
@@ -22,7 +23,8 @@ export default (appInfo: EggAppConfig) => {
    //  - none: don't sync npm package, just redirect it to sourceRegistry
    //  - all: sync all npm packages
    //  - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled
-    syncMode: 'none',
+    syncMode: SyncMode.none,
+    syncDeleteMode: SyncDeleteMode.delete,
    hookEnable: false,
    syncPackageWorkerMaxConcurrentTasks: 10,
    triggerHookWorkerMaxConcurrentTasks: 10,
--- a/test/common/adapter/binary/ImageminBinary.test.ts
+++ b/test/common/adapter/binary/ImageminBinary.test.ts
@@ -80,7 +80,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => {
      data: await TestUtil.readFixturesFile('registry.npmjs.com/advpng-bin.json'),
    });
    let result = await binary.fetch('/', 'advpng-bin');
-    console.log(result?.items.map(_ => _.name));
+    // console.log(result?.items.map(_ => _.name));
    assert(result);
    assert(result.items.length > 0);
    let matchDir1 = false;
@@ -308,7 +308,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => {
      data: await TestUtil.readFixturesFile('registry.npmjs.com/guetzli.json'),
    });
    const result = await binary.fetch('/', 'guetzli-bin');
-    console.log(result);
+    // console.log(result);
    assert(result);
    // console.log(result.items);
    assert(result.items.length > 0);
--- a/test/core/service/PackageSyncerService/executeTask.test.ts
+++ b/test/core/service/PackageSyncerService/executeTask.test.ts
@@ -18,6 +18,7 @@ import { TaskService } from 'app/core/service/TaskService';
 import { ScopeManagerService } from 'app/core/service/ScopeManagerService';
 import { UserService } from 'app/core/service/UserService';
 import { ChangeRepository } from 'app/repository/ChangeRepository';
+import { PackageVersion } from 'app/repository/model/PackageVersion';

 describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
  let packageSyncerService: PackageSyncerService;
@@ -218,7 +219,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
      assert(stream);
      let log = await TestUtil.readStreamToLog(stream);
      // console.log(log);
-      assert(log.includes(`] 🟢 Package "${name}" was unpublished caused by 404 response`));
+      assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`));

      manifests = await packageManagerService.listPackageFullManifests('', name);
      assert(manifests.data.time.unpublished);
@@ -908,7 +909,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
      assert(stream);
      let log = await TestUtil.readStreamToLog(stream);
      // console.log(log);
-      assert(log.includes('] 📖 Ignore unpublished package: {'));
+      assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`));
      let data = await packageManagerService.listPackageFullManifests('', name);
      assert(data.data === null);

@@ -949,7 +950,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
      assert(stream);
      log = await TestUtil.readStreamToLog(stream);
      // console.log(log);
-      assert(log.includes('] 🟢 Sync unpublished package: {'));
+      assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`));
      data = await packageManagerService.listPackageFullManifests('', name);
      // console.log(data.data);
      assert(data.data.time.unpublished);
@@ -1565,7 +1566,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
      const log = await TestUtil.readStreamToLog(stream);
      // console.log(log);

-      assert(log.includes(`🟢 Package "${name}" was unpublished caused by 451 response`));
+      assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`));
    });

    it('should stop sync by block list', async () => {
@@ -1670,9 +1671,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
      assert(stream);
      const log = await TestUtil.readStreamToLog(stream);
      // console.log(log);
-      assert(log.includes('🟢🟢🟢🟢🟢'));
-      assert(log.includes('🟢 [1] Synced version 0.0.1-security success'));
-      assert(log.includes('Syncing maintainers: [{\"name\":\"npm\",\"email\":\"npm@npmjs.com\"}]'));
+      assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`));
    });

    it('should mock getFullManifests missing tarball error and downloadTarball error', async () => {
@@ -2126,5 +2125,99 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
        assert(log.includes('skip sync'));
      });
    });
+
+    describe('syncDeleteMode = ignore', async () => {
+
+      // already synced pkg
+      beforeEach(async () => {
+        app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', {
+          data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar.json'),
+          persist: false,
+          repeats: 1,
+        });
+        app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', {
+          data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
+          persist: false,
+        });
+        app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.1.0.tgz', 'GET', {
+          data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.1.0.tgz'),
+          persist: false,
+        });
+        await packageSyncerService.createTask('foobar', { skipDependencies: true });
+        const task = await packageSyncerService.findExecuteTask();
+        assert(task);
+        await packageSyncerService.executeTask(task);
+        assert(!await TaskModel.findOne({ taskId: task.taskId }));
+        assert(await HistoryTaskModel.findOne({ taskId: task.taskId }));
+
+      });
+
+      it('should ignore when upstream is removed', async () => {
+        // removed in remote
+        mock(app.config.cnpmcore, 'syncDeleteMode', 'ignore');
+        app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', {
+          data: await TestUtil.readFixturesFile('registry.npmjs.org/security-holding-package.json'),
+        });
+        await packageSyncerService.createTask('foobar', { skipDependencies: true });
+        const task = await packageSyncerService.findExecuteTask();
+        assert(task);
+        await packageSyncerService.executeTask(task);
+        assert(!await TaskModel.findOne({ taskId: task.taskId }));
+        assert(await HistoryTaskModel.findOne({ taskId: task.taskId }));
+        const stream = await packageSyncerService.findTaskLog(task);
+        assert(stream);
+        const log = await TestUtil.readStreamToLog(stream);
+        assert(log);
+        // console.log(log);
+        const model = await PackageModel.findOne({ scope: '', name: 'foobar' });
+        assert(model);
+        const versions = await PackageVersion.find({ packageId: model.packageId });
+        assert.equal(model!.isPrivate, false);
+        assert(versions.length === 2);
+
+      });
+
+      it('should block when upstream is removed', async () => {
+        // removed in remote
+        mock(app.config.cnpmcore, 'syncDeleteMode', 'block');
+        app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', {
+          data: await TestUtil.readFixturesFile('registry.npmjs.org/security-holding-package.json'),
+        });
+        await packageSyncerService.createTask('foobar', { skipDependencies: true });
+        const task = await packageSyncerService.findExecuteTask();
+        assert(task);
+        await packageSyncerService.executeTask(task);
+        assert(!await TaskModel.findOne({ taskId: task.taskId }));
+        assert(await HistoryTaskModel.findOne({ taskId: task.taskId }));
+        const stream = await packageSyncerService.findTaskLog(task);
+        assert(stream);
+        const log = await TestUtil.readStreamToLog(stream);
+        assert(log);
+        // console.log(log);
+        const model = await PackageModel.findOne({ scope: '', name: 'foobar' });
+        assert(model);
+        const versions = await PackageVersion.find({ packageId: model.packageId });
+        assert.equal(model!.isPrivate, false);
+        assert(versions.length === 2);
+
+        const manifests = await packageManagerService.listPackageFullManifests('', 'foobar');
+        assert(manifests.blockReason === 'Removed in remote registry');
+        assert(manifests.data.block === 'Removed in remote registry');
+        const pkg = await packageRepository.findPackage('', 'foobar');
+        assert(pkg);
+
+        await app.httpRequest()
+          .get(`/${pkg.name}`)
+          .expect(451);
+
+        // could resotre
+        await packageManagerService.unblockPackage(pkg);
+
+        await app.httpRequest()
+          .get(`/${pkg.name}`)
+          .expect(200);
+
+      });
+    });
  });
 });