Release 1.11.0

Refs:
- https://github.com/cnpm/cnpmcore/issues/282

History.md

@@ -1,4 +1,60 @@
 
+1.11.0 / 2022-08-23
+==================
+
+**features**
+  * [[`a91c8ac`](http://github.com/cnpm/cnpmcore/commit/a91c8ac4d05dc903780fda516b09364a05a2b1e6)] - feat: sync package from spec regsitry (#293) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`de37008`](http://github.com/cnpm/cnpmcore/commit/de37008261b05845f392d66764cdfe14ae324756)] - feat: changesStream adapter & needSync() method (#292) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`4b506c8`](http://github.com/cnpm/cnpmcore/commit/4b506c8371697ddacdbe99a8ecb330bfc1911ec6)] - feat: init registry & scope (#286) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`41c6e24`](http://github.com/cnpm/cnpmcore/commit/41c6e24c84d546eb9d5515cc0940cc3e4274687b)] - feat: impl trigger Hooks (#289) (killa <<killa123@126.com>>)
+  * [[`79cb826`](http://github.com/cnpm/cnpmcore/commit/79cb82615f04bdb3da3ccbe09bb6a861608b69c5)] - feat: impl migration sql (#290) (killa <<killa123@126.com>>)
+  * [[`4cfa8ed`](http://github.com/cnpm/cnpmcore/commit/4cfa8ed9d687ce7d950d7d20c0ea28221763ba5f)] - feat: impl hooks api (#287) (killa <<killa123@126.com>>)
+  * [[`47d53d2`](http://github.com/cnpm/cnpmcore/commit/47d53d22ad03c02ee9cb9035a38ae205a6d38381)] - feat: add bizId for task (#285) (killa <<killa123@126.com>>)
+  * [[`3b1536b`](http://github.com/cnpm/cnpmcore/commit/3b1536b070b2f9062bc2cc377db96d2f4a160efc)] - feat: add node-webrtc mirror (#274) (Opportunity <<opportunity@live.in>>)
+
+**others**
+  * [[`7106807`](http://github.com/cnpm/cnpmcore/commit/710680742a078b2faf4cb18c3a39c0397308712e)] - 🐛 FIX: Should show queue size on logging (#280) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`3a41b21`](http://github.com/cnpm/cnpmcore/commit/3a41b2161cc99bb2f6f6dd7cbaa7abef25ff4393)] - 🐛 FIX: Handle binary configuration value (#278) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.10.0 / 2022-08-04
+==================
+
+**features**
+  * [[`c2b7d5a`](http://github.com/cnpm/cnpmcore/commit/c2b7d5aa98b5ba8649ec246c616574a22e9a74b8)] - feat: use sort set to impl queue (#277) (killa <<killa123@126.com>>)
+
+1.9.1 / 2022-07-29
+==================
+
+**fixes**
+  * [[`c54aa21`](http://github.com/cnpm/cnpmcore/commit/c54aa2165c3938dcbb5a2b3b54e66a0d961cc813)] - fix: check executingCount after task is done (#276) (killa <<killa123@126.com>>)
+
+**others**
+  * [[`3268d03`](http://github.com/cnpm/cnpmcore/commit/3268d030b620825c8c2e6331e1745c1788066c61)] - 🤖 TEST: show package not use cache if isSync (#273) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.9.0 / 2022-07-25
+==================
+
+**features**
+  * [[`af6a75a`](http://github.com/cnpm/cnpmcore/commit/af6a75af32ea04c90fda82be3a56c99ec77e5807)] - feat: add forceSyncHistory options (#271) (killa <<killa123@126.com>>)
+
+1.8.0 / 2022-07-21
+==================
+
+**features**
+  * [[`b49a38c`](http://github.com/cnpm/cnpmcore/commit/b49a38c77e044c978e6de32a9d3e257cc90ea7c1)] - feat: use Model with inject (#269) (killa <<killa123@126.com>>)
+
+1.7.1 / 2022-07-20
+==================
+
+**fixes**
+  * [[`52fca55`](http://github.com/cnpm/cnpmcore/commit/52fca55aa883865f0ae70bfc1ff274c313b8f76a)] - fix: show package not use cache if isSync (#268) (killa <<killa123@126.com>>)
+
+1.7.0 / 2022-07-12
+==================
+
+**others**
+  * [[`4f7ce8b`](http://github.com/cnpm/cnpmcore/commit/4f7ce8b4b2a5806a225ce67228388e14388b7059)] - deps: upgrade leoric to 2.x (#262) (killa <<killa123@126.com>>)
+
 1.6.0 / 2022-07-11
 ==================
 

app/common/LogUtil.ts

@@ -0,0 +1,3 @@
+export function isoNow() {
+  return new Date().toISOString();
+}

app/common/SyncUtil.ts

@@ -6,7 +6,7 @@ export function isSyncWorkerRequest(ctx: EggContext) {
   if (!isSyncWorkerRequest) {
     const ua = ctx.headers['user-agent'] || '';
     // old sync client will request with these user-agent
-    if (ua.indexOf('npm_service.cnpmjs.org/') !== -1) {
+    if (ua.includes('npm_service.cnpmjs.org/')) {
       isSyncWorkerRequest = true;
     }
   }

app/common/adapter/NPMRegistry.ts

@@ -25,9 +25,14 @@ export class NPMRegistry {
   @Inject()
   private config: EggAppConfig;
   private timeout = 10000;
+  public registryHost: string;
 
   get registry(): string {
-    return this.config.cnpmcore.sourceRegistry;
+    return this.registryHost || this.config.cnpmcore.sourceRegistry;
+  }
+
+  public setRegistryHost(registryHost = '') {
+    this.registryHost = registryHost;
   }
 
   public async getFullManifests(fullname: string, retries = 3) {

app/common/adapter/QueueAdapter.ts

@@ -1,32 +0,0 @@
-import {
-  AccessLevel,
-  Inject,
-  ContextProto,
-} from '@eggjs/tegg';
-import { Redis } from 'ioredis';
-
-@ContextProto({
-  accessLevel: AccessLevel.PUBLIC,
-})
-export class QueueAdapter {
-  @Inject()
-  private readonly redis: Redis;
-
-  private getQueueName(key: string) {
-    return `CNPMCORE_Q_${key}`;
-  }
-
-  async push<T>(key: string, item: T) {
-    return await this.redis.lpush(this.getQueueName(key), JSON.stringify(item));
-  }
-
-  async pop<T>(key: string) {
-    const json = await this.redis.rpop(this.getQueueName(key));
-    if (!json) return null;
-    return JSON.parse(json) as T;
-  }
-
-  async length(key: string) {
-    return await this.redis.llen(this.getQueueName(key));
-  }
-}

app/common/adapter/binary/NodePreGypBinary.ts

@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { AbstractBinary, FetchResult, BinaryItem } from './AbstractBinary';
 
 export class NodePreGypBinary extends AbstractBinary {
@@ -32,7 +33,7 @@ export class NodePreGypBinary extends AbstractBinary {
 
         let currentDir = this.dirItems['/'];
         let versionPrefix = '';
-        const remotePath = pkgVersion.binary.remote_path;
+        let remotePath = pkgVersion.binary.remote_path;
         const napiVersions = pkgVersion.binary.napi_versions ?? [];
         if (this.binaryConfig.options?.requiredNapiVersions && napiVersions.length === 0) continue;
         if (remotePath?.includes('{version}')) {
@@ -148,17 +149,31 @@ export class NodePreGypBinary extends AbstractBinary {
           //   "package_name": "{platform}-{arch}.tar.gz",
           //   "module_path": "bin"
           // },
+          // handle {configuration}
+          // "binary": {
+          //   "module_name": "wrtc",
+          //   "module_path": "./build/{configuration}/",
+          //   "remote_path": "./{module_name}/v{version}/{configuration}/",
+          //   "package_name": "{platform}-{arch}.tar.gz",
+          //   "host": "https://node-webrtc.s3.amazonaws.com"
+          // },
           for (const platform of nodePlatforms) {
             const archs = nodeArchs[platform];
             for (const arch of archs) {
-              const name = binaryFile.replace('{platform}', platform)
+              const binaryFileName = binaryFile.replace('{platform}', platform)
                 .replace('{arch}', arch);
+              remotePath = remotePath.replace('{module_name}', moduleName)
+                .replace('{name}', this.binaryConfig.category)
+                .replace('{version}', version)
+                .replace('{configuration}', 'Release');
+              const binaryFilePath = join('/', remotePath, binaryFileName);
+              const remoteUrl = `${this.binaryConfig.distUrl}${binaryFilePath}`;
               currentDir.push({
-                name,
+                name: binaryFileName,
                 date,
                 size: '-',
                 isDir: false,
-                url: `${this.binaryConfig.distUrl}/${this.binaryConfig.category}${versionPrefix}/${name}`,
+                url: remoteUrl,
                 ignoreDownloadStatuses: [ 404 ],
               });
             }

app/common/adapter/changesStream/AbstractChangesStream.ts

@@ -0,0 +1,41 @@
+import {
+  ImplDecorator,
+  Inject,
+  QualifierImplDecoratorUtil,
+} from '@eggjs/tegg';
+import { Readable } from 'node:stream';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import {
+  EggHttpClient,
+  EggLogger,
+} from 'egg';
+
+export const CHANGE_STREAM_ATTRIBUTE = 'CHANGE_STREAM_ATTRIBUTE';
+export type ChangesStreamChange = {
+  seq: string;
+  fullname: string;
+};
+
+export abstract class AbstractChangeStream {
+  @Inject()
+  protected logger: EggLogger;
+
+  @Inject()
+  protected httpclient: EggHttpClient;
+
+  abstract getInitialSince(registry: Registry): Promise<string>;
+  abstract fetchChanges(registry: Registry, since: string): Promise<Readable>;
+
+  getChangesStreamUrl(registry: Registry, since: string, limit?: number): string {
+    const url = new URL(registry.changeStream);
+    url.searchParams.set('since', since);
+    if (limit) {
+      url.searchParams.set('limit', String(limit));
+    }
+    return url.toString();
+  }
+}
+
+export const RegistryChangesStream: ImplDecorator<AbstractChangeStream, typeof RegistryType> =
+  QualifierImplDecoratorUtil.generatorDecorator(AbstractChangeStream, CHANGE_STREAM_ATTRIBUTE);

app/common/adapter/changesStream/CnpmcoreChangesStream.ts

@@ -0,0 +1,56 @@
+import { Readable } from 'node:stream';
+import { ContextProto } from '@eggjs/tegg';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { E500 } from 'egg-errors';
+import { AbstractChangeStream, ChangesStreamChange, RegistryChangesStream } from './AbstractChangesStream';
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Cnpmcore)
+export class CnpmcoreChangesStream extends AbstractChangeStream {
+
+  async getInitialSince(registry: Registry): Promise<string> {
+    const db = (new URL(registry.changeStream)).origin;
+    const { status, data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 10000,
+      dataType: 'json',
+    });
+    if (!data.update_seq) {
+      throw new E500(`get getInitialSince failed: ${data.update_seq}`);
+    }
+    const since = String(data.update_seq - 10);
+    this.logger.warn('[NpmChangesStream.getInitialSince:firstSeq] GET %s status: %s, data: %j, since: %s',
+      registry.name, status, data, since);
+    return since;
+  }
+
+  async fetchChanges(registry: Registry, since: string): Promise<Readable> {
+    const changes: ChangesStreamChange[] = [];
+
+    const db = this.getChangesStreamUrl(registry, since);
+    // json mode
+    const { data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 30000,
+      dataType: 'json',
+      gzip: true,
+    });
+
+    if (data.results?.length > 0) {
+      for (const change of data.results) {
+        const seq = String(change.seq);
+        const fullname = change.id;
+        // cnpmcore 默认返回 >= 需要做特殊判断
+        if (seq && fullname && seq !== since) {
+          changes.push({
+            fullname,
+            seq,
+          });
+        }
+      }
+    }
+
+    return Readable.from(changes);
+  }
+}

app/common/adapter/changesStream/CnpmjsorgChangesStream.ts

@@ -0,0 +1,69 @@
+import { ContextProto } from '@eggjs/tegg';
+import { Readable } from 'node:stream';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { E500 } from 'egg-errors';
+import { AbstractChangeStream, ChangesStreamChange, RegistryChangesStream } from './AbstractChangesStream';
+
+const MAX_LIMIT = 10000;
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Cnpmjsorg)
+export class CnpmjsorgChangesStream extends AbstractChangeStream {
+
+  // cnpmjsorg 未实现 update_seq 字段
+  // 默认返回当前时间戳字符串
+  async getInitialSince(registry: Registry): Promise<string> {
+    const since = String((new Date()).getTime());
+    this.logger.warn(`[CnpmjsorgChangesStream.getInitialSince] since: ${since}, skip query ${registry.changeStream}`);
+    return since;
+  }
+
+  private async tryFetch(registry: Registry, since: string, limit = 1000) {
+    if (limit > MAX_LIMIT) {
+      throw new E500(`limit too large, current since: ${since}, limit: ${limit}`);
+    }
+    const db = this.getChangesStreamUrl(registry, since, limit);
+    // json mode
+    const res = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 30000,
+      dataType: 'json',
+      gzip: true,
+    });
+    const { results = [] } = res.data;
+    if (results?.length > 1) {
+      const [ first ] = results;
+      const last = results[results.length - 1];
+      if (first.gmt_modified === last.gmt_modified) {
+        return await this.tryFetch(registry, last.seq, limit + 1000);
+      }
+    }
+
+    return res;
+  }
+
+  async fetchChanges(registry: Registry, since: string): Promise<Readable> {
+    const changes: ChangesStreamChange[] = [];
+
+    // ref: https://github.com/cnpm/cnpmjs.org/pull/1734
+    // 由于 cnpmjsorg 无法计算准确的 seq
+    // since 是一个时间戳，需要确保一次返回的结果中首尾两个 gmtModified 不相等
+    const { data } = await this.tryFetch(registry, since);
+
+    if (data.results?.length > 0) {
+      for (const change of data.results) {
+        const seq = new Date(change.gmt_modified).getTime() + '';
+        const fullname = change.id;
+        if (seq && fullname && seq !== since) {
+          changes.push({
+            fullname,
+            seq,
+          });
+        }
+      }
+    }
+
+    return Readable.from(changes);
+  }
+}

app/common/adapter/changesStream/NpmChangesStream.ts

@@ -0,0 +1,42 @@
+import { ContextProto } from '@eggjs/tegg';
+import { Readable, pipeline } from 'node:stream';
+import { E500 } from 'egg-errors';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { AbstractChangeStream, RegistryChangesStream } from './AbstractChangesStream';
+import ChangesStreamTransform from '../../../core/util/ChangesStreamTransform';
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Npm)
+export class NpmChangesStream extends AbstractChangeStream {
+
+  async getInitialSince(registry: Registry): Promise<string> {
+    const db = (new URL(registry.changeStream)).origin;
+    const { status, data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 10000,
+      dataType: 'json',
+    });
+    const since = String(data.update_seq - 10);
+    if (!data.update_seq) {
+      throw new E500(`get getInitialSince failed: ${data.update_seq}`);
+    }
+    this.logger.warn('[NpmChangesStream.getInitialSince] GET %s status: %s, data: %j, since: %s',
+      registry.name, registry.changeStream, status, data, since);
+    return since;
+  }
+
+  async fetchChanges(registry: Registry, since: string): Promise<Readable> {
+    const db = this.getChangesStreamUrl(registry, since);
+    const { res } = await this.httpclient.request(db, {
+      streaming: true,
+      timeout: 10000,
+    });
+
+    const transform = new ChangesStreamTransform();
+    return pipeline(res, transform, error => {
+      this.logger.error('[NpmChangesStream.fetchChanges] pipeline error: %s', error);
+    });
+  }
+
+}

app/common/enum/Hook.ts

@@ -0,0 +1,19 @@
+export enum HookType {
+  Package = 'package',
+  Scope = 'scope',
+  Owner = 'owner',
+}
+
+export enum HookEventType {
+  Star = 'package:star',
+  Unstar = 'package:unstar',
+  Publish = 'package:publish',
+  Unpublish = 'package:unpublish',
+  Owner = 'package:owner',
+  OwnerRm = 'package:owner-rm',
+  DistTag = 'package:dist-tag',
+  DistTagRm = 'package:dist-tag-rm',
+  Deprecated = 'package:deprecated',
+  Undeprecated = 'package:undeprecated',
+  Change = 'package:change',
+}

app/common/enum/Registry.ts

@@ -0,0 +1,5 @@
+export enum RegistryType {
+  Npm = 'npm',
+  Cnpmcore = 'cnpmcore',
+  Cnpmjsorg = 'cnpmjsorg',
+}

app/common/enum/Task.ts

@@ -2,6 +2,8 @@ export enum TaskType {
   SyncPackage = 'sync_package',
   ChangesStream = 'changes_stream',
   SyncBinary = 'sync_binary',
+  CreateHook = 'create_hook',
+  TriggerHook = 'trigger_hook',
 }
 
 export enum TaskState {

app/common/typing.ts

@@ -35,3 +35,9 @@ export interface NFSClient {
 
   url?(key: string): string;
 }
+
+export interface QueueAdapter {
+  push<T>(key: string, item: T): Promise<boolean>;
+  pop<T>(key: string): Promise<T | null>;
+  length(key: string): Promise<number>;
+}

app/core/entity/Hook.ts

@@ -0,0 +1,61 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+import { HookType } from '../../common/enum/Hook';
+import crypto from 'crypto';
+
+export type CreateHookData = Omit<EasyData<HookData, 'hookId'>, 'enable' | 'latestTaskId'>;
+
+export interface HookData extends EntityData {
+  hookId: string;
+  type: HookType;
+  ownerId: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  latestTaskId?: string;
+  enable: boolean;
+}
+
+export class Hook extends Entity {
+  readonly hookId: string;
+  readonly type: HookType;
+  readonly ownerId: string;
+  readonly name: string;
+  endpoint: string;
+  secret: string;
+  enable: boolean;
+  latestTaskId?: string;
+
+  constructor(data: HookData) {
+    super(data);
+    this.hookId = data.hookId;
+    this.type = data.type;
+    this.ownerId = data.ownerId;
+    this.name = data.name;
+    this.endpoint = data.endpoint;
+    this.secret = data.secret;
+    this.latestTaskId = data.latestTaskId;
+    this.enable = data.enable;
+  }
+
+  static create(data: CreateHookData): Hook {
+    const hookData: EasyData<HookData, 'hookId'> = Object.assign({}, data, {
+      enable: true,
+      latestTaskId: undefined,
+    });
+    const newData = EntityUtil.defaultData(hookData, 'hookId');
+    return new Hook(newData);
+  }
+
+  // payload 可能会特别大，如果做多次 stringify 浪费太多 cpu
+  signPayload(payload: object): { digest, payloadStr } {
+    const payloadStr = JSON.stringify(payload);
+    const digest = crypto.createHmac('sha256', this.secret)
+      .update(JSON.stringify(payload))
+      .digest('hex');
+    return {
+      digest,
+      payloadStr,
+    };
+  }
+}

app/core/entity/HookEvent.ts

@@ -0,0 +1,93 @@
+import { HookEventType } from '../../common/enum/Hook';
+
+export interface PublishChangePayload {
+  'dist-tag'?: string;
+  version: string;
+}
+
+export interface UnpublishChangePayload {
+  'dist-tag'?: string;
+  version?: string;
+}
+
+export interface DistTagChangePayload {
+  'dist-tag': string;
+}
+
+export interface PackageOwnerPayload {
+  maintainer: string;
+}
+
+export interface DeprecatedChangePayload {
+  deprecated: string;
+}
+
+export class HookEvent<T = object> {
+  readonly changeId: string;
+  readonly event: HookEventType;
+  readonly fullname: string;
+  readonly type: 'package';
+  readonly version: '1.0.0';
+  readonly change: T;
+  readonly time: number;
+
+  constructor(event: HookEventType, changeId: string, fullname: string, change: T) {
+    this.changeId = changeId;
+    this.event = event;
+    this.fullname = fullname;
+    this.type = 'package';
+    this.version = '1.0.0';
+    this.change = change;
+    this.time = Date.now();
+  }
+
+  static createPublishEvent(fullname: string, changeId: string, version: string, distTag?: string): HookEvent<PublishChangePayload> {
+    return new HookEvent(HookEventType.Publish, changeId, fullname, {
+      'dist-tag': distTag,
+      version,
+    });
+  }
+
+  static createUnpublishEvent(fullname: string, changeId: string, version?: string, distTag?: string): HookEvent<UnpublishChangePayload> {
+    return new HookEvent(HookEventType.Unpublish, changeId, fullname, {
+      'dist-tag': distTag,
+      version,
+    });
+  }
+
+  static createOwnerEvent(fullname: string, changeId: string, maintainer: string): HookEvent<PackageOwnerPayload> {
+    return new HookEvent(HookEventType.Owner, changeId, fullname, {
+      maintainer,
+    });
+  }
+
+  static createOwnerRmEvent(fullname: string, changeId: string, maintainer: string): HookEvent<PackageOwnerPayload> {
+    return new HookEvent(HookEventType.OwnerRm, changeId, fullname, {
+      maintainer,
+    });
+  }
+
+  static createDistTagEvent(fullname: string, changeId: string, distTag: string): HookEvent<DistTagChangePayload> {
+    return new HookEvent(HookEventType.DistTag, changeId, fullname, {
+      'dist-tag': distTag,
+    });
+  }
+
+  static createDistTagRmEvent(fullname: string, changeId: string, distTag: string): HookEvent<DistTagChangePayload> {
+    return new HookEvent(HookEventType.DistTagRm, changeId, fullname, {
+      'dist-tag': distTag,
+    });
+  }
+
+  static createDeprecatedEvent(fullname: string, changeId: string, deprecated: string): HookEvent<DeprecatedChangePayload> {
+    return new HookEvent(HookEventType.Deprecated, changeId, fullname, {
+      deprecated,
+    });
+  }
+
+  static createUndeprecatedEvent(fullname: string, changeId: string, deprecated: string): HookEvent<DeprecatedChangePayload> {
+    return new HookEvent(HookEventType.Undeprecated, changeId, fullname, {
+      deprecated,
+    });
+  }
+}

app/core/entity/Package.ts

@@ -11,6 +11,7 @@ interface PackageData extends EntityData {
   description: string;
   abbreviatedsDist?: Dist;
   manifestsDist?: Dist;
+  registryId?: string;
 }
 
 export enum DIST_NAMES {
@@ -36,6 +37,7 @@ export class Package extends Entity {
   description: string;
   abbreviatedsDist?: Dist;
   manifestsDist?: Dist;
+  registryId?: string;
 
   constructor(data: PackageData) {
     super(data);
@@ -46,6 +48,7 @@ export class Package extends Entity {
     this.description = data.description;
     this.abbreviatedsDist = data.abbreviatedsDist;
     this.manifestsDist = data.manifestsDist;
+    this.registryId = data.registryId;
   }
 
   static create(data: EasyData<PackageData, 'packageId'>): Package {

app/core/entity/Registry.ts

@@ -0,0 +1,38 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+import type { RegistryType } from '../../common/enum/Registry';
+
+interface RegistryData extends EntityData {
+  name: string;
+  registryId: string;
+  host: string;
+  changeStream: string;
+  userPrefix: string;
+  type: RegistryType;
+}
+
+export type CreateRegistryData = Omit<EasyData<RegistryData, 'registryId'>, 'id'>;
+
+export class Registry extends Entity {
+  name: string;
+  registryId: string;
+  host: string;
+  changeStream: string;
+  userPrefix: string;
+  type: RegistryType;
+
+  constructor(data: RegistryData) {
+    super(data);
+    this.name = data.name;
+    this.registryId = data.registryId;
+    this.host = data.host;
+    this.changeStream = data.changeStream;
+    this.userPrefix = data.userPrefix;
+    this.type = data.type;
+  }
+
+  public static create(data: CreateRegistryData): Registry {
+    const newData = EntityUtil.defaultData(data, 'registryId');
+    return new Registry(newData);
+  }
+}

app/core/entity/Scope.ts

@@ -0,0 +1,28 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+
+interface ScopeData extends EntityData {
+  name: string;
+  scopeId: string;
+  registryId: string;
+}
+
+export type CreateScopeData = Omit<EasyData<ScopeData, 'scopeId'>, 'id'>;
+
+export class Scope extends Entity {
+  name: string;
+  registryId: string;
+  scopeId: string;
+
+  constructor(data: ScopeData) {
+    super(data);
+    this.name = data.name;
+    this.registryId = data.registryId;
+    this.scopeId = data.scopeId;
+  }
+
+  static create(data: CreateScopeData): Scope {
+    const newData = EntityUtil.defaultData(data, 'scopeId');
+    return new Scope(newData);
+  }
+}

app/core/entity/Task.ts

@@ -4,19 +4,28 @@ import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
 import { TaskType, TaskState } from '../../common/enum/Task';
 import dayjs from '../../common/dayjs';
+import { HookEvent } from './HookEvent';
 
-interface TaskData extends EntityData {
+export const HOST_NAME = os.hostname();
+export const PID = process.pid;
+
+export interface TaskBaseData {
+  taskWorker: string;
+}
+
+export interface TaskData<T = TaskBaseData> extends EntityData {
   taskId: string;
   type: TaskType;
   state: TaskState;
   targetName: string;
   authorId: string;
   authorIp: string;
-  data: any;
+  data: T;
   logPath?: string;
   logStorePosition?: string;
   attempts?: number;
   error?: string;
+  bizId?: string;
 }
 
 export type SyncPackageTaskOptions = {
@@ -25,22 +34,57 @@ export type SyncPackageTaskOptions = {
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
+  // force sync history version
+  forceSyncHistory?: boolean;
+  registryId?: string;
 };
 
-export class Task extends Entity {
+export interface CreateHookTaskData extends TaskBaseData {
+  hookEvent: HookEvent;
+}
+
+export interface TriggerHookTaskData extends TaskBaseData {
+  hookEvent: HookEvent;
+  hookId: string;
+  responseStatus?: number;
+}
+
+export interface CreateSyncPackageTaskData extends TaskBaseData {
+  tips?: string;
+  skipDependencies?: boolean;
+  syncDownloadData?: boolean;
+  forceSyncHistory?: boolean;
+}
+
+export interface ChangesStreamTaskData extends TaskBaseData {
+  since: string;
+  last_package?: string,
+  last_package_created?: Date,
+  task_count?: number,
+  registryId?: string,
+}
+
+export type CreateHookTask = Task<CreateHookTaskData>;
+export type TriggerHookTask = Task<TriggerHookTaskData>;
+export type CreateSyncPackageTask = Task<CreateSyncPackageTaskData>;
+export type ChangesStreamTask = Task<ChangesStreamTaskData>;
+
+export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
   taskId: string;
   type: TaskType;
   state: TaskState;
   targetName: string;
+  taskWorker: string;
   authorId: string;
   authorIp: string;
-  data: any;
+  data: T;
   logPath: string;
   logStorePosition: string;
   attempts: number;
   error: string;
+  bizId?: string;
 
-  constructor(data: TaskData) {
+  constructor(data: TaskData<T>) {
     super(data);
     this.taskId = data.taskId;
     this.type = data.type;
@@ -53,6 +97,7 @@ export class Task extends Entity {
     this.logStorePosition = data.logStorePosition ?? '';
     this.attempts = data.attempts ?? 0;
     this.error = data.error ?? '';
+    this.bizId = data.bizId;
   }
 
   public resetLogPath() {
@@ -61,15 +106,15 @@ export class Task extends Entity {
   }
 
   public setExecuteWorker() {
-    this.data.taskWorker = `${os.hostname()}:${process.pid}`;
+    this.data.taskWorker = `${HOST_NAME}:${PID}`;
   }
 
-  private static create(data: EasyData<TaskData, 'taskId'>): Task {
+  private static create<T extends TaskBaseData>(data: EasyData<TaskData<T>, 'taskId'>): Task<T> {
     const newData = EntityUtil.defaultData(data, 'taskId');
     return new Task(newData);
   }
 
-  public static createSyncPackage(fullname: string, options?: SyncPackageTaskOptions): Task {
+  public static createSyncPackage(fullname: string, options?: SyncPackageTaskOptions): CreateSyncPackageTask {
     const data = {
       type: TaskType.SyncPackage,
       state: TaskState.Waiting,
@@ -80,8 +125,10 @@ export class Task extends Entity {
         // task execute worker
         taskWorker: '',
         tips: options?.tips,
+        registryId: options?.registryId ?? '',
         skipDependencies: options?.skipDependencies,
         syncDownloadData: options?.syncDownloadData,
+        forceSyncHistory: options?.forceSyncHistory,
       },
     };
     const task = this.create(data);
@@ -89,20 +136,71 @@ export class Task extends Entity {
     return task;
   }
 
-  public static createChangesStream(targetName: string): Task {
+  public static createChangesStream(targetName: string, since = ''): ChangesStreamTask {
     const data = {
       type: TaskType.ChangesStream,
       state: TaskState.Waiting,
       targetName,
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
+      data: {
+        // task execute worker
+        taskWorker: '',
+        since,
+      },
+    };
+    return this.create(data) as ChangesStreamTask;
+  }
+
+  public updateSyncData({ lastSince, taskCount, lastPackage }: SyncInfo) {
+    const syncData = this.data as unknown as ChangesStreamTaskData;
+    // 更新任务记录信息
+    syncData.since = lastSince;
+    syncData.task_count = (syncData.task_count || 0) + taskCount;
+
+    if (taskCount > 0) {
+      syncData.last_package = lastPackage;
+      syncData.last_package_created = new Date();
+    }
+  }
+
+  public static createCreateHookTask(hookEvent: HookEvent): CreateHookTask {
+    const data = {
+      type: TaskType.CreateHook,
+      state: TaskState.Waiting,
+      targetName: hookEvent.fullname,
       authorId: `pid_${process.pid}`,
       authorIp: os.hostname(),
+      bizId: `CreateHook:${hookEvent.changeId}`,
+      data: {
+        // task execute worker
+        taskWorker: '',
+        hookEvent,
+      },
+    };
+    const task = this.create(data);
+    task.logPath = `/packages/${hookEvent.fullname}/hooks/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
+  }
+
+  public static createTriggerHookTask(hookEvent: HookEvent, hookId: string): TriggerHookTask {
+    const data = {
+      type: TaskType.TriggerHook,
+      state: TaskState.Waiting,
+      targetName: hookEvent.fullname,
+      authorId: `pid_${process.pid}`,
+      bizId: `TriggerHook:${hookEvent.changeId}:${hookId}`,
+      authorIp: os.hostname(),
       data: {
         // task execute worker
         taskWorker: '',
-        since: '',
+        hookEvent,
+        hookId,
       },
     };
-    return this.create(data);
+    const task = this.create(data);
+    task.logPath = `/packages/${hookEvent.fullname}/hooks/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
   }
 
   public static createSyncBinary(targetName: string, lastData: any): Task {
@@ -110,8 +208,8 @@ export class Task extends Entity {
       type: TaskType.SyncBinary,
       state: TaskState.Waiting,
       targetName,
-      authorId: `pid_${process.pid}`,
-      authorIp: os.hostname(),
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
       data: {
         // task execute worker
         taskWorker: '',
@@ -123,3 +221,9 @@ export class Task extends Entity {
     return task;
   }
 }
+
+export type SyncInfo = {
+  lastSince: string;
+  taskCount: number;
+  lastPackage?: string;
+};

app/core/event/ChangesStream.ts

@@ -8,83 +8,116 @@ import {
   PACKAGE_TAG_REMOVED,
   PACKAGE_MAINTAINER_CHANGED,
   PACKAGE_MAINTAINER_REMOVED,
-  PACKAGE_META_CHANGED,
+  PACKAGE_META_CHANGED, PackageMetaChange,
 } from './index';
 import { ChangeRepository } from '../../repository/ChangeRepository';
 import { Change } from '../entity/Change';
+import { HookEvent } from '../entity/HookEvent';
+import { Task } from '../entity/Task';
+import { User } from '../entity/User';
+import { TaskService } from '../service/TaskService';
 
 class ChangesStreamEvent {
   @Inject()
   private readonly changeRepository: ChangeRepository;
 
-  protected async addChange(type: string, fullname: string, data: object) {
-    await this.changeRepository.addChange(Change.create({
+  @Inject()
+  protected readonly taskService: TaskService;
+
+  protected async addChange(type: string, fullname: string, data: object): Promise<Change> {
+    const change = Change.create({
       type,
       targetName: fullname,
       data,
-    }));
+    });
+    await this.changeRepository.addChange(change);
+    return change;
   }
 }
 
 @Event(PACKAGE_UNPUBLISHED)
 export class PackageUnpublished extends ChangesStreamEvent {
   async handle(fullname: string) {
-    await this.addChange(PACKAGE_UNPUBLISHED, fullname, {});
+    const change = await this.addChange(PACKAGE_UNPUBLISHED, fullname, {});
+    const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(fullname, change.changeId));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
 export class PackageVersionAdded extends ChangesStreamEvent {
-  async handle(fullname: string, version: string) {
-    await this.addChange(PACKAGE_VERSION_ADDED, fullname, { version });
+  async handle(fullname: string, version: string, tag?: string) {
+    const change = await this.addChange(PACKAGE_VERSION_ADDED, fullname, { version });
+    const task = Task.createCreateHookTask(HookEvent.createPublishEvent(fullname, change.changeId, version, tag));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_VERSION_REMOVED)
 export class PackageVersionRemoved extends ChangesStreamEvent {
-  async handle(fullname: string, version: string) {
-    await this.addChange(PACKAGE_VERSION_REMOVED, fullname, { version });
+  async handle(fullname: string, version: string, tag?: string) {
+    const change = await this.addChange(PACKAGE_VERSION_REMOVED, fullname, { version });
+    const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(fullname, change.changeId, version, tag));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_TAG_ADDED)
 export class PackageTagAdded extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_ADDED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_ADDED, fullname, { tag });
+    const task = Task.createCreateHookTask(HookEvent.createDistTagEvent(fullname, change.changeId, tag));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_TAG_CHANGED)
 export class PackageTagChanged extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_CHANGED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_CHANGED, fullname, { tag });
+    const task = Task.createCreateHookTask(HookEvent.createDistTagEvent(fullname, change.changeId, tag));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_TAG_REMOVED)
 export class PackageTagRemoved extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_REMOVED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_REMOVED, fullname, { tag });
+    const task = Task.createCreateHookTask(HookEvent.createDistTagRmEvent(fullname, change.changeId, tag));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_MAINTAINER_CHANGED)
 export class PackageMaintainerChanged extends ChangesStreamEvent {
-  async handle(fullname: string) {
-    await this.addChange(PACKAGE_MAINTAINER_CHANGED, fullname, {});
+  async handle(fullname: string, maintainers: User[]) {
+    const change = await this.addChange(PACKAGE_MAINTAINER_CHANGED, fullname, {});
+    // TODO 应该比较差值，而不是全量推送
+    for (const maintainer of maintainers) {
+      const task = Task.createCreateHookTask(HookEvent.createOwnerEvent(fullname, change.changeId, maintainer.name));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_MAINTAINER_REMOVED)
 export class PackageMaintainerRemoved extends ChangesStreamEvent {
   async handle(fullname: string, maintainer: string) {
-    await this.addChange(PACKAGE_MAINTAINER_REMOVED, fullname, { maintainer });
+    const change = await this.addChange(PACKAGE_MAINTAINER_REMOVED, fullname, { maintainer });
+    const task = Task.createCreateHookTask(HookEvent.createOwnerRmEvent(fullname, change.changeId, maintainer));
+    await this.taskService.createTask(task, true);
   }
 }
 
 @Event(PACKAGE_META_CHANGED)
 export class PackageMetaChanged extends ChangesStreamEvent {
-  async handle(fullname: string, meta: object) {
-    await this.addChange(PACKAGE_META_CHANGED, fullname, { ...meta });
+  async handle(fullname: string, meta: PackageMetaChange) {
+    const change = await this.addChange(PACKAGE_META_CHANGED, fullname, { ...meta });
+    const { deprecateds } = meta;
+    for (const deprecated of deprecateds || []) {
+      const task = Task.createCreateHookTask(HookEvent.createDeprecatedEvent(fullname, change.changeId, deprecated.version));
+      await this.taskService.createTask(task, true);
+    }
   }
 }

app/core/event/index.ts

@@ -1,4 +1,5 @@
 import '@eggjs/tegg';
+import { User } from '../entity/User';
 
 export const PACKAGE_UNPUBLISHED = 'PACKAGE_UNPUBLISHED';
 export const PACKAGE_BLOCKED = 'PACKAGE_BLOCKED';
@@ -12,18 +13,28 @@ export const PACKAGE_MAINTAINER_CHANGED = 'PACKAGE_MAINTAINER_CHANGED';
 export const PACKAGE_MAINTAINER_REMOVED = 'PACKAGE_MAINTAINER_REMOVED';
 export const PACKAGE_META_CHANGED = 'PACKAGE_META_CHANGED';
 
+export interface PackageDeprecated {
+  version: string;
+  deprecated: string;
+}
+
+export interface PackageMetaChange {
+  deprecateds?: Array<PackageDeprecated>;
+}
+
+
 declare module '@eggjs/tegg' {
   interface Events {
     [PACKAGE_UNPUBLISHED]: (fullname: string) => Promise<void>;
     [PACKAGE_BLOCKED]: (fullname: string) => Promise<void>;
     [PACKAGE_UNBLOCKED]: (fullname: string) => Promise<void>;
-    [PACKAGE_VERSION_ADDED]: (fullname: string, version: string) => Promise<void>;
-    [PACKAGE_VERSION_REMOVED]: (fullname: string, version: string) => Promise<void>;
+    [PACKAGE_VERSION_ADDED]: (fullname: string, version: string, tag?: string) => Promise<void>;
+    [PACKAGE_VERSION_REMOVED]: (fullname: string, version: string, tag?: string) => Promise<void>;
     [PACKAGE_TAG_ADDED]: (fullname: string, tag: string) => Promise<void>;
     [PACKAGE_TAG_CHANGED]: (fullname: string, tag: string) => Promise<void>;
     [PACKAGE_TAG_REMOVED]: (fullname: string, tag: string) => Promise<void>;
-    [PACKAGE_MAINTAINER_CHANGED]: (fullname: string) => Promise<void>;
+    [PACKAGE_MAINTAINER_CHANGED]: (fullname: string, maintainers: User[]) => Promise<void>;
     [PACKAGE_MAINTAINER_REMOVED]: (fullname: string, maintainer: string) => Promise<void>;
-    [PACKAGE_META_CHANGED]: (fullname: string, meta: object) => Promise<void>;
+    [PACKAGE_META_CHANGED]: (fullname: string, meta: PackageMetaChange) => Promise<void>;
   }
 }

app/core/service/ChangesStreamService.ts

@@ -3,17 +3,23 @@ import { setTimeout } from 'timers/promises';
 import {
   AccessLevel,
   ContextProto,
+  EggObjectFactory,
   Inject,
 } from '@eggjs/tegg';
-import {
-  EggContextHttpClient,
-} from 'egg';
 import { TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { Task } from '../entity/Task';
+import { HOST_NAME, ChangesStreamTask, Task } from '../entity/Task';
 import { PackageSyncerService } from './PackageSyncerService';
 import { TaskService } from './TaskService';
+import { RegistryManagerService } from './RegistryManagerService';
+import { RegistryType } from '../../common/enum/Registry';
+import { E500 } from 'egg-errors';
+import { Registry } from '../entity/Registry';
+import { AbstractChangeStream, ChangesStreamChange } from '../../common/adapter/changesStream/AbstractChangesStream';
+import { getScopeAndName } from '../../common/PackageUtil';
+import { ScopeManagerService } from './ScopeManagerService';
+import { PackageRepository } from '../../repository/PackageRepository';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -22,49 +28,48 @@ export class ChangesStreamService extends AbstractService {
   @Inject()
   private readonly taskRepository: TaskRepository;
   @Inject()
-  private readonly httpclient: EggContextHttpClient;
-  @Inject()
   private readonly packageSyncerService: PackageSyncerService;
   @Inject()
   private readonly taskService: TaskService;
+  @Inject()
+  private readonly registryManagerService : RegistryManagerService;
+  @Inject()
+  private readonly scopeManagerService : ScopeManagerService;
+  @Inject()
+  private readonly eggObjectFactory: EggObjectFactory;
+  @Inject()
+  private readonly packageRepository: PackageRepository;
 
-  public async findExecuteTask() {
+  // 出于向下兼容考虑, changes_stream 类型 Task 分为
+  // GLOBAL_WORKER: 默认的同步源
+  // `{registryName}_WORKER`: 自定义 scope 的同步源
+  public async findExecuteTask(): Promise<ChangesStreamTask | null> {
     const targetName = 'GLOBAL_WORKER';
-    const existsTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
-    if (!existsTask) {
+    const globalRegistryTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
+    // 如果没有配置默认同步源，先进行初始化
+    if (!globalRegistryTask) {
       await this.taskService.createTask(Task.createChangesStream(targetName), false);
     }
-    return await this.taskService.findExecuteTask(TaskType.ChangesStream);
+    // 自定义 scope 由 admin 手动创建
+    // 根据 TaskType.ChangesStream 从队列中获取
+    return await this.taskService.findExecuteTask(TaskType.ChangesStream) as ChangesStreamTask;
   }
 
-  public async executeTask(task: Task) {
+  public async executeTask(task: ChangesStreamTask) {
     task.authorIp = os.hostname();
     task.authorId = `pid_${process.pid}`;
     await this.taskRepository.saveTask(task);
 
-    const changesStreamRegistry: string = this.config.cnpmcore.changesStreamRegistry;
-    // https://github.com/npm/registry-follower-tutorial
-    // default "update_seq": 7138885,
+    // 初始化 changeStream 任务
+    // since 默认从 1 开始
     try {
       let since: string = task.data.since;
-      // get update_seq from ${changesStreamRegistry} on the first time
       if (!since) {
-        const { status, data } = await this.httpclient.request(changesStreamRegistry, {
-          followRedirect: true,
-          timeout: 10000,
-          dataType: 'json',
-        });
-        if (data.update_seq) {
-          since = String(data.update_seq - 10);
-        } else {
-          since = '7139538';
-        }
-        this.logger.warn('[ChangesStreamService.executeTask:firstSeq] GET %s status: %s, data: %j, since: %s',
-          changesStreamRegistry, status, data, since);
+        since = await this.getInitialSince(task);
       }
       // allow disable changesStream dynamic
       while (since && this.config.cnpmcore.enableChangesStream) {
-        const { lastSince, taskCount } = await this.handleChanges(since, task);
+        const { lastSince, taskCount } = await this.executeSync(since, task);
         this.logger.warn('[ChangesStreamService.executeTask:changes] since: %s => %s, %d new tasks, taskId: %s, updatedAt: %j',
           since, lastSince, taskCount, task.taskId, task.updatedAt);
         since = lastSince;
@@ -81,101 +86,105 @@ export class ChangesStreamService extends AbstractService {
     }
   }
 
-  private async handleChanges(since: string, task: Task) {
-    const changesStreamRegistry: string = this.config.cnpmcore.changesStreamRegistry;
-    const changesStreamRegistryMode: string = this.config.cnpmcore.changesStreamRegistryMode;
-    const db = `${changesStreamRegistry}/_changes?since=${since}`;
-    let lastSince = since;
-    let taskCount = 0;
-    if (changesStreamRegistryMode === 'streaming') {
-      const { res } = await this.httpclient.request(db, {
-        streaming: true,
-        timeout: 10000,
-      });
-      for await (const chunk of res) {
-        const text: string = chunk.toString();
-        // {"seq":7138879,"id":"@danydodson/prettier-config","changes":[{"rev":"5-a56057032714af25400d93517773a82a"}]}
-        // console.log('😄%j😄', text);
-        // 😄"{\"seq\":7138738,\"id\":\"wargerm\",\"changes\":[{\"rev\":\"59-f0a0d326db4c62ed480987a04ba3bf8f\"}]}"😄
-        // 😄",\n{\"seq\":7138739,\"id\":\"@laffery/webpack-starter-kit\",\"changes\":[{\"rev\":\"4-84a8dc470a07872f4cdf85cf8ef892a1\"}]},\n{\"seq\":7138741,\"id\":\"venom-bot\",\"changes\":[{\"rev\":\"103-908654b1ad4b0e0fd40b468d75730674\"}]}"😄
-        // 😄",\n{\"seq\":7138743,\"id\":\"react-native-template-pytorch-live\",\"changes\":[{\"rev\":\"40-871c686b200312303ba7c4f7f93e0362\"}]}"😄
-        // 😄",\n{\"seq\":7138745,\"id\":\"ccxt\",\"changes\":[{\"rev\":\"10205-25367c525a0a3bd61be3a72223ce212c\"}]}"😄
-        const matchs = text.matchAll(/"seq":(\d+),"id":"([^"]+)"/gm);
-        let count = 0;
-        let lastPackage = '';
-        for (const match of matchs) {
-          const seq = match[1];
-          const fullname = match[2];
-          if (seq && fullname) {
-            await this.packageSyncerService.createTask(fullname, {
-              authorIp: os.hostname(),
-              authorId: 'ChangesStreamService',
-              skipDependencies: true,
-              tips: `Sync cause by changes_stream(${changesStreamRegistry}) update seq: ${seq}`,
-            });
-            count++;
-            lastSince = seq;
-            lastPackage = fullname;
-          }
-        }
-        if (count > 0) {
-          taskCount += count;
-          task.data = {
-            ...task.data,
-            since: lastSince,
-            last_package: lastPackage,
-            last_package_created: new Date(),
-            task_count: (task.data.task_count || 0) + count,
-          };
-          await this.taskRepository.saveTask(task);
-        }
+  // 优先从 registryId 获取，如果没有的话再返回默认的 registry
+  public async prepareRegistry(task: ChangesStreamTask): Promise<Registry> {
+    const { registryId } = task.data || {};
+    // 如果已有 registryId, 查询 DB 直接获取
+    if (registryId) {
+      const registry = await this.registryManagerService.findByRegistryId(registryId);
+      if (!registry) {
+        this.logger.error('[ChangesStreamService.getRegistry:error] registryId %s not found', registryId);
+        throw new E500(`invalid change stream registry: ${registryId}`);
       }
-    } else {
-      // json mode
-      // {"results":[{"seq":1988653,"type":"PACKAGE_VERSION_ADDED","id":"dsr-package-mercy-magot-thorp-sward","changes":[{"version":"1.0.1"}]},
-      const { data } = await this.httpclient.request(db, {
-        followRedirect: true,
-        timeout: 30000,
-        dataType: 'json',
-        gzip: true,
-      });
-      if (data.results?.length > 0) {
-        let count = 0;
-        let lastPackage = '';
-        for (const change of data.results) {
-          const seq = change.seq;
-          const fullname = change.id;
-          if (seq && fullname && seq !== since) {
-            await this.packageSyncerService.createTask(fullname, {
-              authorIp: os.hostname(),
-              authorId: 'ChangesStreamService',
-              skipDependencies: true,
-              tips: `Sync cause by changes_stream(${changesStreamRegistry}) update seq: ${seq}, change: ${JSON.stringify(change)}`,
-            });
-            count++;
-            lastSince = seq;
-            lastPackage = fullname;
-          }
-        }
-        if (count > 0) {
-          taskCount += count;
-          task.data = {
-            ...task.data,
-            since: lastSince,
-            last_package: lastPackage,
-            last_package_created: new Date(),
-            task_count: (task.data.task_count || 0) + count,
-          };
-          await this.taskRepository.saveTask(task);
-        }
+      return registry;
+    }
+
+    // 从配置文件默认生成
+    const { changesStreamRegistryMode, changesStreamRegistry: host } = this.config.cnpmcore;
+    const type = changesStreamRegistryMode === 'json' ? 'cnpmcore' : 'npm';
+    const registry = await this.registryManagerService.createRegistry({
+      name: 'default',
+      type: type as RegistryType,
+      userPrefix: 'npm:',
+      host,
+      changeStream: `${host}/_changes`,
+    });
+    task.data = {
+      ...(task.data || {}),
+      registryId: registry.registryId,
+    };
+    await this.taskRepository.saveTask(task);
+
+    return registry;
+  }
+
+  // 根据 regsitry 判断是否需要添加同步任务
+  // 1. 如果该包已经指定了 registryId 则以 registryId 为准
+  // 1. 该包的 scope 在当前 registry 下
+  // 2. 如果 registry 下没有配置 scope (认为是通用 registry 地址) ，且该包的 scope 不在其他 registry 下
+  public async needSync(registry: Registry, fullname: string): Promise<boolean> {
+    const [ scopeName, name ] = getScopeAndName(fullname);
+    const packageEntity = await this.packageRepository.findPackage(scopeName, name);
+
+    if (packageEntity?.registryId) {
+      return registry.registryId === packageEntity.registryId;
+    }
+
+    const scope = await this.scopeManagerService.findByName(scopeName);
+    const inCurrentRegistry = scope && scope?.registryId === registry.registryId;
+    if (inCurrentRegistry) {
+      return true;
+    }
+
+    const registryScopeCount = await this.scopeManagerService.countByRegistryId(registry.registryId);
+    // 当前包没有 scope 信息，且当前 registry 下没有 scope，是通用 registry，需要同步
+    return !scope && !registryScopeCount;
+  }
+  public async getInitialSince(task: ChangesStreamTask): Promise<string> {
+    const registry = await this.prepareRegistry(task);
+    const changesStreamAdapter = await this.eggObjectFactory.getEggObject(AbstractChangeStream, registry.type) as AbstractChangeStream;
+    const since = await changesStreamAdapter.getInitialSince(registry);
+    return since;
+  }
+
+  // 从 changesStream 获取需要同步的数据
+  // 更新任务的 since 和 taskCount 相关字段
+  public async executeSync(since: string, task: ChangesStreamTask) {
+    const registry = await this.prepareRegistry(task);
+    const changesStreamAdapter = await this.eggObjectFactory.getEggObject(AbstractChangeStream, registry.type) as AbstractChangeStream;
+    let taskCount = 0;
+    let lastSince = since;
+
+    // 获取需要同步的数据
+    // 只获取需要同步的 task 信息
+    const stream = await changesStreamAdapter.fetchChanges(registry, since);
+    let lastPackage: string | undefined;
+
+    // 创建同步任务
+    for await (const change of stream) {
+      const { fullname, seq } = change as ChangesStreamChange;
+      lastPackage = fullname;
+      const valid = await this.needSync(registry, fullname);
+      if (valid) {
+        taskCount++;
+        lastSince = seq;
+        await this.packageSyncerService.createTask(fullname, {
+          authorIp: HOST_NAME,
+          authorId: 'ChangesStreamService',
+          registryId: registry.registryId,
+          skipDependencies: true,
+          tips: `Sync cause by changes_stream(${registry.changeStream}) update seq: ${seq}`,
+        });
+        // 实时更新 task 信息
+        task.updateSyncData({
+          lastSince,
+          lastPackage,
+          taskCount,
+        });
+        await this.taskRepository.saveTask(task);
       }
     }
 
-    if (taskCount === 0) {
-      // keep update task, make sure updatedAt changed
-      task.updatedAt = new Date();
-      await this.taskRepository.saveTask(task);
-    }
     return { lastSince, taskCount };
   }
 }

app/core/service/CreateHookTriggerService.ts

@@ -0,0 +1,78 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { AbstractService } from '../../common/AbstractService';
+import { HookType } from '../../common/enum/Hook';
+import { TaskState } from '../../common/enum/Task';
+import { HookEvent } from '../entity/HookEvent';
+import { CreateHookTask, Task } from '../entity/Task';
+import { HookRepository } from '../../repository/HookRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import pMap from 'p-map';
+import { Hook } from '../entity/Hook';
+import { TaskService } from './TaskService';
+import { isoNow } from '../../common/LogUtil';
+import { getScopeAndName } from '../../common/PackageUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class CreateHookTriggerService extends AbstractService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async executeTask(task: CreateHookTask): Promise<void> {
+    const { hookEvent } = task.data;
+    const [ scope, name ] = getScopeAndName(hookEvent.fullname);
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][Hooks] package ${hookEvent.fullname} not exits`);
+      return;
+    }
+
+    const startLog = [
+      `[${isoNow()}][Hooks] Start Create Trigger for ${pkg.fullname} ${task.data.hookEvent.changeId}`,
+      `[${isoNow()}][Hooks] change content ${JSON.stringify(task.data.hookEvent.change)}`,
+    ];
+    await this.taskService.finishTask(task, TaskState.Processing, startLog.join('\n'));
+
+    try {
+      await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Package} ${pkg.fullname}\n`);
+      await this.createTriggerByMethod(task, HookType.Package, pkg.fullname, hookEvent);
+      await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Scope} ${pkg.scope}\n`);
+      await this.createTriggerByMethod(task, HookType.Scope, pkg.scope, hookEvent);
+
+      const maintainers = await this.packageRepository.listPackageMaintainers(pkg.packageId);
+      for (const maintainer of maintainers) {
+        await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Owner} ${maintainer.name}\n`);
+        await this.createTriggerByMethod(task, HookType.Owner, maintainer.name, hookEvent);
+      }
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][Hooks] create trigger succeed \n`);
+    } catch (e) {
+      e.message = 'create trigger failed: ' + e.message;
+      await this.taskService.finishTask(task, TaskState.Fail, `[${isoNow()}][Hooks] ${e.stack} \n`);
+      return;
+    }
+  }
+
+  private async createTriggerByMethod(task: Task, type: HookType, name: string, hookEvent: HookEvent) {
+    let hooks = await this.hookRepository.listHooksByTypeAndName(type, name);
+    while (hooks.length) {
+      await this.createTriggerTasks(hooks, hookEvent);
+      hooks = await this.hookRepository.listHooksByTypeAndName(type, name, hooks[hooks.length - 1].id);
+      await this.taskService.appendTaskLog(task,
+        `[${isoNow()}][Hooks] PushHooks to ${type} ${name} ${hooks.length} \n`);
+    }
+  }
+
+  private async createTriggerTasks(hooks: Array<Hook>, hookEvent: HookEvent) {
+    await pMap(hooks, async hook => {
+      const triggerHookTask = Task.createTriggerHookTask(hookEvent, hook.hookId);
+      await this.taskService.createTask(triggerHookTask, true);
+    }, { concurrency: 5 });
+  }
+}

app/core/service/HookManageService.ts

@@ -0,0 +1,96 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { Hook } from '../entity/Hook';
+import { HookType } from '../../common/enum/Hook';
+import {
+  ForbiddenError,
+  NotFoundError,
+} from 'egg-errors';
+import { HookRepository } from '../../repository/HookRepository';
+import { EggAppConfig } from 'egg';
+
+export interface CreateHookCommand {
+  type: HookType;
+  ownerId: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface UpdateHookCommand {
+  operatorId: string;
+  hookId: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface DeleteHookCommand {
+  operatorId: string;
+  hookId: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookManageService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  get hooksLimit() {
+    return this.config.cnpmcore.hooksLimit;
+  }
+
+  async createHook(cmd: CreateHookCommand): Promise<Hook> {
+    const hooks = await this.hookRepository.listHooksByOwnerId(cmd.ownerId);
+    // FIXME: 会有并发问题，需要有一个用户全局锁去记录
+    if (hooks.length >= this.hooksLimit) {
+      throw new ForbiddenError('hooks limit exceeded');
+    }
+    const hook = Hook.create(cmd);
+    await this.hookRepository.saveHook(hook);
+    return hook;
+  }
+
+  async updateHook(cmd: UpdateHookCommand): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(cmd.hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${cmd.hookId} not found`);
+    }
+    if (hook.ownerId !== cmd.operatorId) {
+      throw new ForbiddenError(`hook ${cmd.hookId} not belong to ${cmd.operatorId}`);
+    }
+    hook.endpoint = cmd.endpoint;
+    hook.secret = cmd.secret;
+    await this.hookRepository.saveHook(hook);
+    return hook;
+  }
+
+  async deleteHook(cmd: DeleteHookCommand): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(cmd.hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${cmd.hookId} not found`);
+    }
+    if (hook.ownerId !== cmd.operatorId) {
+      throw new ForbiddenError(`hook ${cmd.hookId} not belong to ${cmd.operatorId}`);
+    }
+    await this.hookRepository.removeHook(cmd.hookId);
+    return hook;
+  }
+
+  async listHooksByOwnerId(ownerId: string): Promise<Hook[]> {
+    return await this.hookRepository.listHooksByOwnerId(ownerId);
+  }
+
+  async getHookByOwnerId(hookId: string, userId: string): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${hookId} not found`);
+    }
+    if (hook.ownerId !== userId) {
+      throw new ForbiddenError(`hook ${hookId} not belong to ${userId}`);
+    }
+    return hook;
+  }
+}

app/core/service/HookTriggerService.ts

@@ -0,0 +1,111 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { TriggerHookTask } from '../entity/Task';
+import { HookEvent } from '../entity/HookEvent';
+import { HookRepository } from '../../repository/HookRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { DistRepository } from '../../repository/DistRepository';
+import { UserRepository } from '../../repository/UserRepository';
+import { Hook } from '../entity/Hook';
+import { EggContextHttpClient } from 'egg';
+import { isoNow } from '../../common/LogUtil';
+import { TaskState } from '../../common/enum/Task';
+import { TaskService } from './TaskService';
+import { getScopeAndName } from '../../common/PackageUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookTriggerService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly distRepository: DistRepository;
+
+  @Inject()
+  private readonly userRepository: UserRepository;
+
+  @Inject()
+  private readonly httpclient: EggContextHttpClient;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async executeTask(task: TriggerHookTask) {
+    const { hookId, hookEvent } = task.data;
+    const hook = await this.hookRepository.findHookById(hookId);
+    if (!hook) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] hook ${hookId} not exits`);
+      return;
+    }
+    try {
+      const payload = await this.createTriggerPayload(task, hookEvent, hook);
+      if (!payload) {
+        await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] generate payload failed \n`);
+        return;
+      }
+      const status = await this.doExecuteTrigger(hook, payload);
+      hook.latestTaskId = task.taskId;
+      task.data.responseStatus = status;
+      await this.hookRepository.saveHook(hook);
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] trigger hook succeed ${status} \n`);
+    } catch (e) {
+      e.message = 'trigger hook failed: ' + e.message;
+      task.error = e.message;
+      await this.taskService.finishTask(task, TaskState.Fail, `[${isoNow()}][TriggerHooks] ${e.stack} \n`);
+      return;
+    }
+  }
+
+  async doExecuteTrigger(hook: Hook, payload: object): Promise<number> {
+    const { digest, payloadStr } = hook.signPayload(payload);
+    const url = new URL(hook.endpoint);
+    const res = await this.httpclient.request(hook.endpoint, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'x-npm-signature': `sha256=${digest}`,
+        host: url.host,
+      },
+      // webhook 场景下，由于 endpoint 都不同
+      // 因此几乎不存在连接复用的情况，因此这里不使用 keepAlive
+      agent: false,
+      httpsAgent: false,
+      data: payloadStr,
+    } as any);
+    if (res.status >= 200 && res.status < 300) {
+      return res.status;
+    }
+    throw new Error(`hook response with ${res.status}`);
+  }
+
+  async createTriggerPayload(task: TriggerHookTask, hookEvent: HookEvent, hook: Hook): Promise<object | undefined> {
+    const [ scope, name ] = getScopeAndName(hookEvent.fullname);
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] can not found pkg for ${hookEvent.fullname} \n`);
+      return;
+    }
+    const user = await this.userRepository.findUserByUserId(hook.ownerId);
+    if (!user) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] can not found user for ${hook.ownerId} \n`);
+      return;
+    }
+    const manifest = await this.distRepository.readDistBytesToJSON(pkg!.manifestsDist!);
+    return {
+      event: hookEvent.event,
+      name: pkg.fullname,
+      type: 'package',
+      version: '1.0.0',
+      hookOwner: {
+        username: user.name,
+      },
+      payload: manifest,
+      change: hookEvent.change,
+      time: hookEvent.time,
+    };
+  }
+}

app/core/service/PackageManagerService.ts

@@ -46,6 +46,7 @@ export interface PublishPackageCmd {
   version: string;
   description: string;
   packageJson: any;
+  registryId?: string;
   readme: string;
   // require content or localFile field
   dist: RequireAtLeastOne<{
@@ -95,6 +96,7 @@ export class PackageManagerService extends AbstractService {
         name: cmd.name,
         isPrivate: cmd.isPrivate,
         description: cmd.description,
+        registryId: cmd.registryId,
       });
     } else {
       // update description
@@ -216,7 +218,7 @@ export class PackageManagerService extends AbstractService {
     if (cmd.tag) {
       await this.savePackageTag(pkg, cmd.tag, cmd.version, true);
     }
-    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version);
+    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, cmd.tag);
     return pkgVersion;
   }
 
@@ -273,7 +275,7 @@ export class PackageManagerService extends AbstractService {
   async replacePackageMaintainers(pkg: Package, maintainers: User[]) {
     await this.packageRepository.replacePackageMaintainers(pkg.packageId, maintainers.map(m => m.userId));
     await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
-    this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname);
+    this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
   }
 
   async savePackageMaintainers(pkg: Package, maintainers: User[]) {
@@ -286,7 +288,7 @@ export class PackageManagerService extends AbstractService {
     }
     if (hasNewRecord) {
       await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
-      this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname);
+      this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
     }
   }
 
@@ -468,6 +470,7 @@ export class PackageManagerService extends AbstractService {
     // all versions removed
     const versions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
     if (versions.length > 0) {
+      let updateTag: string | undefined;
       // make sure latest tag exists
       const latestTag = await this.packageRepository.findPackageTag(pkg.packageId, 'latest');
       if (latestTag?.version === pkgVersion.version) {
@@ -475,12 +478,13 @@ export class PackageManagerService extends AbstractService {
         // https://github.com/npm/libnpmpublish/blob/main/unpublish.js#L62
         const latestVersion = versions.sort(semver.compareLoose).pop();
         if (latestVersion) {
+          updateTag = latestTag.tag;
           await this.savePackageTag(pkg, latestTag.tag, latestVersion, true);
         }
       }
       if (skipRefreshPackageManifests !== true) {
         await this.refreshPackageChangeVersionsToDists(pkg, undefined, [ pkgVersion.version ]);
-        this.eventBus.emit(PACKAGE_VERSION_REMOVED, pkg.fullname, pkgVersion.version);
+        this.eventBus.emit(PACKAGE_VERSION_REMOVED, pkg.fullname, pkgVersion.version, updateTag);
       }
       return;
     }

app/core/service/PackageSyncerService.ts

@@ -19,13 +19,15 @@ import { PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { UserRepository } from '../../repository/UserRepository';
 import { DistRepository } from '../../repository/DistRepository';
-import { Task, SyncPackageTaskOptions } from '../entity/Task';
+import { Task, SyncPackageTaskOptions, CreateSyncPackageTask } from '../entity/Task';
 import { Package } from '../entity/Package';
 import { UserService } from './UserService';
 import { TaskService } from './TaskService';
 import { PackageManagerService } from './PackageManagerService';
 import { CacheService } from './CacheService';
 import { User } from '../entity/User';
+import { RegistryManagerService } from './RegistryManagerService';
+import { Registry } from '../entity/Registry';
 
 function isoNow() {
   return new Date().toISOString();
@@ -57,6 +59,8 @@ export class PackageSyncerService extends AbstractService {
   private readonly httpclient: EggContextHttpClient;
   @Inject()
   private readonly distRepository: DistRepository;
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
 
   public async createTask(fullname: string, options?: SyncPackageTaskOptions) {
     return await this.taskService.createTask(Task.createSyncPackage(fullname, options), true);
@@ -71,7 +75,7 @@ export class PackageSyncerService extends AbstractService {
   }
 
   public async findExecuteTask() {
-    return await this.taskService.findExecuteTask(TaskType.SyncPackage);
+    return await this.taskService.findExecuteTask(TaskType.SyncPackage) as CreateSyncPackageTask;
   }
 
   public get allowSyncDownloadData() {
@@ -190,10 +194,26 @@ export class PackageSyncerService extends AbstractService {
     await this.taskService.appendTaskLog(task, logs.join('\n'));
   }
 
+  public async initSpecRegistry(task: Task): Promise<Registry | null> {
+    const { registryId } = task.data as SyncPackageTaskOptions;
+    let targetHost: string = this.config.cnpmcore.sourceRegistry;
+    let registry: Registry | null = null;
+    // 历史 Task 可能没有配置 registryId
+    if (registryId) {
+      registry = await this.registryManagerService.findByRegistryId(registryId);
+      if (registry?.host) {
+        targetHost = registry.host;
+      }
+    }
+    this.npmRegistry.setRegistryHost(targetHost);
+    return registry;
+  }
+
   public async executeTask(task: Task) {
     const fullname = task.targetName;
-    const { tips, skipDependencies: originSkipDependencies, syncDownloadData } = task.data as SyncPackageTaskOptions;
-    const registry = this.npmRegistry.registry;
+    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory } = task.data as SyncPackageTaskOptions;
+    const registry = await this.initSpecRegistry(task);
+    const registryHost = this.npmRegistry.registry;
     let logs: string[] = [];
     if (tips) {
       logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`);
@@ -206,7 +226,7 @@ export class PackageSyncerService extends AbstractService {
     const logUrl = `${this.config.cnpmcore.registry}/-/package/${fullname}/syncs/${task.taskId}/log`;
     this.logger.info('[PackageSyncerService.executeTask:start] taskId: %s, targetName: %s, attempts: %s, taskQueue: %s/%s, syncUpstream: %s, log: %s',
       task.taskId, task.targetName, task.attempts, taskQueueLength, taskQueueHighWaterSize, syncUpstream, logUrl);
-    logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registry}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
+    logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registryHost}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, forceSyncHistory: ${!!forceSyncHistory} attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
     logs.push(`[${isoNow()}] 🚧 log: ${logUrl}`);
 
     const [ scope, name ] = getScopeAndName(fullname);
@@ -322,7 +342,7 @@ export class PackageSyncerService extends AbstractService {
       for (const maintainer of maintainers) {
         if (maintainer.name && maintainer.email) {
           maintainersMap[maintainer.name] = maintainer;
-          const { changed, user } = await this.userService.savePublicUser(maintainer.name, maintainer.email);
+          const { changed, user } = await this.userService.saveUser(registry?.userPrefix, maintainer.name, maintainer.email);
           users.push(user);
           if (changed) {
             changedUserCount++;
@@ -391,7 +411,7 @@ export class PackageSyncerService extends AbstractService {
       const version: string = item.version;
       if (!version) continue;
       let existsItem = existsVersionMap[version];
-      const existsAbbreviatedItem = abbreviatedVersionMap[version];
+      let existsAbbreviatedItem = abbreviatedVersionMap[version];
       const shouldDeleteReadme = !!(existsItem && 'readme' in existsItem);
       if (pkg) {
         if (existsItem) {
@@ -412,6 +432,18 @@ export class PackageSyncerService extends AbstractService {
             logs.push(`[${isoNow()}] 🐛 Remote version ${version} not exists on local manifests, need to refresh`);
           }
         }
+
+        if (existsItem && forceSyncHistory === true) {
+          const pkgVer = await this.packageRepository.findPackageVersion(pkg.packageId, version);
+          if (pkgVer) {
+            logs.push(`[${isoNow()}] 🚧 [${syncIndex}] Remove version ${version} for force sync history`);
+            await this.packageManagerService.removePackageVersion(pkg, pkgVer, true);
+            existsItem = undefined;
+            existsAbbreviatedItem = undefined;
+            existsVersionMap[version] = undefined;
+            abbreviatedVersionMap[version] = undefined;
+          }
+        }
       }
 
       if (existsItem) {
@@ -505,6 +537,7 @@ export class PackageSyncerService extends AbstractService {
         description,
         packageJson: item,
         readme,
+        registryId: registry?.registryId,
         dist: {
           localFile,
         },

app/core/service/RegistryManagerService.ts

@@ -0,0 +1,115 @@
+import {
+  AccessLevel,
+  ContextProto,
+  Inject,
+} from '@eggjs/tegg';
+import { E400, NotFoundError } from 'egg-errors';
+import { RegistryRepository } from '../../repository/RegistryRepository';
+import { AbstractService } from '../../common/AbstractService';
+import { Registry } from '../entity/Registry';
+import { PageOptions, PageResult } from '../util/EntityUtil';
+import { ScopeManagerService } from './ScopeManagerService';
+import { TaskService } from './TaskService';
+import { Task } from '../entity/Task';
+
+export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name'> {
+  operatorId?: string;
+}
+export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'registryId'> {
+  operatorId?: string;
+}
+export interface RemoveRegistryCmd extends Pick<Registry, 'registryId'> {
+  operatorId?: string;
+}
+
+export interface StartSyncCmd {
+  registryId: string;
+  since?: string;
+  operatorId?: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class RegistryManagerService extends AbstractService {
+  @Inject()
+  private readonly registryRepository: RegistryRepository;
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async createSyncChangesStream(startSyncCmd: StartSyncCmd): Promise<void> {
+    const { registryId, operatorId = '-', since } = startSyncCmd;
+    this.logger.info('[RegistryManagerService.startSyncChangesStream:prepare] operatorId: %s, registryId: %s, since: %s', operatorId, registryId, since);
+    const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
+    if (!registry) {
+      throw new NotFoundError(`registry ${registryId} not found`);
+    }
+
+    // 防止和 GLOBAL_WORKER 冲突，只能有一个默认的全局 registry
+    const scopesCount = await this.scopeManagerService.countByRegistryId(registryId);
+    if (scopesCount === 0) {
+      throw new E400(`registry ${registryId} has no scopes, please create scopes first`);
+    }
+
+    // 启动 changeStream
+    const targetName = `${registry.name.toUpperCase()}_WORKER`;
+    await this.taskService.createTask(Task.createChangesStream(targetName, since), false);
+  }
+
+  async createRegistry(createCmd: CreateRegistryCmd): Promise<Registry> {
+    const { name, changeStream, host, userPrefix, type, operatorId = '-' } = createCmd;
+    this.logger.info('[RegistryManagerService.createRegistry:prepare] operatorId: %s, createCmd: %j', operatorId, createCmd);
+    const registry = Registry.create({
+      name,
+      changeStream,
+      host,
+      userPrefix,
+      type,
+    });
+    await this.registryRepository.saveRegistry(registry);
+    return registry;
+  }
+
+  // 更新部分 registry 信息
+  // 不允许 userPrefix 字段变更
+  async updateRegistry(updateCmd: UpdateRegistryCmd) {
+    const { name, changeStream, host, type, registryId, operatorId = '-' } = updateCmd;
+    this.logger.info('[RegistryManagerService.updateRegistry:prepare] operatorId: %s, updateCmd: %j', operatorId, updateCmd);
+    const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
+    if (!registry) {
+      throw new NotFoundError(`registry ${registryId} not found`);
+    }
+    Object.assign(registry, {
+      name,
+      changeStream,
+      host,
+      type,
+    });
+    await this.registryRepository.saveRegistry(registry);
+  }
+
+  // list all registries with scopes
+  async listRegistries(page: PageOptions): Promise<PageResult<Registry>> {
+    return await this.registryRepository.listRegistries(page);
+  }
+
+  async findByRegistryId(registryId: string): Promise<Registry | null> {
+    return await this.registryRepository.findRegistryByRegistryId(registryId);
+  }
+
+  async findByRegistryName(registryName?: string): Promise<Registry | null> {
+    return await this.registryRepository.findRegistry(registryName);
+  }
+
+  // 删除 Registry 方法
+  // 可选传入 operatorId 作为参数，用于记录操作人员
+  // 同时删除对应的 scope 数据
+  async remove(removeCmd: RemoveRegistryCmd): Promise<void> {
+    const { registryId, operatorId = '-' } = removeCmd;
+    this.logger.info('[RegistryManagerService.remove:prepare] operatorId: %s, registryId: %s', operatorId, registryId);
+    await this.registryRepository.removeRegistry(registryId);
+    await this.scopeManagerService.removeByRegistryId({ registryId, operatorId });
+  }
+}

app/core/service/ScopeManagerService.ts

@@ -0,0 +1,74 @@
+import {
+  AccessLevel,
+  ContextProto,
+  Inject,
+} from '@eggjs/tegg';
+import { ScopeRepository } from '../../repository/ScopeRepository';
+import { AbstractService } from '../../common/AbstractService';
+import { Scope } from '../entity/Scope';
+import { PageOptions, PageResult } from '../util/EntityUtil';
+
+export interface CreateScopeCmd extends Pick<Scope, 'name' | 'registryId'> {
+  operatorId?: string;
+}
+export interface UpdateRegistryCmd extends Pick<Scope, 'name' | 'scopeId' | 'registryId'> {
+  operatorId?: string;
+}
+
+export interface RemoveScopeCmd {
+  scopeId: string;
+  operatorId?: string;
+}
+
+export interface RemoveScopeByRegistryIdCmd {
+  registryId: string;
+  operatorId?: string;
+}
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class ScopeManagerService extends AbstractService {
+  @Inject()
+  private readonly scopeRepository: ScopeRepository;
+
+  async findByName(name: string): Promise<Scope | null> {
+    const scope = await this.scopeRepository.findByName(name);
+    return scope;
+  }
+
+  async countByRegistryId(registryId: string): Promise<number> {
+    const count = await this.scopeRepository.countByRegistryId(registryId);
+    return count;
+  }
+
+  async createScope(createCmd: CreateScopeCmd): Promise<Scope> {
+    const { name, registryId, operatorId } = createCmd;
+    this.logger.info('[ScopeManagerService.CreateScope:prepare] operatorId: %s, createCmd: %s', operatorId, createCmd);
+    const scope = Scope.create({
+      name,
+      registryId,
+    });
+    await this.scopeRepository.saveScope(scope);
+    return scope;
+  }
+
+  async listScopes(page: PageOptions): Promise<PageResult<Scope>> {
+    return await this.scopeRepository.listScopes(page);
+  }
+
+  async listScopesByRegistryId(registryId: string, page: PageOptions): Promise<PageResult<Scope>> {
+    return await this.scopeRepository.listScopesByRegistryId(registryId, page);
+  }
+
+  async removeByRegistryId(removeCmd: RemoveScopeByRegistryIdCmd): Promise<void> {
+    const { registryId, operatorId } = removeCmd;
+    this.logger.info('[ScopeManagerService.remove:prepare] operatorId: %s, registryId: %s', operatorId, registryId);
+    return await this.scopeRepository.removeScopeByRegistryId(registryId);
+  }
+
+  async remove(removeCmd: RemoveScopeCmd): Promise<void> {
+    const { scopeId, operatorId } = removeCmd;
+    this.logger.info('[ScopeManagerService.remove:prepare] operatorId: %s, scopeId: %s', operatorId, scopeId);
+    return await this.scopeRepository.removeScope(scopeId);
+  }
+}

app/core/service/TaskService.ts

@@ -8,7 +8,7 @@ import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
 import { Task } from '../entity/Task';
-import { QueueAdapter } from '../../common/adapter/QueueAdapter';
+import { QueueAdapter } from '../../common/typing';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -40,9 +40,10 @@ export class TaskService extends AbstractService {
       return existsTask;
     }
     await this.taskRepository.saveTask(task);
-    const queueSize = await this.queueAdapter.push<string>(task.type, task.taskId);
+    await this.queueAdapter.push<string>(task.type, task.taskId);
+    const queueLength = await this.getTaskQueueLength(task.type);
     this.logger.info('[TaskService.createTask:new] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
-      task.type, task.targetName, task.taskId, queueSize);
+      task.type, task.targetName, task.taskId, queueLength);
     return task;
   }
 
@@ -54,15 +55,20 @@ export class TaskService extends AbstractService {
     // make sure updatedAt changed
     task.updatedAt = new Date();
     await this.taskRepository.saveTask(task);
-    const queueSize = await this.queueAdapter.push<string>(task.type, task.taskId);
+    await this.queueAdapter.push<string>(task.type, task.taskId);
+    const queueLength = await this.getTaskQueueLength(task.type);
     this.logger.info('[TaskService.retryTask:save] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
-      task.type, task.targetName, task.taskId, queueSize);
+      task.type, task.targetName, task.taskId, queueLength);
   }
 
   public async findTask(taskId: string) {
     return await this.taskRepository.findTask(taskId);
   }
 
+  public async findTasks(taskIdList: Array<string>) {
+    return await this.taskRepository.findTasks(taskIdList);
+  }
+
   public async findTaskLog(task: Task) {
     return await this.nfsAdapter.getDownloadUrlOrStream(task.logPath);
   }

app/core/service/UserService.ts

@@ -70,8 +70,8 @@ export class UserService extends AbstractService {
     return { user: userEntity, token };
   }
 
-  async savePublicUser(name: string, email: string): Promise<{ changed: boolean, user: UserEntity }> {
-    const storeName = name.startsWith('name:') ? name : `npm:${name}`;
+  async saveUser(userPrefix = 'npm:', name: string, email: string): Promise<{ changed: boolean, user: UserEntity }> {
+    const storeName = name.startsWith('name:') ? name : `${userPrefix}${name}`;
     let user = await this.userRepository.findUserByName(storeName);
     if (!user) {
       const passwordSalt = crypto.randomBytes(20).toString('hex');

app/core/util/ChangesStreamTransform.ts

@@ -0,0 +1,49 @@
+import { ChangesStreamChange } from '../../common/adapter/changesStream/AbstractChangesStream';
+import { Transform, TransformCallback, TransformOptions } from 'node:stream';
+
+// 网络问题可能会导致获取到的数据不完整
+// 最后数据可能会发生截断，需要按行读取，例如:
+// "seq": 1, "id": "test1",
+// "seq"
+// :2,
+// "id": "test2",
+// 先保存在 legacy 中，参与下次解析
+export default class ChangesStreamTransform extends Transform {
+  constructor(opts: TransformOptions = {}) {
+    super({
+      ...opts,
+      readableObjectMode: true,
+    });
+  }
+  private legacy = '';
+  _transform(chunk: any, _: BufferEncoding, callback: TransformCallback): void {
+    const text = chunk.toString();
+    const lines = text.split('\n');
+
+    for (const line of lines) {
+      const content = this.legacy + line;
+      const match = /"seq":(\d+),"id":"([^"]+)"/g.exec(content);
+      const seq = match?.[1];
+      const fullname = match?.[2];
+      if (seq && fullname) {
+        this.legacy = '';
+        // https://nodejs.org/en/docs/guides/backpressuring-in-streams/
+        // 需要处理 backpressure 场景
+        // 如果下游无法消费数据，就先暂停发送数据
+        // 自定义的 push 事件需要特殊处理
+        const pushed = this.push({ fullname, seq } as ChangesStreamChange);
+        if (!pushed) {
+          this.pause();
+          // 需要使用 drain 会重复触发，使用 once
+          this.once('drain', () => {
+            this.resume();
+          });
+        }
+      } else {
+        this.legacy += line;
+      }
+    }
+
+    callback();
+  }
+}

app/core/util/EntityUtil.ts

@@ -1,10 +1,24 @@
 import { EntityData } from '../entity/Entity';
 import ObjectID from 'bson-objectid';
+import { E400 } from 'egg-errors';
 
 type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
 
 export type EasyData<T extends EntityData, Id extends keyof T> = PartialBy<T, 'createdAt' | 'updatedAt' | Id>;
 
+const MAX_PAGE_SIZE = 100 as const;
+export interface PageOptions {
+  pageSize?: number;
+  pageIndex?: number;
+}
+export interface PageResult<T> {
+  count: number;
+  data: Array<T>
+}
+export interface PageLimitOptions {
+  offset: number;
+  limit: number;
+}
 
 export class EntityUtil {
   static defaultData<T extends EntityData, Id extends keyof T>(data: EasyData<T, Id>, id: Id): T {
@@ -17,4 +31,15 @@ export class EntityUtil {
   static createId(): string {
     return new ObjectID().toHexString();
   }
+
+  static convertPageOptionsToLimitOption(page: PageOptions): PageLimitOptions {
+    const { pageIndex = 0, pageSize = 20 } = page;
+    if (pageSize > MAX_PAGE_SIZE) {
+      throw new E400(`max page size is 100, current request is ${pageSize}`);
+    }
+    return {
+      offset: pageIndex * pageSize,
+      limit: pageSize,
+    };
+  }
 }

app/infra/QueueAdapter.ts

@@ -0,0 +1,47 @@
+import {
+  AccessLevel,
+  Inject,
+  ContextProto,
+} from '@eggjs/tegg';
+import { Redis } from 'ioredis';
+import { QueueAdapter } from '../common/typing';
+
+/**
+ * Use sort set to keep queue in order and keep same value only insert once
+ */
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+  name: 'queueAdapter',
+})
+export class RedisQueueAdapter implements QueueAdapter {
+  @Inject()
+  private readonly redis: Redis;
+
+  private getQueueName(key: string) {
+    return `CNPMCORE_Q_V2_${key}`;
+  }
+
+  private getQueueScoreName(key: string) {
+    return `CNPMCORE_Q_S_V2_${key}`;
+  }
+
+  /**
+   * If queue has the same item, return false
+   * If queue not has the same item, return true
+   */
+  async push<T>(key: string, item: T): Promise<boolean> {
+    const score = await this.redis.incr(this.getQueueScoreName(key));
+    const res = await this.redis.zadd(this.getQueueName(key), score, JSON.stringify(item));
+    return res !== 0;
+  }
+
+  async pop<T>(key: string) {
+    const [ json ] = await this.redis.zpopmin(this.getQueueName(key));
+    if (!json) return null;
+    return JSON.parse(json) as T;
+  }
+
+  async length(key: string) {
+    return await this.redis.zcount(this.getQueueName(key), '-inf', '+inf');
+  }
+}

app/port/UserRoleManager.ts

@@ -14,7 +14,7 @@ import { Token as TokenEntity } from '../core/entity/Token';
 import { sha512 } from '../common/UserUtil';
 
 // https://docs.npmjs.com/creating-and-viewing-access-tokens#creating-tokens-on-the-website
-type TokenRole = 'read' | 'publish' | 'setting';
+export type TokenRole = 'read' | 'publish' | 'setting';
 
 @ContextProto({
   // only inject on port module

app/port/controller/HookController.ts

@@ -0,0 +1,128 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  Inject,
+} from '@eggjs/tegg';
+import { HookManageService } from '../../core/service/HookManageService';
+import { TaskService } from '../../core/service/TaskService';
+import { UserRoleManager } from '../UserRoleManager';
+import { HookType } from '../../common/enum/Hook';
+import { TriggerHookTask } from '../../core/entity/Task';
+import { HookConvertor } from './convertor/HookConvertor';
+import { CreateHookRequestRule, UpdateHookRequestRule } from '../typebox';
+
+export interface CreateHookRequest {
+  type: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface UpdateHookRequest {
+  endpoint: string;
+  secret: string;
+}
+
+@HTTPController({
+  path: '/-/npm',
+})
+export class HookController {
+  @Inject()
+  private readonly hookManageService: HookManageService;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  @Inject()
+  private readonly userRoleManager: UserRoleManager;
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook',
+    method: HTTPMethodEnum.POST,
+  })
+  async createHook(@Context() ctx: EggContext, @HTTPBody() req: CreateHookRequest) {
+    ctx.tValidate(CreateHookRequestRule, req);
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.createHook({
+      ownerId: user.userId,
+      type: req.type as HookType,
+      name: req.name,
+      endpoint: req.endpoint,
+      secret: req.secret,
+    });
+    return HookConvertor.convertToHookVo(hook, user);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.PUT,
+  })
+  async updateHook(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() req: UpdateHookRequest) {
+    ctx.tValidate(UpdateHookRequestRule, req);
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.updateHook({
+      operatorId: user.userId,
+      hookId: id,
+      endpoint: req.endpoint,
+      secret: req.secret,
+    });
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToHookVo(hook, user, task);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  async deleteHook(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.deleteHook({
+      operatorId: user.userId,
+      hookId: id,
+    });
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToDeleteHookVo(hook, user, task);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks',
+    method: HTTPMethodEnum.GET,
+  })
+  async listHooks(@Context() ctx: EggContext) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const hooks = await this.hookManageService.listHooksByOwnerId(user.userId);
+    const tasks = await this.taskService.findTasks(hooks.map(t => t.latestTaskId).filter((t): t is string => !!t));
+    const res = hooks.map(hook => {
+      const task = tasks.find(t => t.taskId === hook.latestTaskId) as TriggerHookTask;
+      return HookConvertor.convertToHookVo(hook, user, task);
+    });
+    return {
+      objects: res,
+    };
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.GET,
+  })
+  async getHook(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const hook = await this.hookManageService.getHookByOwnerId(id, user.userId);
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToHookVo(hook, user, task);
+  }
+}

app/port/controller/PackageSyncController.ts

@@ -8,12 +8,14 @@ import {
   EggContext,
   Inject,
   HTTPQuery,
+  BackgroundTaskHelper,
 } from '@eggjs/tegg';
 import { ForbiddenError, NotFoundError } from 'egg-errors';
 import { AbstractController } from './AbstractController';
 import { FULLNAME_REG_STRING, getScopeAndName } from '../../common/PackageUtil';
 import { Task } from '../../core/entity/Task';
 import { PackageSyncerService } from '../../core/service/PackageSyncerService';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
 import { TaskState } from '../../common/enum/Task';
 import { SyncPackageTaskRule, SyncPackageTaskType } from '../typebox';
 
@@ -22,6 +24,12 @@ export class PackageSyncController extends AbstractController {
   @Inject()
   private packageSyncerService: PackageSyncerService;
 
+  @Inject()
+  private backgroundTaskHelper: BackgroundTaskHelper;
+
+  @Inject()
+  private registryManagerService: RegistryManagerService;
+
   private async _executeTaskAsync(task: Task) {
     const startTime = Date.now();
     this.logger.info('[PackageSyncController:executeTask:start] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
@@ -50,22 +58,34 @@ export class PackageSyncController extends AbstractController {
       throw new ForbiddenError('Not allow to sync package');
     }
     const tips = data.tips || `Sync cause by "${ctx.href}", parent traceId: ${ctx.tracer.traceId}`;
+    const isAdmin = await this.userRoleManager.isAdmin(ctx);
+
     const params = {
       fullname,
       tips,
       skipDependencies: !!data.skipDependencies,
       syncDownloadData: !!data.syncDownloadData,
       force: !!data.force,
+      // only admin allow to sync history version
+      forceSyncHistory: !!data.forceSyncHistory && isAdmin,
     };
     ctx.tValidate(SyncPackageTaskRule, params);
     const [ scope, name ] = getScopeAndName(params.fullname);
     const packageEntity = await this.packageRepository.findPackage(scope, name);
-    if (packageEntity?.isPrivate) {
+    const registry = await this.registryManagerService.findByRegistryName(data?.registryName);
+
+    if (!registry && data.registryName) {
+      throw new ForbiddenError(`Can\'t find target registry "${data.registryName}"`);
+    }
+    if (packageEntity?.isPrivate && !registry) {
       throw new ForbiddenError(`Can\'t sync private package "${params.fullname}"`);
     }
     if (params.syncDownloadData && !this.packageSyncerService.allowSyncDownloadData) {
       throw new ForbiddenError('Not allow to sync package download data');
     }
+    if (packageEntity?.registryId && packageEntity.registryId !== registry!.registryId) {
+      throw new ForbiddenError(`The package is synced from ${packageEntity.registryId}`);
+    }
     const authorized = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
     const task = await this.packageSyncerService.createTask(params.fullname, {
       authorIp: ctx.ip,
@@ -73,16 +93,21 @@ export class PackageSyncController extends AbstractController {
       tips: params.tips,
       skipDependencies: params.skipDependencies,
       syncDownloadData: params.syncDownloadData,
+      forceSyncHistory: params.forceSyncHistory,
+      registryId: registry?.registryId,
     });
     ctx.logger.info('[PackageSyncController.createSyncTask:success] taskId: %s, fullname: %s',
       task.taskId, fullname);
     if (data.force) {
-      const isAdmin = await this.userRoleManager.isAdmin(ctx);
       if (isAdmin) {
-        // execute task in background
-        this._executeTaskAsync(task);
-        ctx.logger.info('[PackageSyncController.createSyncTask:execute-immediately] taskId: %s',
-          task.taskId);
+        // set background task timeout to 5min
+        this.backgroundTaskHelper.timeout = 1000 * 60 * 5;
+        this.backgroundTaskHelper.run(async () => {
+          ctx.logger.info('[PackageSyncController.createSyncTask:execute-immediately] taskId: %s',
+            task.taskId);
+          // execute task in background
+          await this._executeTaskAsync(task);
+        });
       }
     }
     ctx.status = 201;
@@ -153,6 +178,7 @@ export class PackageSyncController extends AbstractController {
       skipDependencies: nodeps === 'true',
       syncDownloadData: false,
       force: false,
+      forceSyncHistory: false,
     };
     const task = await this.createSyncTask(ctx, fullname, options);
     return {

app/port/controller/RegistryController.ts

@@ -0,0 +1,110 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  HTTPQuery,
+  Inject,
+  Middleware,
+} from '@eggjs/tegg';
+import { NotFoundError } from 'egg-errors';
+import { AbstractController } from './AbstractController';
+import { Static } from 'egg-typebox-validate/typebox';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { AdminAccess } from '../middleware/AdminAccess';
+import { ScopeManagerService } from '../../core/service/ScopeManagerService';
+import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions } from '../typebox';
+
+@HTTPController()
+export class RegistryController extends AbstractController {
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+
+  @HTTPMethod({
+    path: '/-/registry',
+    method: HTTPMethodEnum.GET,
+  })
+  async listRegistries(@HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'], @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex']) {
+    const registries = await this.registryManagerService.listRegistries({ pageSize, pageIndex });
+    return registries;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.GET,
+  })
+  async showRegistry(@HTTPParam() id: string) {
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    return registry;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id/scopes',
+    method: HTTPMethodEnum.GET,
+  })
+  async showRegistryScopes(@HTTPParam() id: string, @HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'], @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex']) {
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    const scopes = await this.scopeManagerService.listScopesByRegistryId(id, { pageIndex, pageSize });
+    return scopes;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createRegistry(@Context() ctx: EggContext, @HTTPBody() registryOptions: Static<typeof RegistryCreateOptions>) {
+    ctx.tValidate(RegistryCreateOptions, registryOptions);
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const { name, changeStream, host, userPrefix = '', type } = registryOptions;
+    await this.registryManagerService.createRegistry({
+      name,
+      changeStream,
+      host,
+      userPrefix,
+      operatorId: authorizedUser.userId,
+      type,
+    });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id/sync',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createRegistrySyncTask(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() registryOptions: Static<typeof RegistryCreateSyncOptions>) {
+    ctx.tValidate(RegistryCreateSyncOptions, registryOptions);
+    const { since } = registryOptions;
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.registryManagerService.createSyncChangesStream({ registryId: registry.registryId, since, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async removeRegistry(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.registryManagerService.remove({ registryId: id, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+}

app/port/controller/ScopeController.ts

@@ -0,0 +1,63 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  Inject,
+  Middleware,
+} from '@eggjs/tegg';
+import { E400 } from 'egg-errors';
+import { AbstractController } from './AbstractController';
+import { Static } from 'egg-typebox-validate/typebox';
+import { AdminAccess } from '../middleware/AdminAccess';
+import { ScopeManagerService } from '../../core/service/ScopeManagerService';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { ScopeCreateOptions } from '../typebox';
+
+
+@HTTPController()
+export class ScopeController extends AbstractController {
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
+
+  @HTTPMethod({
+    path: '/-/scope',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createScope(@Context() ctx: EggContext, @HTTPBody() scopeOptions: Static<typeof ScopeCreateOptions>) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    ctx.tValidate(ScopeCreateOptions, scopeOptions);
+    const { name, registryId } = scopeOptions;
+
+    const registry = await this.registryManagerService.findByRegistryId(registryId);
+    if (!registry) {
+      throw new E400(`registry ${registryId} not found`);
+    }
+
+    await this.scopeManagerService.createScope({
+      name,
+      registryId,
+      operatorId: authorizedUser.userId,
+    });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/scope/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async removeScope(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.scopeManagerService.remove({ scopeId: id, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+}

app/port/controller/convertor/HookConvertor.ts

@@ -0,0 +1,61 @@
+import { Hook } from '../../../core/entity/Hook';
+import { TriggerHookTask } from '../../../core/entity/Task';
+import { User } from '../../../core/entity/User';
+import { HookType } from '../../../common/enum/Hook';
+
+export interface HookVo {
+  id: string;
+  username: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  type: HookType;
+  created: Date;
+  updated: Date;
+  delivered: boolean,
+  last_delivery: Date | null,
+  response_code: number,
+  status: 'active',
+}
+
+export interface DeleteHookVo {
+  id: string;
+  username: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  type: HookType;
+  created: Date;
+  updated: Date;
+  delivered: boolean,
+  last_delivery: Date | null,
+  response_code: number,
+  status: 'active',
+  deleted: boolean,
+}
+
+export class HookConvertor {
+  static convertToHookVo(hook: Hook, user: User, task?: TriggerHookTask | null | undefined): HookVo {
+    return {
+      id: hook.hookId,
+      username: user.name,
+      name: hook.name,
+      endpoint: hook.endpoint,
+      secret: hook.secret,
+      type: hook.type,
+      created: hook.createdAt,
+      updated: hook.updatedAt,
+      delivered: !!task,
+      last_delivery: task?.updatedAt || null,
+      response_code: task?.data.responseStatus || 0,
+      status: 'active',
+    };
+  }
+
+  static convertToDeleteHookVo(hook: Hook, user: User, task?: TriggerHookTask | null): DeleteHookVo {
+    const vo = HookConvertor.convertToHookVo(hook, user, task);
+    return Object.assign(vo, {
+      deleted: true,
+    });
+  }
+}

app/port/controller/package/ShowPackageController.ts

@@ -33,7 +33,7 @@ export class ShowPackageController extends AbstractController {
     const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType;
     // handle cache
     const cacheEtag = await this.cacheService.getPackageEtag(fullname, isFullManifests);
-    if (cacheEtag) {
+    if (!isSync && cacheEtag) {
       let requestEtag = ctx.request.get('if-none-match');
       if (requestEtag.startsWith('W/')) {
         requestEtag = requestEtag.substring(2);

app/port/middleware/AdminAccess.ts

@@ -0,0 +1,12 @@
+import { EggContext, Next } from '@eggjs/tegg';
+import { ForbiddenError } from 'egg-errors';
+import { UserRoleManager } from '../UserRoleManager';
+
+export async function AdminAccess(ctx: EggContext, next: Next) {
+  const userRoleManager = await ctx.getEggObject(UserRoleManager);
+  const isAdmin = await userRoleManager.isAdmin(ctx);
+  if (!isAdmin) {
+    throw new ForbiddenError('Not allow to access');
+  }
+  await next();
+}

app/port/typebox.ts

@@ -1,10 +1,32 @@
 import { Type, Static } from '@sinclair/typebox';
+import { RegistryType } from '../common/enum/Registry';
 import semver from 'semver';
+import { HookType } from '../common/enum/Hook';
 
 export const Name = Type.String({
   transform: [ 'trim' ],
 });
 
+export const Url = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 2048,
+});
+
+export const Secret = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 200,
+});
+
+export const HookName = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 428,
+});
+
+export const HookTypeType = Type.Enum(HookType);
+
 export const Tag = Type.String({
   format: 'semver-tag',
   transform: [ 'trim' ],
@@ -42,6 +64,10 @@ export const SyncPackageTaskRule = Type.Object({
   syncDownloadData: Type.Boolean(),
   // force sync immediately, only allow by admin
   force: Type.Boolean(),
+  // sync history version
+  forceSyncHistory: Type.Boolean(),
+  // source registry
+  registryName: Type.Optional(Type.String()),
 });
 export type SyncPackageTaskType = Static<typeof SyncPackageTaskRule>;
 
@@ -54,6 +80,18 @@ export const BlockPackageRule = Type.Object({
 });
 export type BlockPackageType = Static<typeof BlockPackageRule>;
 
+export const UpdateHookRequestRule = Type.Object({
+  endpoint: Url,
+  secret: Secret,
+});
+
+export const CreateHookRequestRule = Type.Object({
+  endpoint: Url,
+  secret: Secret,
+  name: HookName,
+  type: HookTypeType,
+});
+
 // https://github.com/xiekw2010/egg-typebox-validate#%E5%A6%82%E4%BD%95%E5%86%99%E8%87%AA%E5%AE%9A%E4%B9%89%E6%A0%A1%E9%AA%8C%E8%A7%84%E5%88%99
 // add custom validate to ajv
 export function patchAjv(ajv: any) {
@@ -70,3 +108,103 @@ export function patchAjv(ajv: any) {
     },
   });
 }
+
+export const QueryPageOptions = Type.Object({
+  pageSize: Type.Optional(Type.Number({
+    transform: [ 'trim' ],
+    minimum: 1,
+    maximum: 100,
+  })),
+  pageIndex: Type.Optional(Type.Number({
+    transform: [ 'trim' ],
+    minimum: 0,
+  })),
+});
+
+export const RegistryCreateSyncOptions = Type.Object({
+  since: Type.Optional(Type.String()),
+});
+
+export const RegistryCreateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  host: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  changeStream: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  userPrefix: Type.Optional(Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  })),
+  type: Type.Enum(RegistryType),
+});
+
+export const RegistryUpdateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  host: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  changeStream: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  userPrefix: Type.Optional(Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  })),
+  type: Type.Enum(RegistryType),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});
+
+export const ScopeCreateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});
+
+export const ScopeUpdateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  scopeId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});

app/repository/BinaryRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Binary as BinaryModel } from './model/Binary';
+import type { Binary as BinaryModel } from './model/Binary';
 import { Binary as BinaryEntity } from '../core/entity/Binary';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,25 +8,28 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class BinaryRepository extends AbstractRepository {
+  @Inject()
+  private readonly Binary: typeof BinaryModel;
+
   async saveBinary(binary: BinaryEntity): Promise<void> {
     if (binary.id) {
-      const model = await BinaryModel.findOne({ id: binary.id });
+      const model = await this.Binary.findOne({ id: binary.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(binary, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(binary, BinaryModel);
+      const model = await ModelConvertor.convertEntityToModel(binary, this.Binary);
       this.logger.info('[BinaryRepository:saveBinary:new] id: %s, binaryId: %s', model.id, model.binaryId);
     }
   }
 
   async findBinary(category: string, parent: string, name: string) {
-    const model = await BinaryModel.findOne({ category, parent, name });
+    const model = await this.Binary.findOne({ category, parent, name });
     if (model) return ModelConvertor.convertModelToEntity(model, BinaryEntity);
     return null;
   }
 
   async listBinaries(category: string, parent: string): Promise<BinaryEntity[]> {
-    const models = await BinaryModel.find({ category, parent });
+    const models = await this.Binary.find({ category, parent });
     return models.map(model => ModelConvertor.convertModelToEntity(model, BinaryEntity));
   }
 }

app/repository/ChangeRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Change as ChangeModel } from './model/Change';
+import type { Change as ChangeModel } from './model/Change';
 import { Change as ChangeEntity } from '../core/entity/Change';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,16 +8,19 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class ChangeRepository extends AbstractRepository {
+  @Inject()
+  private readonly Change: typeof ChangeModel;
+
   async addChange(change: ChangeEntity) {
-    await ModelConvertor.convertEntityToModel(change, ChangeModel);
+    await ModelConvertor.convertEntityToModel(change, this.Change);
   }
 
   async query(since: number, limit: number): Promise<Array<ChangeEntity>> {
-    const models = await ChangeModel.find({ id: { $gte: since } }).order('id', 'asc').limit(limit);
+    const models = await this.Change.find({ id: { $gte: since } }).order('id', 'asc').limit(limit);
     return models.toObject() as ChangeEntity[];
   }
 
   async getLastChange() {
-    return await ChangeModel.findOne().order('id', 'desc').limit(1);
+    return await this.Change.findOne().order('id', 'desc').limit(1);
   }
 }

app/repository/HookRepository.ts

@@ -0,0 +1,66 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { Hook } from '../core/entity/Hook';
+import type { Hook as HookModel } from './model/Hook';
+import { ModelConvertor } from './util/ModelConvertor';
+import { HookType } from '../common/enum/Hook';
+
+export interface UpdateHookCommand {
+  hookId: string;
+  endpoint: string;
+  secret: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookRepository {
+  @Inject()
+  private readonly Hook: typeof HookModel;
+
+  async saveHook(hook: Hook) {
+    if (hook.id) {
+      const model = await this.Hook.findOne({ id: hook.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(hook, model);
+    } else {
+      await ModelConvertor.convertEntityToModel(hook, this.Hook);
+    }
+  }
+
+  async findHookById(hookId: string): Promise<Hook | undefined> {
+    const model = await this.Hook.findOne({ hookId });
+    if (!model) return;
+    return ModelConvertor.convertModelToEntity(model, Hook);
+  }
+
+  async removeHook(hookId: string): Promise<void> {
+    await this.Hook.remove({ hookId });
+  }
+
+  /**
+   * only endpoint and secret can be updated
+   */
+  async updateHook(cmd: UpdateHookCommand) {
+    this.Hook.update({
+      hookId: cmd.hookId,
+    }, {
+      endpoint: cmd.endpoint,
+      secret: cmd.secret,
+    });
+  }
+
+  async listHooksByOwnerId(ownerId: string) {
+    const hookRows = await this.Hook.find({ ownerId });
+    return hookRows.map(row => ModelConvertor.convertModelToEntity(row, Hook));
+  }
+
+  async listHooksByTypeAndName(type: HookType, name: string, since?: bigint): Promise<Array<Hook>> {
+    let hookRows: Array<HookModel>;
+    if (typeof since !== 'undefined') {
+      hookRows = await this.Hook.find({ type, name, id: { $gt: since } }).limit(100);
+    } else {
+      hookRows = await this.Hook.find({ type, name }).limit(100);
+    }
+    return hookRows.map(row => ModelConvertor.convertModelToEntity(row, Hook));
+  }
+}

app/repository/PackageRepository.ts

@@ -1,17 +1,17 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
-import { Package as PackageModel } from './model/Package';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import type { Package as PackageModel } from './model/Package';
 import { Package as PackageEntity } from '../core/entity/Package';
 import { ModelConvertor } from './util/ModelConvertor';
 import { PackageVersion as PackageVersionEntity } from '../core/entity/PackageVersion';
-import { PackageVersion as PackageVersionModel } from './model/PackageVersion';
+import type { PackageVersion as PackageVersionModel } from './model/PackageVersion';
 import { PackageVersionManifest as PackageVersionManifestEntity } from '../core/entity/PackageVersionManifest';
-import { PackageVersionManifest as PackageVersionManifestModel } from './model/PackageVersionManifest';
-import { Dist as DistModel } from './model/Dist';
+import type { PackageVersionManifest as PackageVersionManifestModel } from './model/PackageVersionManifest';
+import type { Dist as DistModel } from './model/Dist';
 import { Dist as DistEntity } from '../core/entity/Dist';
 import { PackageTag as PackageTagEntity } from '../core/entity/PackageTag';
-import { PackageTag as PackageTagModel } from './model/PackageTag';
-import { Maintainer as MaintainerModel } from './model/Maintainer';
-import { User as UserModel } from './model/User';
+import type { PackageTag as PackageTagModel } from './model/PackageTag';
+import type { Maintainer as MaintainerModel } from './model/Maintainer';
+import type { User as UserModel } from './model/User';
 import { User as UserEntity } from '../core/entity/User';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -19,11 +19,32 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageRepository extends AbstractRepository {
+  @Inject()
+  private readonly Package: typeof PackageModel;
+
+  @Inject()
+  private readonly Dist: typeof DistModel;
+
+  @Inject()
+  private readonly PackageVersion: typeof PackageVersionModel;
+
+  @Inject()
+  private readonly PackageVersionManifest: typeof PackageVersionManifestModel;
+
+  @Inject()
+  private readonly PackageTag: typeof PackageTagModel;
+
+  @Inject()
+  private readonly Maintainer: typeof MaintainerModel;
+
+  @Inject()
+  private readonly User: typeof UserModel;
+
   async findPackage(scope: string, name: string): Promise<PackageEntity | null> {
-    const model = await PackageModel.findOne({ scope, name });
+    const model = await this.Package.findOne({ scope, name });
     if (!model) return null;
-    const manifestsDistModel = model.manifestsDistId ? await DistModel.findOne({ distId: model.manifestsDistId }) : null;
-    const abbreviatedsDistModel = model.abbreviatedsDistId ? await DistModel.findOne({ distId: model.abbreviatedsDistId }) : null;
+    const manifestsDistModel = model.manifestsDistId ? await this.Dist.findOne({ distId: model.manifestsDistId }) : null;
+    const abbreviatedsDistModel = model.abbreviatedsDistId ? await this.Dist.findOne({ distId: model.abbreviatedsDistId }) : null;
     const data = {
       manifestsDist: manifestsDistModel && ModelConvertor.convertModelToEntity(manifestsDistModel, DistEntity),
       abbreviatedsDist: abbreviatedsDistModel && ModelConvertor.convertModelToEntity(abbreviatedsDistModel, DistEntity),
@@ -33,18 +54,18 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageId(scope: string, name: string) {
-    const model = await PackageModel.findOne({ scope, name }).select('packageId');
+    const model = await this.Package.findOne({ scope, name }).select('packageId');
     if (!model) return null;
     return model.packageId;
   }
 
   async savePackage(pkgEntity: PackageEntity): Promise<void> {
     if (pkgEntity.id) {
-      const model = await PackageModel.findOne({ id: pkgEntity.id });
+      const model = await this.Package.findOne({ id: pkgEntity.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(pkgEntity, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(pkgEntity, PackageModel);
+      const model = await ModelConvertor.convertEntityToModel(pkgEntity, this.Package);
       this.logger.info('[PackageRepository:savePackage:new] id: %s, packageId: %s', model.id, model.packageId);
     }
   }
@@ -53,11 +74,11 @@ export class PackageRepository extends AbstractRepository {
     const dist = isFullManifests ? pkgEntity.manifestsDist : pkgEntity.abbreviatedsDist;
     if (!dist) return;
     if (dist.id) {
-      const model = await DistModel.findOne({ id: dist.id });
+      const model = await this.Dist.findOne({ id: dist.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(dist, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(dist, DistModel);
+      const model = await ModelConvertor.convertEntityToModel(dist, this.Dist);
       this.logger.info('[PackageRepository:savePackageDist:new] id: %s, distId: %s, packageId: %s',
         model.id, model.distId, pkgEntity.packageId);
     }
@@ -67,7 +88,7 @@ export class PackageRepository extends AbstractRepository {
   async removePackageDist(pkgEntity: PackageEntity, isFullManifests: boolean): Promise<void> {
     const dist = isFullManifests ? pkgEntity.manifestsDist : pkgEntity.abbreviatedsDist;
     if (!dist) return;
-    const model = await DistModel.findOne({ id: dist.id });
+    const model = await this.Dist.findOne({ id: dist.id });
     if (!model) return;
     await model.remove();
     this.logger.info('[PackageRepository:removePackageDist:remove] id: %s, distId: %s, packageId: %s',
@@ -79,9 +100,9 @@ export class PackageRepository extends AbstractRepository {
   // Package Maintainers
   // return true meaning create new record
   async savePackageMaintainer(packageId: string, userId: string): Promise<undefined | true> {
-    let model = await MaintainerModel.findOne({ packageId, userId });
+    let model = await this.Maintainer.findOne({ packageId, userId });
     if (!model) {
-      model = await MaintainerModel.create({ packageId, userId });
+      model = await this.Maintainer.create({ packageId, userId });
       this.logger.info('[PackageRepository:addPackageMaintainer:new] id: %s, packageId: %s, userId: %s',
         model.id, model.packageId, model.userId);
       return true;
@@ -89,22 +110,22 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageMaintainers(packageId: string): Promise<UserEntity[]> {
-    const models = await MaintainerModel.find({ packageId });
-    const userModels = await UserModel.find({ userId: models.map(m => m.userId) });
+    const models = await this.Maintainer.find({ packageId });
+    const userModels = await this.User.find({ userId: models.map(m => m.userId) });
     return userModels.map(user => ModelConvertor.convertModelToEntity(user, UserEntity));
   }
 
   async replacePackageMaintainers(packageId: string, userIds: string[]): Promise<void> {
-    await MaintainerModel.transaction(async () => {
+    await this.Maintainer.transaction(async ({ connection }) => {
       // delete exists
-      // const removeCount = await MaintainerModel.remove({ packageId }, true, { transaction });
-      const removeCount = await MaintainerModel.remove({ packageId });
+      // const removeCount = await this.Maintainer.remove({ packageId }, true, { transaction });
+      const removeCount = await this.Maintainer.remove({ packageId }, true, { connection });
       this.logger.info('[PackageRepository:replacePackageMaintainers:remove] %d rows, packageId: %s',
         removeCount, packageId);
       // add news
       for (const userId of userIds) {
-        // const model = await MaintainerModel.create({ packageId, userId }, transaction);
-        const model = await MaintainerModel.create({ packageId, userId });
+        // const model = await this.Maintainer.create({ packageId, userId }, transaction);
+        const model = await this.Maintainer.create({ packageId, userId }, { connection });
         this.logger.info('[PackageRepository:replacePackageMaintainers:new] id: %s, packageId: %s, userId: %s',
           model.id, model.packageId, model.userId);
       }
@@ -112,7 +133,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async removePackageMaintainer(packageId: string, userId: string) {
-    const model = await MaintainerModel.findOne({ packageId, userId });
+    const model = await this.Maintainer.findOne({ packageId, userId });
     if (model) {
       await model.remove();
       this.logger.info('[PackageRepository:removePackageMaintainer:remove] id: %s, packageId: %s, userId: %s',
@@ -124,36 +145,36 @@ export class PackageRepository extends AbstractRepository {
 
   // TODO: support paging
   async listPackagesByUserId(userId: string): Promise<PackageEntity[]> {
-    const models = await MaintainerModel.find({ userId });
-    const packageModels = await PackageModel.find({ packageId: models.map(m => m.packageId) });
+    const models = await this.Maintainer.find({ userId });
+    const packageModels = await this.Package.find({ packageId: models.map(m => m.packageId) });
     return packageModels.map(pkg => ModelConvertor.convertModelToEntity(pkg, PackageEntity));
   }
 
   async createPackageVersion(pkgVersionEntity: PackageVersionEntity) {
-    await PackageVersionModel.transaction(async function(transaction) {
+    await this.PackageVersion.transaction(async transaction => {
       await Promise.all([
         // FIXME: transaction is not the options
-        ModelConvertor.convertEntityToModel(pkgVersionEntity, PackageVersionModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.manifestDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.tarDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.readmeDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.abbreviatedDist, DistModel, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity, this.PackageVersion, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.manifestDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.tarDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.readmeDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.abbreviatedDist, this.Dist, transaction),
       ]);
     });
   }
 
   async savePackageVersion(pkgVersionEntity: PackageVersionEntity) {
     // only abbreviatedDist and manifestDist allow to change, like `deprecated` message
-    let model = await DistModel.findOne({ id: pkgVersionEntity.manifestDist.id });
+    let model = await this.Dist.findOne({ id: pkgVersionEntity.manifestDist.id });
     if (model) {
       await ModelConvertor.saveEntityToModel(pkgVersionEntity.manifestDist, model);
     }
-    model = await DistModel.findOne({ id: pkgVersionEntity.abbreviatedDist.id });
+    model = await this.Dist.findOne({ id: pkgVersionEntity.abbreviatedDist.id });
     if (model) {
       await ModelConvertor.saveEntityToModel(pkgVersionEntity.abbreviatedDist, model);
     }
     if (pkgVersionEntity.id) {
-      const model = await PackageVersionModel.findOne({ id: pkgVersionEntity.id });
+      const model = await this.PackageVersion.findOne({ id: pkgVersionEntity.id });
       if (model) {
         await ModelConvertor.saveEntityToModel(pkgVersionEntity, model);
       }
@@ -161,14 +182,14 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageVersion(packageId: string, version: string): Promise<PackageVersionEntity | null> {
-    const pkgVersionModel = await PackageVersionModel.findOne({ packageId, version });
+    const pkgVersionModel = await this.PackageVersion.findOne({ packageId, version });
     if (!pkgVersionModel) return null;
     return await this.fillPackageVersionEntitiyData(pkgVersionModel);
   }
 
   async listPackageVersions(packageId: string): Promise<PackageVersionEntity[]> {
     // FIXME: read all versions will hit the memory limit
-    const models = await PackageVersionModel.find({ packageId }).order('id desc');
+    const models = await this.PackageVersion.find({ packageId }).order('id desc');
     const entities: PackageVersionEntity[] = [];
     for (const model of models) {
       entities.push(await this.fillPackageVersionEntitiyData(model));
@@ -177,19 +198,19 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageVersionNames(packageId: string): Promise<string[]> {
-    const rows = await PackageVersionModel.find({ packageId }).select('version').order('id desc');
+    const rows = await this.PackageVersion.find({ packageId }).select('version').order('id desc');
     return rows.map(row => row.version);
   }
 
   // only for unittest now
   async removePackageVersions(packageId: string): Promise<void> {
-    const removeCount = await PackageVersionModel.remove({ packageId });
+    const removeCount = await this.PackageVersion.remove({ packageId });
     this.logger.info('[PackageRepository:removePackageVersions:remove] %d rows, packageId: %s',
       removeCount, packageId);
   }
 
   async removePackageVersion(pkgVersion: PackageVersionEntity): Promise<void> {
-    const distRemoveCount = await DistModel.remove({
+    const distRemoveCount = await this.Dist.remove({
       distId: [
         pkgVersion.abbreviatedDist.distId,
         pkgVersion.manifestDist.distId,
@@ -197,32 +218,32 @@ export class PackageRepository extends AbstractRepository {
         pkgVersion.tarDist.distId,
       ],
     });
-    const removeCount = await PackageVersionModel.remove({ packageVersionId: pkgVersion.packageVersionId });
+    const removeCount = await this.PackageVersion.remove({ packageVersionId: pkgVersion.packageVersionId });
     this.logger.info('[PackageRepository:removePackageVersion:remove] %d dist rows, %d rows, packageVersionId: %s',
       distRemoveCount, removeCount, pkgVersion.packageVersionId);
   }
 
   async savePackageVersionManifest(manifestEntity: PackageVersionManifestEntity): Promise<void> {
-    let model = await PackageVersionManifestModel.findOne({ packageVersionId: manifestEntity.packageVersionId });
+    let model = await this.PackageVersionManifest.findOne({ packageVersionId: manifestEntity.packageVersionId });
     if (model) {
       model.manifest = manifestEntity.manifest;
       await model.save();
     } else {
-      model = await ModelConvertor.convertEntityToModel(manifestEntity, PackageVersionManifestModel);
+      model = await ModelConvertor.convertEntityToModel(manifestEntity, this.PackageVersionManifest);
       this.logger.info('[PackageRepository:savePackageVersionManifest:new] id: %s, packageVersionId: %s',
         model.id, model.packageVersionId);
     }
   }
 
   async findPackageVersionManifest(packageVersionId: string) {
-    const model = await PackageVersionManifestModel.findOne({ packageVersionId });
+    const model = await this.PackageVersionManifest.findOne({ packageVersionId });
     if (!model) return null;
-    return ModelConvertor.convertModelToEntity(model, PackageVersionManifestModel);
+    return ModelConvertor.convertModelToEntity(model, this.PackageVersionManifest);
   }
 
   public async queryTotal() {
-    const lastPkg = await PackageModel.findOne().order('id', 'desc');
-    const lastVersion = await PackageVersionModel.findOne().order('id', 'desc');
+    const lastPkg = await this.Package.findOne().order('id', 'desc');
+    const lastVersion = await this.PackageVersion.findOne().order('id', 'desc');
     let packageCount = 0;
     let packageVersionCount = 0;
     let lastPackage = '';
@@ -235,7 +256,7 @@ export class PackageRepository extends AbstractRepository {
     }
 
     if (lastVersion) {
-      const pkg = await PackageModel.findOne({ packageId: lastVersion.packageId });
+      const pkg = await this.Package.findOne({ packageId: lastVersion.packageId });
       if (pkg) {
         const fullname = pkg.scope ? `${pkg.scope}/${pkg.name}` : pkg.name;
         lastPackageVersion = `${fullname}@${lastVersion.version}`;
@@ -257,10 +278,10 @@ export class PackageRepository extends AbstractRepository {
       manifestDistModel,
       abbreviatedDistModel,
     ] = await Promise.all([
-      DistModel.findOne({ distId: model.tarDistId }),
-      DistModel.findOne({ distId: model.readmeDistId }),
-      DistModel.findOne({ distId: model.manifestDistId }),
-      DistModel.findOne({ distId: model.abbreviatedDistId }),
+      this.Dist.findOne({ distId: model.tarDistId }),
+      this.Dist.findOne({ distId: model.readmeDistId }),
+      this.Dist.findOne({ distId: model.manifestDistId }),
+      this.Dist.findOne({ distId: model.abbreviatedDistId }),
     ]);
     const data = {
       tarDist: tarDistModel && ModelConvertor.convertModelToEntity(tarDistModel, DistEntity),
@@ -272,7 +293,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageTag(packageId: string, tag: string): Promise<PackageTagEntity | null> {
-    const model = await PackageTagModel.findOne({ packageId, tag });
+    const model = await this.PackageTag.findOne({ packageId, tag });
     if (!model) return null;
     const entity = ModelConvertor.convertModelToEntity(model, PackageTagEntity);
     return entity;
@@ -280,18 +301,18 @@ export class PackageRepository extends AbstractRepository {
 
   async savePackageTag(packageTagEntity: PackageTagEntity) {
     if (packageTagEntity.id) {
-      const model = await PackageTagModel.findOne({ id: packageTagEntity.id });
+      const model = await this.PackageTag.findOne({ id: packageTagEntity.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(packageTagEntity, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(packageTagEntity, PackageTagModel);
+      const model = await ModelConvertor.convertEntityToModel(packageTagEntity, this.PackageTag);
       this.logger.info('[PackageRepository:savePackageTag:new] id: %s, packageTagId: %s, tags: %s => %s',
         model.id, model.packageTagId, model.tag, model.version);
     }
   }
 
   async removePackageTag(packageTagEntity: PackageTagEntity) {
-    const model = await PackageTagModel.findOne({ id: packageTagEntity.id });
+    const model = await this.PackageTag.findOne({ id: packageTagEntity.id });
     if (!model) return;
     await model.remove();
     this.logger.info('[PackageRepository:removePackageTag:remove] id: %s, packageTagId: %s, packageId: %s',
@@ -299,7 +320,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageTags(packageId: string): Promise<PackageTagEntity[]> {
-    const models = await PackageTagModel.find({ packageId });
+    const models = await this.PackageTag.find({ packageId });
     const entities: PackageTagEntity[] = [];
     for (const model of models) {
       entities.push(ModelConvertor.convertModelToEntity(model, PackageTagEntity));

app/repository/PackageVersionBlockRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { PackageVersionBlock as PackageVersionBlockModel } from './model/PackageVersionBlock';
+import type { PackageVersionBlock as PackageVersionBlockModel } from './model/PackageVersionBlock';
 import { PackageVersionBlock as PackageVersionBlockEntity } from '../core/entity/PackageVersionBlock';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,13 +8,16 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionBlockRepository extends AbstractRepository {
+  @Inject()
+  private readonly PackageVersionBlock: typeof PackageVersionBlockModel;
+
   async savePackageVersionBlock(block: PackageVersionBlockEntity) {
     if (block.id) {
-      const model = await PackageVersionBlockModel.findOne({ id: block.id });
+      const model = await this.PackageVersionBlock.findOne({ id: block.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(block, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(block, PackageVersionBlockModel);
+      const model = await ModelConvertor.convertEntityToModel(block, this.PackageVersionBlock);
       this.logger.info('[PackageVersionBlockRepository:savePackageVersionBlock:new] id: %s, packageVersionBlockId: %s',
         model.id, model.packageVersionBlockId);
     }
@@ -25,17 +28,17 @@ export class PackageVersionBlockRepository extends AbstractRepository {
   }
 
   async findPackageVersionBlock(packageId: string, version: string) {
-    const model = await PackageVersionBlockModel.findOne({ packageId, version });
+    const model = await this.PackageVersionBlock.findOne({ packageId, version });
     if (model) return ModelConvertor.convertModelToEntity(model, PackageVersionBlockEntity);
     return null;
   }
 
   async listPackageVersionBlocks(packageId: string) {
-    return await PackageVersionBlockModel.find({ packageId });
+    return await this.PackageVersionBlock.find({ packageId });
   }
 
   async removePackageVersionBlock(packageVersionBlockId: string) {
-    const removeCount = await PackageVersionBlockModel.remove({ packageVersionBlockId });
+    const removeCount = await this.PackageVersionBlock.remove({ packageVersionBlockId });
     this.logger.info('[PackageVersionBlockRepository:removePackageVersionBlock:remove] %d rows, packageVersionBlockId: %s',
       removeCount, packageVersionBlockId);
   }

app/repository/PackageVersionDownloadRepository.ts

@@ -1,17 +1,20 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { AbstractRepository } from './AbstractRepository';
-import { PackageVersionDownload as PackageVersionDownloadModel } from './model/PackageVersionDownload';
+import type { PackageVersionDownload as PackageVersionDownloadModel } from './model/PackageVersionDownload';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionDownloadRepository extends AbstractRepository {
+  @Inject()
+  private readonly PackageVersionDownload: typeof PackageVersionDownloadModel;
+
   async plus(packageId: string, version: string, counter: number): Promise<void> {
     const now = new Date();
     const yearMonth = now.getFullYear() * 100 + now.getMonth() + 1;
     const date = new Date().getDate();
     const field = date < 10 ? `d0${date}` : `d${date}`;
-    let model = await PackageVersionDownloadModel.findOne({
+    let model = await this.PackageVersionDownload.findOne({
       packageId,
       version,
       yearMonth,
@@ -23,11 +26,11 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
         version,
         yearMonth,
       };
-      model = await PackageVersionDownloadModel.create(attributes);
+      model = await this.PackageVersionDownload.create(attributes);
       this.logger.info('[PackageVersionDownloadRepository:plus:new] id: %s, packageId: %s, version: %s, yearMonth: %s',
         model.id, model.packageId, model.version, model.yearMonth);
     }
-    await PackageVersionDownloadModel
+    await this.PackageVersionDownload
       .where({ id: model.id })
       .increment(field, counter);
     this.logger.info('[PackageVersionDownloadRepository:plus:increment] id: %s, packageId: %s, version: %s, field: %s%s, plus: %d',
@@ -37,7 +40,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
   async query(packageId: string, start: Date, end: Date) {
     const startYearMonth = start.getFullYear() * 100 + start.getMonth() + 1;
     const endYearMonth = end.getFullYear() * 100 + end.getMonth() + 1;
-    const models = await PackageVersionDownloadModel.find({
+    const models = await this.PackageVersionDownload.find({
       packageId,
       yearMonth: { $gte: startYearMonth, $lte: endYearMonth },
     });
@@ -46,7 +49,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
 
   async saveSyncDataByMonth(packageId: string, yearMonth: number, counters: [string, number][]): Promise<void> {
     const version = '*';
-    let model = await PackageVersionDownloadModel.findOne({
+    let model = await this.PackageVersionDownload.findOne({
       packageId,
       version,
       yearMonth,
@@ -58,7 +61,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
         version,
         yearMonth,
       };
-      model = await PackageVersionDownloadModel.create(attributes);
+      model = await this.PackageVersionDownload.create(attributes);
     }
     for (const [ date, counter ] of counters) {
       const field = `d${date}`;

app/repository/RegistryRepository.ts

@@ -0,0 +1,59 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { ModelConvertor } from './util/ModelConvertor';
+import { Registry, Registry as RegistryEntity } from '../core/entity/Registry';
+import { AbstractRepository } from './AbstractRepository';
+import type { Registry as RegistryModel } from './model/Registry';
+import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class RegistryRepository extends AbstractRepository {
+  @Inject()
+  private readonly Registry: typeof RegistryModel;
+
+  async listRegistries(page: PageOptions): Promise<PageResult<Registry>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Registry.find().count();
+    const models = await this.Registry.find().offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, RegistryEntity)),
+    };
+  }
+
+  async findRegistry(name?: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ name });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
+  async findRegistryByRegistryId(registryId: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ registryId });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
+  async saveRegistry(registry: Registry) {
+    if (registry.id) {
+      const model = await this.Registry.findOne({ id: registry.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(registry, model);
+      return model;
+    }
+    const model = await ModelConvertor.convertEntityToModel(registry, this.Registry);
+    this.logger.info('[RegistryRepository:saveRegistry:new] id: %s, registryId: %s',
+      model.id, model.registryId);
+    return model;
+
+  }
+
+  async removeRegistry(registryId: string): Promise<void> {
+    await this.Registry.remove({ registryId });
+  }
+
+}

app/repository/ScopeRepository.ts

@@ -0,0 +1,67 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { ModelConvertor } from './util/ModelConvertor';
+import { AbstractRepository } from './AbstractRepository';
+import { Scope as ScopeModel } from './model/Scope';
+import { Scope } from '../core/entity/Scope';
+import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class ScopeRepository extends AbstractRepository {
+  @Inject()
+  private readonly Scope: typeof ScopeModel;
+
+  async countByRegistryId(registryId: string): Promise<number> {
+    return await this.Scope.find({ registryId }).count();
+  }
+  async findByName(name: string): Promise<Scope | null> {
+    const model = await this.Scope.findOne({ name });
+    if (!model) {
+      return null;
+    }
+    return ModelConvertor.convertModelToEntity(model, Scope);
+  }
+  async listScopesByRegistryId(registryId: string, page: PageOptions): Promise<PageResult<Scope>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Scope.find({ registryId }).count();
+    const models = await this.Scope.find({ registryId }).offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, Scope)),
+    };
+  }
+
+  async listScopes(page: PageOptions): Promise<PageResult<Scope>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Scope.find().count();
+    const models = await this.Scope.find().offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, Scope)),
+    };
+  }
+
+  async saveScope(scope: Scope) {
+    if (scope.id) {
+      const model = await this.Scope.findOne({ id: scope.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(scope, model);
+      return model;
+    }
+    const model = await ModelConvertor.convertEntityToModel(scope, this.Scope);
+    this.logger.info('[ScopeRepository:saveScope:new] id: %s, scopeId: %s',
+      model.id, model.scopeId);
+    await model.save();
+    return model;
+  }
+
+  async removeScope(scopeId: string): Promise<void> {
+    await this.Scope.remove({ scopeId });
+  }
+
+  async removeScopeByRegistryId(registryId: string): Promise<void> {
+    await this.Scope.remove({ registryId });
+  }
+
+}

app/repository/TaskRepository.ts

@@ -1,7 +1,7 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Task as TaskModel } from './model/Task';
-import { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
+import type { Task as TaskModel } from './model/Task';
+import type { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
 import { Task as TaskEntity } from '../core/entity/Task';
 import { AbstractRepository } from './AbstractRepository';
 import { TaskType, TaskState } from '../../app/common/enum/Task';
@@ -10,47 +10,84 @@ import { TaskType, TaskState } from '../../app/common/enum/Task';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class TaskRepository extends AbstractRepository {
+  @Inject()
+  private readonly Task: typeof TaskModel;
+
+  @Inject()
+  private readonly HistoryTask: typeof HistoryTaskModel;
+
   async saveTask(task: TaskEntity): Promise<void> {
     if (task.id) {
-      const model = await TaskModel.findOne({ id: task.id });
+      const model = await this.Task.findOne({ id: task.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(task, model);
     } else {
-      await ModelConvertor.convertEntityToModel(task, TaskModel);
+      try {
+        await ModelConvertor.convertEntityToModel(task, this.Task);
+      } catch (e) {
+        e.message = '[TaskRepository] insert Task failed: ' + e.message;
+        if (e.code === 'ER_DUP_ENTRY') {
+          this.logger.warn(e);
+          const taskModel = await this.Task.findOne({ bizId: task.bizId });
+          // 覆盖 bizId 相同的 id 和 taskId
+          if (taskModel) {
+            task.id = taskModel.id;
+            task.taskId = taskModel.taskId;
+            return;
+          }
+          // taskModel 可能不存在，遇到数据错误
+          // 重新将错误抛出。
+          throw e;
+        }
+        throw e;
+      }
     }
   }
 
   async saveTaskToHistory(task: TaskEntity): Promise<void> {
-    const model = await TaskModel.findOne({ id: task.id });
+    const model = await this.Task.findOne({ id: task.id });
     if (!model) return;
-    const history = await HistoryTaskModel.findOne({ taskId: task.taskId });
+    const history = await this.HistoryTask.findOne({ taskId: task.taskId });
     if (history) {
       await ModelConvertor.saveEntityToModel(task, history);
     } else {
-      await ModelConvertor.convertEntityToModel(task, HistoryTaskModel);
+      await ModelConvertor.convertEntityToModel(task, this.HistoryTask);
     }
     await model.remove();
   }
 
   async findTask(taskId: string) {
-    const task = await TaskModel.findOne({ taskId });
+    const task = await this.Task.findOne({ taskId });
     if (task) {
       return ModelConvertor.convertModelToEntity(task, TaskEntity);
     }
     // try to read from history
-    const history = await HistoryTaskModel.findOne({ taskId });
+    const history = await this.HistoryTask.findOne({ taskId });
     if (history) {
       return ModelConvertor.convertModelToEntity(history, TaskEntity);
     }
     return null;
   }
 
+  async findTaskByBizId(bizId: string) {
+    const task = await this.Task.findOne({ bizId });
+    if (task) {
+      return ModelConvertor.convertModelToEntity(task, TaskEntity);
+    }
+    return null;
+  }
+
+  async findTasks(taskIds: Array<string>): Promise<Array<TaskEntity>> {
+    const tasks = await this.HistoryTask.find({ taskId: { $in: taskIds } });
+    return tasks.map(task => ModelConvertor.convertModelToEntity(task, TaskEntity));
+  }
+
   async findTaskByTargetName(targetName: string, type: TaskType, state?: TaskState) {
     const where: any = { targetName, type };
     if (state) {
       where.state = state;
     }
-    const task = await TaskModel.findOne(where);
+    const task = await this.Task.findOne(where);
     if (task) {
       return ModelConvertor.convertModelToEntity(task, TaskEntity);
     }
@@ -60,7 +97,7 @@ export class TaskRepository extends AbstractRepository {
   async findTimeoutTasks(taskState: TaskState, timeout: number) {
     const timeoutDate = new Date();
     timeoutDate.setTime(timeoutDate.getTime() - timeout);
-    const models = await TaskModel.find({
+    const models = await this.Task.find({
       state: taskState,
       updatedAt: {
         $lt: timeoutDate,

app/repository/UserRepository.ts

@@ -1,7 +1,7 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { User as UserModel } from './model/User';
-import { Token as TokenModel } from './model/Token';
+import type { User as UserModel } from './model/User';
+import type { Token as TokenModel } from './model/Token';
 import { User as UserEntity } from '../core/entity/User';
 import { Token as TokenEntity } from '../core/entity/Token';
 import { AbstractRepository } from './AbstractRepository';
@@ -10,19 +10,31 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class UserRepository extends AbstractRepository {
+  @Inject()
+  private readonly User: typeof UserModel;
+
+  @Inject()
+  private readonly Token: typeof TokenModel;
+
   async saveUser(user: UserEntity): Promise<void> {
     if (user.id) {
-      const model = await UserModel.findOne({ id: user.id });
+      const model = await this.User.findOne({ id: user.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(user, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(user, UserModel);
+      const model = await ModelConvertor.convertEntityToModel(user, this.User);
       this.logger.info('[UserRepository:saveUser:new] id: %s, userId: %s', model.id, model.userId);
     }
   }
 
   async findUserByName(name: string) {
-    const model = await UserModel.findOne({ name });
+    const model = await this.User.findOne({ name });
+    if (!model) return null;
+    return ModelConvertor.convertModelToEntity(model, UserEntity);
+  }
+
+  async findUserByUserId(userId: string) {
+    const model = await this.User.findOne({ userId });
     if (!model) return null;
     return ModelConvertor.convertModelToEntity(model, UserEntity);
   }
@@ -30,7 +42,7 @@ export class UserRepository extends AbstractRepository {
   async findUserAndTokenByTokenKey(tokenKey: string) {
     const token = await this.findTokenByTokenKey(tokenKey);
     if (!token) return null;
-    const userModel = await UserModel.findOne({ userId: token.userId });
+    const userModel = await this.User.findOne({ userId: token.userId });
     if (!userModel) return null;
     return {
       token,
@@ -39,30 +51,30 @@ export class UserRepository extends AbstractRepository {
   }
 
   async findTokenByTokenKey(tokenKey: string) {
-    const model = await TokenModel.findOne({ tokenKey });
+    const model = await this.Token.findOne({ tokenKey });
     if (!model) return null;
     return ModelConvertor.convertModelToEntity(model, TokenEntity);
   }
 
   async saveToken(token: TokenEntity): Promise<void> {
     if (token.id) {
-      const model = await TokenModel.findOne({ id: token.id });
+      const model = await this.Token.findOne({ id: token.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(token, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(token, TokenModel);
+      const model = await ModelConvertor.convertEntityToModel(token, this.Token);
       this.logger.info('[UserRepository:saveToken:new] id: %s, tokenId: %s', model.id, model.tokenId);
     }
   }
 
   async removeToken(tokenId: string) {
-    const removeCount = await TokenModel.remove({ tokenId });
+    const removeCount = await this.Token.remove({ tokenId });
     this.logger.info('[UserRepository:removeToken:remove] %d rows, tokenId: %s',
       removeCount, tokenId);
   }
 
   async listTokens(userId: string): Promise<TokenEntity[]> {
-    const models = await TokenModel.find({ userId });
+    const models = await this.Token.find({ userId });
     return models.map(model => ModelConvertor.convertModelToEntity(model, TokenEntity));
   }
 }

app/repository/model/Hook.ts

@@ -0,0 +1,47 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { DataTypes, Bone } from 'leoric';
+import { HookType } from '../../common/enum/Hook';
+
+@Model()
+export class Hook extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(24), {
+    unique: true,
+  })
+  hookId: string;
+
+  @Attribute(DataTypes.STRING(20))
+  type: HookType;
+
+  @Attribute(DataTypes.STRING(24))
+  ownerId: string;
+
+  @Attribute(DataTypes.STRING(428))
+  name: string;
+
+  @Attribute(DataTypes.STRING(2048))
+  endpoint: string;
+
+  @Attribute(DataTypes.STRING(200))
+  secret: string;
+
+  @Attribute(DataTypes.STRING(24), {
+    allowNull: true,
+  })
+  latestTaskId: string;
+
+  @Attribute(DataTypes.BOOLEAN)
+  enable: boolean;
+}

app/repository/model/Package.ts

@@ -21,6 +21,9 @@ export class Package extends Bone {
   })
   packageId: string;
 
+  @Attribute(DataTypes.STRING(24))
+  registryId: string;
+
   @Attribute(DataTypes.STRING(214))
   scope: string;
 

app/repository/model/Registry.ts

@@ -0,0 +1,39 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { RegistryType } from '../../common/enum/Registry';
+import { DataTypes, Bone } from 'leoric';
+
+@Model()
+export class Registry extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(24), {
+    unique: true,
+  })
+  registryId: string;
+
+  @Attribute(DataTypes.STRING(256))
+  name: string;
+
+  @Attribute(DataTypes.STRING(4096))
+  host: string;
+
+  @Attribute(DataTypes.STRING(4096), { name: 'change_stream' })
+  changeStream: string;
+
+  @Attribute(DataTypes.STRING(4096), { name: 'user_prefix' })
+  userPrefix: string;
+
+  @Attribute(DataTypes.STRING(256))
+  type: RegistryType;
+
+}

app/repository/model/Scope.ts

@@ -0,0 +1,26 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { DataTypes, Bone } from 'leoric';
+
+@Model()
+export class Scope extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(214))
+  name: string;
+
+  @Attribute(DataTypes.STRING(256))
+  registryId: string;
+
+  @Attribute(DataTypes.STRING(256))
+  scopeId: string;
+}

app/repository/model/Task.ts

@@ -50,4 +50,9 @@ export class Task extends Bone {
 
   @Attribute(DataTypes.TEXT('long'))
   error: string;
+
+  @Attribute(DataTypes.STRING(48), {
+    unique: true,
+  })
+  bizId: string;
 }

app/repository/util/ModelConvertor.ts

@@ -10,7 +10,7 @@ const ID = 'id';
 
 export class ModelConvertor {
   static async convertEntityToModel<T extends Bone>(entity: object, ModelClazz: EggProtoImplClass<T>, options?): Promise<T> {
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }
@@ -35,7 +35,7 @@ export class ModelConvertor {
   // Find out which attributes changed and set `updatedAt` to now
   static async saveEntityToModel<T extends Bone>(entity: object, model: T, options?): Promise<boolean> {
     const ModelClazz = model.constructor as EggProtoImplClass<T>;
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }
@@ -57,7 +57,7 @@ export class ModelConvertor {
   static convertModelToEntity<T>(bone: Bone, entityClazz: EggProtoImplClass<T>, data?: object): T {
     data = data || {};
     const ModelClazz = bone.constructor;
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }

app/repository/util/ModelConvertorUtil.ts

@@ -12,7 +12,7 @@ export class ModelConvertorUtil {
    * If has no entity property info, use modelProperty as default value
    */
   static getEntityPropertyName(clazz: EggProtoImplClass, modelProperty: string): string {
-    const propertyMap: Map<string, string> | undefined = MetadataUtil.getOwnMetaData(ENTITY_PROPERTY_MAP_ATTRIBUTE, clazz);
+    const propertyMap: Map<string, string> | undefined = MetadataUtil.getMetaData(ENTITY_PROPERTY_MAP_ATTRIBUTE, clazz);
     return propertyMap?.get(modelProperty) ?? modelProperty;
   }
 }

app/schedule/CreateTriggerHookWorker.ts

@@ -0,0 +1,51 @@
+import { Subscription } from 'egg';
+import { TaskService } from '../core/service/TaskService';
+import { TaskType } from '../common/enum/Task';
+import { CreateHookTask } from '../core/entity/Task';
+import { CreateHookTriggerService } from '../core/service/CreateHookTriggerService';
+
+let executingCount = 0;
+export default class CreateTriggerHookWorker extends Subscription {
+  static get schedule() {
+    return {
+      interval: 1000,
+      type: 'all',
+    };
+  }
+
+  async subscribe() {
+    const { ctx, app } = this;
+    if (!app.config.cnpmcore.hookEnable) return;
+    if (executingCount >= app.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks) return;
+
+    await ctx.beginModuleScope(async () => {
+      const createHookTriggerService = await ctx.getEggObject(CreateHookTriggerService);
+      const taskService = await ctx.getEggObject(TaskService);
+      executingCount++;
+      try {
+        let task = await taskService.findExecuteTask(TaskType.CreateHook) as CreateHookTask;
+        while (task) {
+          const startTime = Date.now();
+          ctx.logger.info('[CreateTriggerHookWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+            executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+            startTime - task.updatedAt.getTime());
+          await createHookTriggerService.executeTask(task);
+          const use = Date.now() - startTime;
+          ctx.logger.info('[CreateTriggerHookWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
+            executingCount, task.taskId, task.targetName, use);
+          if (executingCount >= app.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks) {
+            ctx.logger.info('[CreateTriggerHookWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+              executingCount, app.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks);
+            break;
+          }
+          // try next task
+          task = await taskService.findExecuteTask(TaskType.CreateHook) as CreateHookTask;
+        }
+      } catch (err) {
+        ctx.logger.error('[TriggerHookWorker:subscribe:executeTask:error][%s] %s', executingCount, err);
+      } finally {
+        executingCount--;
+      }
+    });
+  }
+}

app/schedule/SyncPackageWorker.ts

@@ -31,6 +31,11 @@ export default class SyncPackageWorker extends Subscription {
           const use = Date.now() - startTime;
           ctx.logger.info('[SyncPackageWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
             executingCount, task.taskId, task.targetName, use);
+          if (executingCount >= app.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) {
+            ctx.logger.info('[SyncPackageWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+              executingCount, app.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks);
+            break;
+          }
           // try next task
           task = await packageSyncerService.findExecuteTask();
         }

app/schedule/TriggerHookWorker.ts

@@ -0,0 +1,50 @@
+import { Subscription } from 'egg';
+import { HookTriggerService } from '../core/service/HookTriggerService';
+import { TaskService } from '../core/service/TaskService';
+import { TaskType } from '../common/enum/Task';
+import { TriggerHookTask } from '../core/entity/Task';
+
+let executingCount = 0;
+export default class TriggerHookWorker extends Subscription {
+  static get schedule() {
+    return {
+      interval: 1000,
+      type: 'all',
+    };
+  }
+
+  async subscribe() {
+    const { ctx, app } = this;
+    if (executingCount >= app.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks) return;
+
+    await ctx.beginModuleScope(async () => {
+      const hookTriggerService = await ctx.getEggObject(HookTriggerService);
+      const taskService = await ctx.getEggObject(TaskService);
+      executingCount++;
+      try {
+        let task = await taskService.findExecuteTask(TaskType.TriggerHook) as TriggerHookTask;
+        while (task) {
+          const startTime = Date.now();
+          ctx.logger.info('[TriggerHookWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+            executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+            startTime - task.updatedAt.getTime());
+          await hookTriggerService.executeTask(task);
+          const use = Date.now() - startTime;
+          ctx.logger.info('[TriggerHookWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
+            executingCount, task.taskId, task.targetName, use);
+          if (executingCount >= app.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks) {
+            ctx.logger.info('[TriggerHookWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+              executingCount, app.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks);
+            break;
+          }
+          // try next task
+          task = await taskService.findExecuteTask(TaskType.TriggerHook) as TriggerHookTask;
+        }
+      } catch (err) {
+        ctx.logger.error('[TriggerHookWorker:subscribe:executeTask:error][%s] %s', executingCount, err);
+      } finally {
+        executingCount--;
+      }
+    });
+  }
+}

config/binaries.ts

@@ -145,6 +145,13 @@ const binaries: {
       requiredNapiVersions: true,
     },
   },
+  wrtc: {
+    category: 'wrtc',
+    description: 'node-webrtc is a Node.js Native Addon that provides bindings to WebRTC M87.',
+    syncer: SyncerClass.NodePreGypBinary,
+    repo: 'node-webrtc/node-webrtc',
+    distUrl: 'https://node-webrtc.s3.amazonaws.com',
+  },
   nodegit: {
     category: 'nodegit',
     description: 'Native Node bindings to Git.',

config/config.default.ts

@@ -9,6 +9,7 @@ export default (appInfo: EggAppConfig) => {
 
   config.cnpmcore = {
     name: 'cnpm',
+    hooksLimit: 20,
     sourceRegistry: 'https://registry.npmjs.org',
     // upstream registry is base on `cnpmcore` or not
     // if your upstream is official npm registry, please turn it off
@@ -21,7 +22,10 @@ export default (appInfo: EggAppConfig) => {
     //  - none: don't sync npm package, just redirect it to sourceRegistry
     //  - all: sync all npm packages
     syncMode: 'none',
+    hookEnable: false,
     syncPackageWorkerMaxConcurrentTasks: 10,
+    triggerHookWorkerMaxConcurrentTasks: 10,
+    createTriggerHookWorkerMaxConcurrentTasks: 10,
     // stop syncing these packages in future
     syncPackageBlockList: [],
     // check recently from https://www.npmjs.com/browse/updated, if use set changesStreamRegistry to cnpmcore,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmcore",
-  "version": "1.6.0",
+  "version": "1.11.0",
   "description": "npm core",
   "files": [
     "dist/**/*"
@@ -70,14 +70,14 @@
     "npm"
   ],
   "dependencies": {
-    "@eggjs/tegg": "^1.2.0",
-    "@eggjs/tegg-aop-plugin": "^1.0.5",
+    "@eggjs/tegg": "^1.3.2",
+    "@eggjs/tegg-aop-plugin": "^1.3.2",
     "@eggjs/tegg-config": "^1.0.0",
-    "@eggjs/tegg-controller-plugin": "^1.0.5",
-    "@eggjs/tegg-eventbus-plugin": "^1.0.5",
-    "@eggjs/tegg-orm-decorator": "^1.0.1",
-    "@eggjs/tegg-orm-plugin": "^1.0.5",
-    "@eggjs/tegg-plugin": "^1.0.5",
+    "@eggjs/tegg-controller-plugin": "^1.3.2",
+    "@eggjs/tegg-eventbus-plugin": "^1.3.2",
+    "@eggjs/tegg-orm-decorator": "^1.4.0",
+    "@eggjs/tegg-orm-plugin": "^2.1.0",
+    "@eggjs/tegg-plugin": "^1.3.2",
     "@eggjs/tsconfig": "^1.0.0",
     "@node-rs/crc32": "^1.2.2",
     "@sinclair/typebox": "^0.23.0",
@@ -93,7 +93,7 @@
     "egg-typebox-validate": "^2.0.0",
     "fs-cnpm": "^2.4.0",
     "ioredis": "^4.28.3",
-    "leoric": "^1.15.0",
+    "leoric": "^2.6.2",
     "lodash": "^4.17.21",
     "mysql": "^2.18.1",
     "mysql2": "^2.3.0",

sql/1.11.0.sql

@@ -0,0 +1,45 @@
+ALTER TABLE `tasks` ADD COLUMN `biz_id` varchar(100) NULL COMMENT 'unique biz id to keep task unique';
+ALTER TABLE `tasks` ADD UNIQUE KEY `uk_biz_id` (`biz_id`);
+ALTER TABLE `packages` ADD COLUMN `registry_id` varchar(24) NULL COMMENT 'source registry';
+
+CREATE TABLE IF NOT EXISTS `hooks` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime(3) NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time',
+  `hook_id` varchar(24) NOT NULL COMMENT 'hook id',
+  `type` varchar(20) NOT NULL COMMENT 'hook type, scope, name, owner',
+  `name` varchar(428) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'hook name',
+  `owner_id` varchar(24) NOT NULL COMMENT 'hook owner id',
+  `endpoint` varchar(2048) NOT NULL COMMENT 'hook url',
+  `secret` varchar(200) NOT NULL COMMENT 'sign secret',
+  `latest_task_id` varchar(24) NULL COMMENT 'latest task id',
+  `enable` tinyint NOT NULL DEFAULT 0 COMMENT 'hook is enable not, 1: true, other: false',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_type_name_owner_id` (`type`, `name`, `owner_id`),
+  KEY `idx_type_name_id` (`type`, `name`, `id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='task info';
+
+CREATE TABLE IF NOT EXISTS `registries` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime(3) NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time',
+  `registry_id` varchar(24) NOT NULL COMMENT 'registry id',
+  `name` varchar(256) DEFAULT NULL COMMENT 'registry name',
+  `host` varchar(4096) DEFAULT NULL COMMENT 'registry host',
+  `change_stream` varchar(4096) DEFAULT NULL COMMENT 'change stream url',
+  `type` varchar(256) DEFAULT NULL COMMENT 'registry type cnpmjsorg/cnpmcore/npm ',
+  `user_prefix` varchar(256) DEFAULT NULL COMMENT 'user prefix',
+  UNIQUE KEY `uk_name` (`name`),
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='registry info';
+
+CREATE TABLE IF NOT EXISTS `scopes` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime(3) NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time',
+  `scope_id` varchar(24) NOT NULL COMMENT 'scope id',
+  `name` varchar(214) DEFAULT NULL COMMENT 'scope name',
+  `registry_id` varchar(24) NOT NULL COMMENT 'registry id',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='scope info';

test/TestUtil.ts

@@ -6,6 +6,7 @@ import mysql from 'mysql';
 import path from 'path';
 import crypto from 'crypto';
 import { getScopeAndName } from '../app/common/PackageUtil';
+import semver from 'semver';
 
 type PackageOptions = {
   name?: string;
@@ -19,6 +20,7 @@ type PackageOptions = {
   isPrivate?: boolean;
   libc?: string[];
   description?: string;
+  registryId?: string;
 };
 
 type UserOptions = {
@@ -53,7 +55,13 @@ export class TestUtil {
   }
 
   static async getTableSqls(): Promise<string> {
-    return await fs.readFile(path.join(__dirname, '../sql/init.sql'), 'utf8');
+    const dirents = await fs.readdir(path.join(__dirname, '../sql'));
+    let versions = dirents.filter(t => path.extname(t) === '.sql').map(t => path.basename(t, '.sql'));
+    versions = semver.sort(versions);
+    const sqls = await Promise.all(versions.map(version => {
+      return fs.readFile(path.join(__dirname, '../sql', `${version}.sql`), 'utf8');
+    }));
+    return sqls.join('\n');
   }
 
   static async query(sql): Promise<any[]> {
@@ -207,10 +215,11 @@ export class TestUtil {
       .set('user-agent', user.ua)
       .send(pkg)
       .expect(201);
+
     if (options?.isPrivate === false) {
       const [ scope, name ] = getScopeAndName(pkg.name);
       const { Package: PackageModel } = require('../app/repository/model/Package');
-      await PackageModel.update({ scope, name }, { isPrivate: false });
+      await PackageModel.update({ scope, name }, { isPrivate: false, registryId: options?.registryId });
     }
     return { user, pkg };
   }
@@ -281,6 +290,20 @@ export class TestUtil {
       name: adminName,
     });
   }
+  static async createRegistryAndScope() {
+    // create success
+    const adminUser = await this.createAdmin();
+    await this.app.httpRequest()
+      .post('/-/registry')
+      .set('authorization', adminUser.authorization)
+      .send(
+        {
+          name: 'custom6',
+          host: 'https://r.cnpmjs.org/',
+          changeStream: 'https://r.cnpmjs.org/_changes',
+          type: 'cnpmcore',
+        });
+  }
 
   static async readStreamToLog(urlOrStream) {
     let stream: Readable;

test/common/adapter/NpmRegistry.test.ts

@@ -0,0 +1,25 @@
+import assert = require('assert');
+import { app, mock } from 'egg-mock/bootstrap';
+import { NPMRegistry } from '../../../app/common/adapter/NPMRegistry';
+
+describe('test/common/adapter/CacheAdapter.test.ts', () => {
+  let npmRegistry: NPMRegistry;
+
+  beforeEach(async () => {
+    const ctx = await app.mockModuleContext();
+    npmRegistry = await ctx.getEggObject(NPMRegistry);
+    mock(app.config.cnpmcore, 'registry', 'https://registry.npmjs.org');
+  });
+
+  describe('setRegistryHost()', () => {
+    it('default registry', async () => {
+      assert(npmRegistry.registry === 'https://registry.npmjs.org');
+    });
+    it('should work', async () => {
+      assert(npmRegistry.registry);
+      const host = 'https://registry.npmmirror.com';
+      npmRegistry.setRegistryHost(host);
+      assert(npmRegistry.registry === 'https://registry.npmmirror.com');
+    });
+  });
+});

test/common/adapter/binary/NodePreGypBinary.test.ts

@@ -177,5 +177,54 @@ describe('test/common/adapter/binary/NodePreGypBinary.test.ts', () => {
       assert(matchFile2);
       assert(matchFile3);
     });
+
+    it('should fetch wrtc', async () => {
+      const binary = new NodePreGypBinary(ctx.httpclient, ctx.logger, binaries.wrtc);
+      let result = await binary.fetch('/');
+      assert(result);
+      assert(result.items.length > 0);
+      // console.log(JSON.stringify(result.items, null, 2));
+      let matchDir = false;
+      for (const item of result.items) {
+        assert(item.isDir === true);
+        if (item.name === 'v0.4.7/') {
+          matchDir = true;
+        }
+      }
+      assert(matchDir);
+
+      result = await binary.fetch('/v0.4.7/');
+      assert(result);
+      assert(result.items.length > 0);
+      // console.log(JSON.stringify(result.items, null, 2));
+      let matchFile1 = false;
+      let matchFile2 = false;
+      let matchFile3 = false;
+      for (const item of result.items) {
+        assert(item.isDir === false);
+        assert.deepEqual(item.ignoreDownloadStatuses, [ 404 ]);
+        if (item.name === 'linux-arm64.tar.gz') {
+          assert(item.date === '2021-01-10T15:43:35.384Z');
+          assert(item.size === '-');
+          assert(item.url === 'https://node-webrtc.s3.amazonaws.com/wrtc/v0.4.7/Release/linux-arm64.tar.gz');
+          matchFile1 = true;
+        }
+        if (item.name === 'linux-x64.tar.gz') {
+          assert(item.date === '2021-01-10T15:43:35.384Z');
+          assert(item.size === '-');
+          assert(item.url === 'https://node-webrtc.s3.amazonaws.com/wrtc/v0.4.7/Release/linux-x64.tar.gz');
+          matchFile2 = true;
+        }
+        if (item.name === 'darwin-x64.tar.gz') {
+          assert(item.date === '2021-01-10T15:43:35.384Z');
+          assert(item.size === '-');
+          assert(item.url === 'https://node-webrtc.s3.amazonaws.com/wrtc/v0.4.7/Release/darwin-x64.tar.gz');
+          matchFile3 = true;
+        }
+      }
+      assert(matchFile1);
+      assert(matchFile2);
+      assert(matchFile3);
+    });
   });
 });

test/common/adapter/changesStream/CnpmcoreChangesStream.test.ts

@@ -0,0 +1,84 @@
+import { ChangesStreamChange } from 'app/common/adapter/changesStream/AbstractChangesStream';
+import { CnpmcoreChangesStream } from 'app/common/adapter/changesStream/CnpmcoreChangesStream';
+import { RegistryType } from 'app/common/enum/Registry';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import assert = require('assert');
+import { Context } from 'egg';
+import { app } from 'egg-mock/bootstrap';
+
+describe('test/common/adapter/changesStream/CnpmcoreChangesStream.test.ts', () => {
+  let ctx: Context;
+  let cnpmcoreChangesStream: CnpmcoreChangesStream;
+  let registryManagerService: RegistryManagerService;
+  let registry: Registry;
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    cnpmcoreChangesStream = await ctx.getEggObject(CnpmcoreChangesStream);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    registry = await registryManagerService.createRegistry({
+      name: 'cnpmcore',
+      changeStream: 'https://r.cnpmjs.org/_changes',
+      host: 'https://registry.npmmirror.com',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmcore,
+    });
+  });
+
+  describe('getInitialSince()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/r\.cnpmjs\.org/, {
+        status: 200,
+        data: {
+          update_seq: 9527,
+        },
+      });
+      const since = await cnpmcoreChangesStream.getInitialSince(registry);
+      assert(since === '9517');
+    });
+
+    it('should throw error', async () => {
+      app.mockHttpclient(/https:\/\/r\.cnpmjs\.org/, () => {
+        throw new Error('mock request replicate _changes error');
+      });
+      await assert.rejects(cnpmcoreChangesStream.getInitialSince(registry), /mock request/);
+    });
+
+    it('should throw error invalid seq', async () => {
+      app.mockHttpclient(/https:\/\/r\.cnpmjs\.org/, { data: { update_seqs: 'invalid' } });
+      await assert.rejects(cnpmcoreChangesStream.getInitialSince(registry), /get getInitialSince failed/);
+    });
+  });
+
+  describe('fetchChanges()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/r\.cnpmjs\.org/, {
+        status: 200,
+        data: {
+          results: [
+            {
+              seq: 1,
+              type: 'PACKAGE_VERSION_ADDED',
+              id: 'create-react-component-helper',
+              changes: [{ version: '1.0.2' }],
+            },
+            {
+              seq: 2,
+              type: 'PACKAGE_VERSION_ADDED',
+              id: 'yj-binaryxml',
+              changes: [{ version: '1.0.0-arisa0' }],
+            },
+          ],
+        },
+      });
+      const stream = await cnpmcoreChangesStream.fetchChanges(registry, '1');
+      const res: ChangesStreamChange[] = [];
+
+      for await (const change of stream) {
+        res.push(change as ChangesStreamChange);
+      }
+
+      assert(res.length === 1);
+    });
+  });
+});

test/common/adapter/changesStream/CnpmjsorgChangesStream.test.ts

@@ -0,0 +1,89 @@
+import { ChangesStreamChange } from 'app/common/adapter/changesStream/AbstractChangesStream';
+import { CnpmjsorgChangesStream } from 'app/common/adapter/changesStream/CnpmjsorgChangesStream';
+import { RegistryType } from 'app/common/enum/Registry';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import assert = require('assert');
+import { Context } from 'egg';
+import { app } from 'egg-mock/bootstrap';
+
+describe('test/common/adapter/changesStream/CnpmjsorgChangesStream.test.ts', () => {
+  let ctx: Context;
+  let cnpmjsorgChangesStream: CnpmjsorgChangesStream;
+  let registryManagerService: RegistryManagerService;
+  let registry: Registry;
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    cnpmjsorgChangesStream = await ctx.getEggObject(CnpmjsorgChangesStream);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    registry = await registryManagerService.createRegistry({
+      name: 'cnpmcore',
+      changeStream: 'https://r2.cnpmjs.org/_changes',
+      host: 'https://registry.npmmirror.com',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmjsorg,
+    });
+  });
+
+  describe('getInitialSince()', () => {
+    it('should work', async () => {
+      const since = await cnpmjsorgChangesStream.getInitialSince(registry);
+      const now = new Date().getTime();
+      assert(now - Number(since) < 10000);
+    });
+
+  });
+
+  describe('fetchChanges()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/r2\.cnpmjs\.org/, {
+        status: 200,
+        data: {
+          results: [
+            {
+              type: 'PACKAGE_VERSION_ADDED',
+              id: 'abc-cli',
+              changes: [{ version: '0.0.1' }],
+              gmt_modified: '2014-01-14T19:35:09.000Z',
+            },
+            {
+              type: 'PACKAGE_TAG_ADDED',
+              id: 'abc-cli',
+              changes: [{ tag: 'latest' }],
+              gmt_modified: '2014-01-15T19:35:09.000Z',
+            },
+          ],
+        },
+      });
+      const stream = await cnpmjsorgChangesStream.fetchChanges(registry, '1');
+      const res: ChangesStreamChange[] = [];
+      for await (const change of stream) {
+        res.push(change as ChangesStreamChange);
+      }
+      assert(res.length === 2);
+    });
+
+    it('should reject when limit', async () => {
+      app.mockHttpclient(/https:\/\/r2\.cnpmjs\.org/, {
+        status: 200,
+        data: {
+          results: [
+            {
+              type: 'PACKAGE_VERSION_ADDED',
+              id: 'abc-cli',
+              changes: [{ version: '0.0.1' }],
+              gmt_modified: '2014-01-14T19:35:09.000Z',
+            },
+            {
+              type: 'PACKAGE_TAG_ADDED',
+              id: 'abc-cli',
+              changes: [{ tag: 'latest' }],
+              gmt_modified: '2014-01-14T19:35:09.000Z',
+            },
+          ],
+        },
+      });
+      await assert.rejects(cnpmjsorgChangesStream.fetchChanges(registry, '1'), /limit too large/);
+    });
+  });
+});

test/common/adapter/changesStream/NpmChangesStream.test.ts

@@ -0,0 +1,92 @@
+import { Readable, Duplex } from 'node:stream';
+import { ChangesStreamChange } from 'app/common/adapter/changesStream/AbstractChangesStream';
+import { NpmChangesStream } from 'app/common/adapter/changesStream/NpmChangesStream';
+import { RegistryType } from 'app/common/enum/Registry';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import assert = require('assert');
+import { Context } from 'egg';
+import { app, mock } from 'egg-mock/bootstrap';
+
+describe('test/common/adapter/changesStream/NpmChangesStream.test.ts', () => {
+  let ctx: Context;
+  let npmChangesStream: NpmChangesStream;
+  let registryManagerService: RegistryManagerService;
+  let registry: Registry;
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    npmChangesStream = await ctx.getEggObject(NpmChangesStream);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    registry = await registryManagerService.createRegistry({
+      name: 'npm',
+      changeStream: 'https://replicate.npmjs.com/_changes',
+      host: 'https://regsitry.npmjs.org',
+      userPrefix: 'npm:',
+      type: RegistryType.Npm,
+    });
+  });
+
+  describe('getInitialSince()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/replicate\.npmjs\.com/, {
+        status: 200,
+        data: {
+          update_seq: 9527,
+        },
+      });
+      const since = await npmChangesStream.getInitialSince(registry);
+      assert(since === '9517');
+    });
+
+    it('should throw error', async () => {
+      app.mockHttpclient(/https:\/\/replicate\.npmjs\.com/, () => {
+        throw new Error('mock request replicate _changes error');
+      });
+      await assert.rejects(npmChangesStream.getInitialSince(registry), /mock request/);
+    });
+
+    it('should throw error invalid seq', async () => {
+      app.mockHttpclient(/https:\/\/replicate\.npmjs\.com/, { data: { update_seqs: 'invalid' } });
+      await assert.rejects(npmChangesStream.getInitialSince(registry), /get getInitialSince failed/);
+    });
+  });
+
+  describe('fetchChanges()', () => {
+    it('should work', async () => {
+      mock(ctx.httpclient, 'request', async () => {
+        return {
+          res: Readable.from(`
+          {"seq":2,"id":"backbone.websql.deferred","changes":[{"rev":"4-f5150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+          {"seq":3,"id":"backbone2.websql.deferred","changes":[{"rev":"4-f6150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+          `),
+        };
+      });
+      const res: ChangesStreamChange[] = [];
+      const stream = await npmChangesStream.fetchChanges(registry, '9517');
+      for await (const change of stream) {
+        res.push(change);
+      }
+      assert(res.length === 2);
+    });
+
+    it('should work for broken chunk', async () => {
+      const rStream = Duplex.from('');
+      mock(ctx.httpclient, 'request', async () => {
+        return {
+          res: rStream,
+        };
+      });
+      const res: ChangesStreamChange[] = [];
+      const stream = await npmChangesStream.fetchChanges(registry, '9517');
+      assert(stream);
+      rStream.push('{"seq":2');
+      rStream.push(',"id":"bac');
+      rStream.push('kbone.websql.deferred","changes":[{"rev":"4-f5150b238ab62cd890211fb57fc9eca5"}],"deleted":true}');
+      for await (const change of stream) {
+        res.push(change);
+      }
+      assert(res.length === 1);
+    });
+  });
+
+});

test/core/service/ChangesStreamService.test.ts

@@ -0,0 +1,168 @@
+import assert = require('assert');
+import { Readable } from 'node:stream';
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { ChangesStreamService } from 'app/core/service/ChangesStreamService';
+import { TaskService } from 'app/core/service/TaskService';
+import { ChangesStreamTask, Task } from 'app/core/entity/Task';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import { RegistryType } from 'app/common/enum/Registry';
+import { ScopeManagerService } from 'app/core/service/ScopeManagerService';
+import { Registry } from 'app/core/entity/Registry';
+import { TestUtil } from 'test/TestUtil';
+
+describe('test/core/service/ChangesStreamService.test.ts', () => {
+  let ctx: Context;
+  let changesStreamService: ChangesStreamService;
+  let scopeManagerService: ScopeManagerService;
+  let registryManagerService: RegistryManagerService;
+  let taskService: TaskService;
+  let task: ChangesStreamTask;
+  let npmRegistry: Registry;
+  let cnpmRegistry: Registry;
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    changesStreamService = await ctx.getEggObject(ChangesStreamService);
+    taskService = await ctx.getEggObject(TaskService);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    scopeManagerService = await ctx.getEggObject(ScopeManagerService);
+    assert(changesStreamService);
+    task = Task.createChangesStream('GLOBAL_WORKER', '9527');
+    taskService.createTask(task, false);
+
+    // create default registry
+    await registryManagerService.createRegistry({
+      name: 'npm',
+      changeStream: 'https://replicate.npmjs.com/_changes',
+      host: 'https://regsitry.npmjs.org',
+      userPrefix: 'npm:',
+      type: RegistryType.Npm,
+    });
+
+    // create custom registry
+    await registryManagerService.createRegistry({
+      name: 'cnpm',
+      changeStream: 'https://r.cnpmjs.org',
+      host: 'https://r.npmjs.org',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmcore,
+    });
+
+    const data = (await registryManagerService.listRegistries({})).data;
+    npmRegistry = data[0];
+    cnpmRegistry = data[1];
+
+    // create custom scope
+    await scopeManagerService.createScope({
+      name: '@cnpm',
+      registryId: cnpmRegistry.registryId,
+    });
+  });
+
+  describe('prepareRegistry()', () => {
+    it('should init since', async () => {
+      assert(task.data.since === '9527');
+    });
+    it('should create default registry by config', async () => {
+      await changesStreamService.prepareRegistry(task);
+      let registries = await registryManagerService.listRegistries({});
+      assert(registries.count === 3);
+
+      // only create once
+      await changesStreamService.prepareRegistry(task);
+      registries = await registryManagerService.listRegistries({});
+      assert(registries.count === 3);
+    });
+
+    it('should throw error when invalid registryId', async () => {
+      await changesStreamService.prepareRegistry(task);
+      const registries = await registryManagerService.listRegistries({});
+      assert(registries.count === 3);
+
+      // remove the registry
+      const registryId = task.data.registryId;
+      assert(registryId);
+      await registryManagerService.remove({ registryId });
+
+      await assert.rejects(changesStreamService.prepareRegistry(task), /invalid change stream registry/);
+    });
+  });
+
+  describe('needSync()', () => {
+    it('follow ', async () => {
+      await TestUtil.createPackage({
+        name: '@cnpm/test',
+        isPrivate: false,
+        registryId: npmRegistry.registryId,
+      });
+      const res = await changesStreamService.needSync(npmRegistry, '@cnpm/test');
+      assert(res);
+    });
+
+    it('unscoped package should sync default registry', async () => {
+      const res = await changesStreamService.needSync(npmRegistry, 'banana');
+      assert(res);
+    });
+
+    it('scoped package should sync default registry', async () => {
+      const res = await changesStreamService.needSync(npmRegistry, '@gogogo/banana');
+      assert(res);
+    });
+
+    it('scoped package should sync custom registry', async () => {
+      let res = await changesStreamService.needSync(cnpmRegistry, '@cnpm/banana');
+      assert(res);
+      res = await changesStreamService.needSync(cnpmRegistry, '@dnpmjs/banana');
+      assert(!res);
+    });
+
+    it('unscoped package should not sync custom registry', async () => {
+      const res = await changesStreamService.needSync(cnpmRegistry, 'banana');
+      assert(!res);
+    });
+
+  });
+
+  describe('getInitialSince()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/replicate\.npmjs\.com/, {
+        status: 200,
+        data: {
+          update_seq: 9527,
+        },
+      });
+      const since = await changesStreamService.getInitialSince(task);
+      assert(since === '9517');
+    });
+  });
+
+  describe('getInitialSince()', () => {
+    it('should work', async () => {
+      app.mockHttpclient(/https:\/\/replicate\.npmjs\.com/, {
+        status: 200,
+        data: {
+          update_seq: 9527,
+        },
+      });
+      const since = await changesStreamService.getInitialSince(task);
+      assert(since === '9517');
+    });
+  });
+
+  describe('fetchChanges()', () => {
+    it('should work', async () => {
+      mock(ctx.httpclient, 'request', async () => {
+        return {
+          res: Readable.from(`
+            {"seq":2,"id":"backbone.websql.deferred","changes":[{"rev":"4-f5150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+            {"seq":3,"id":"backbone2.websql.deferred","changes":[{"rev":"4-f6150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+            `),
+        };
+      });
+      const changes = await changesStreamService.executeSync('1', task);
+      assert(changes.taskCount === 2);
+      assert(changes.lastSince === '3');
+    });
+  });
+
+});

test/core/service/CreateHookTriggerService.test.ts

@@ -0,0 +1,116 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { TestUtil } from '../../TestUtil';
+import { HookManageService } from '../../../app/core/service/HookManageService';
+import { HookType } from '../../../app/common/enum/Hook';
+import { UserRepository } from '../../../app/repository/UserRepository';
+import { PACKAGE_VERSION_ADDED } from '../../../app/core/event';
+import { Change } from '../../../app/core/entity/Change';
+import { ChangeRepository } from '../../../app/repository/ChangeRepository';
+import { Task } from '../../../app/core/entity/Task';
+import { HookEvent } from '../../../app/core/entity/HookEvent';
+import { CreateHookTriggerService } from '../../../app/core/service/CreateHookTriggerService';
+import { TaskRepository } from '../../../app/repository/TaskRepository';
+import { Hook } from '../../../app/core/entity/Hook';
+
+describe('test/core/service/CreateHookTriggerService.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let changeRepository: ChangeRepository;
+  let createHookTriggerService: CreateHookTriggerService;
+  let taskRepository: TaskRepository;
+  const pkgName = '@cnpmcore/foo';
+  const username = 'mock_username';
+  let userId: string;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    changeRepository = await ctx.getEggObject(ChangeRepository);
+    createHookTriggerService = await ctx.getEggObject(CreateHookTriggerService);
+    taskRepository = await ctx.getEggObject(TaskRepository);
+    const userRepository = await ctx.getEggObject(UserRepository);
+    await TestUtil.createPackage({
+      name: pkgName,
+    }, {
+      name: username,
+    });
+    const user = await userRepository.findUserByName(username);
+    userId = user!.userId;
+  });
+
+  describe('executeTask', () => {
+    let change: Change;
+    beforeEach(async () => {
+      change = Change.create({
+        type: PACKAGE_VERSION_ADDED,
+        targetName: pkgName,
+        data: {
+          version: '1.0.0',
+        },
+      });
+      await changeRepository.addChange(change);
+    });
+
+    describe('package hook', () => {
+      let hook: Hook;
+      beforeEach(async () => {
+        hook = await hookManageService.createHook({
+          type: HookType.Package,
+          ownerId: userId,
+          name: pkgName,
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      });
+
+      it('should create package hook trigger', async () => {
+        const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(pkgName, change.changeId));
+        await createHookTriggerService.executeTask(task);
+        const pushTask = await taskRepository.findTaskByBizId(`TriggerHook:${change.changeId}:${hook.hookId}`);
+        assert(pushTask);
+      });
+    });
+
+    describe('scope hook', () => {
+      let hook: Hook;
+      beforeEach(async () => {
+        hook = await hookManageService.createHook({
+          type: HookType.Scope,
+          ownerId: userId,
+          name: '@cnpmcore',
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      });
+
+      it('should create scope hook trigger', async () => {
+        const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(pkgName, change.changeId));
+        await createHookTriggerService.executeTask(task);
+        const pushTask = await taskRepository.findTaskByBizId(`TriggerHook:${change.changeId}:${hook.hookId}`);
+        assert(pushTask);
+      });
+    });
+
+    describe('owner hook', () => {
+      let hook: Hook;
+      beforeEach(async () => {
+        hook = await hookManageService.createHook({
+          type: HookType.Owner,
+          ownerId: userId,
+          name: username,
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      });
+
+      it('should create scope hook trigger', async () => {
+        const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(pkgName, change.changeId));
+        await createHookTriggerService.executeTask(task);
+        const pushTask = await taskRepository.findTaskByBizId(`TriggerHook:${change.changeId}:${hook.hookId}`);
+        assert(pushTask);
+      });
+    });
+  });
+});

test/core/service/HookManageService/createHook.test.ts

@@ -0,0 +1,52 @@
+import assert = require('assert');
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { HookManageService } from '../../../../app/core/service/HookManageService';
+import { TestUtil } from '../../../TestUtil';
+import { HookType } from '../../../../app/common/enum/Hook';
+
+describe('test/core/service/HookManageService/createHook.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+  });
+
+  afterEach(async () => {
+    await TestUtil.truncateDatabase();
+    await app.destroyModuleContext(ctx);
+    mock.restore();
+  });
+
+  describe('limit exceeded', () => {
+    beforeEach(() => {
+      mock(ctx.app.config.cnpmcore, 'hooksLimit', 0);
+    });
+
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.createHook({
+          type: HookType.Package,
+          ownerId: 'mock_owner_id',
+          name: 'foo_package',
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      }, /hooks limit exceeded/);
+    });
+  });
+
+  it('should work', async () => {
+    const hook = await hookManageService.createHook({
+      type: HookType.Package,
+      ownerId: 'mock_owner_id',
+      name: 'foo_package',
+      endpoint: 'http://foo.com',
+      secret: 'mock_secret',
+    });
+    assert(hook);
+    assert(hook.enable === true);
+  });
+});

test/core/service/HookManageService/deleteHook.test.ts

@@ -0,0 +1,61 @@
+import assert from 'assert';
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { HookManageService } from '../../../../app/core/service/HookManageService';
+import { Hook } from '../../../../app/core/entity/Hook';
+import { TestUtil } from '../../../TestUtil';
+import { HookType } from '../../../../app/common/enum/Hook';
+
+describe('test/core/service/HookManageService/deleteHook.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let hook: Hook;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    hook = await hookManageService.createHook({
+      type: HookType.Package,
+      ownerId: 'mock_owner_id',
+      name: 'foo_package',
+      endpoint: 'http://foo.com',
+      secret: 'mock_secret',
+    });
+  });
+
+  afterEach(async () => {
+    await TestUtil.truncateDatabase();
+    await app.destroyModuleContext(ctx);
+    mock.restore();
+  });
+
+  describe('hook not found', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.deleteHook({
+          hookId: 'not_exist_hook_id',
+          operatorId: 'mock_owner_id',
+        });
+      }, /hook not_exist_hook_id not found/);
+    });
+  });
+
+  describe('hook not belong to operator', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.deleteHook({
+          hookId: hook.hookId,
+          operatorId: 'not_exits_owner_id',
+        });
+      }, new RegExp(`hook ${hook.hookId} not belong to not_exits_owner_id`));
+    });
+  });
+
+  it('should work', async () => {
+    const deleteHook = await hookManageService.deleteHook({
+      hookId: hook.hookId,
+      operatorId: 'mock_owner_id',
+    });
+    assert(deleteHook);
+  });
+});

test/core/service/HookManageService/getHookByOwnerId.test.ts

@@ -0,0 +1,52 @@
+import assert from 'assert';
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { HookManageService } from '../../../../app/core/service/HookManageService';
+import { Hook } from '../../../../app/core/entity/Hook';
+import { TestUtil } from '../../../TestUtil';
+import { HookType } from '../../../../app/common/enum/Hook';
+
+describe('test/core/service/HookManageService/getHookByOwnerId.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let hook: Hook;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    hook = await hookManageService.createHook({
+      type: HookType.Package,
+      ownerId: 'mock_owner_id',
+      name: 'foo_package',
+      endpoint: 'http://foo.com',
+      secret: 'mock_secret',
+    });
+  });
+
+  afterEach(async () => {
+    await TestUtil.truncateDatabase();
+    await app.destroyModuleContext(ctx);
+    mock.restore();
+  });
+
+  describe('hook not found', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.getHookByOwnerId('not_exist_hook_id', 'mock_owner_id');
+      }, /hook not_exist_hook_id not found/);
+    });
+  });
+
+  describe('hook not belong to operator', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.getHookByOwnerId(hook.hookId, 'not_exits_owner_id');
+      }, new RegExp(`hook ${hook.hookId} not belong to not_exits_owner_id`));
+    });
+  });
+
+  it('should work', async () => {
+    const getHook = await hookManageService.getHookByOwnerId(hook.hookId, 'mock_owner_id');
+    assert(getHook);
+  });
+});

test/core/service/HookManageService/updateHook.test.ts

@@ -0,0 +1,69 @@
+import assert from 'assert';
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { HookManageService } from '../../../../app/core/service/HookManageService';
+import { Hook } from '../../../../app/core/entity/Hook';
+import { TestUtil } from '../../../TestUtil';
+import { HookType } from '../../../../app/common/enum/Hook';
+
+describe('test/core/service/HookManageService/updateHook.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let hook: Hook;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    hook = await hookManageService.createHook({
+      type: HookType.Package,
+      ownerId: 'mock_owner_id',
+      name: 'foo_package',
+      endpoint: 'http://foo.com',
+      secret: 'mock_secret',
+    });
+  });
+
+  afterEach(async () => {
+    await TestUtil.truncateDatabase();
+    await app.destroyModuleContext(ctx);
+    mock.restore();
+  });
+
+  describe('hook not found', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.updateHook({
+          hookId: 'not_exist_hook_id',
+          operatorId: 'mock_owner_id',
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      }, /hook not_exist_hook_id not found/);
+    });
+  });
+
+  describe('hook not belong to operator', () => {
+    it('should throw error', async () => {
+      await assert.rejects(async () => {
+        await hookManageService.updateHook({
+          hookId: hook.hookId,
+          operatorId: 'not_exits_owner_id',
+          endpoint: 'http://foo.com',
+          secret: 'mock_secret',
+        });
+      }, new RegExp(`hook ${hook.hookId} not belong to not_exits_owner_id`));
+    });
+  });
+
+  it('should work', async () => {
+    const updatedHook = await hookManageService.updateHook({
+      hookId: hook.hookId,
+      operatorId: 'mock_owner_id',
+      endpoint: 'http://new.com',
+      secret: 'new_mock_secret',
+    });
+    assert(updatedHook);
+    assert(updatedHook.endpoint === 'http://new.com');
+    assert(updatedHook.secret === 'new_mock_secret');
+  });
+});

test/core/service/HookTriggerService.test.ts

@@ -0,0 +1,104 @@
+import assert = require('assert');
+import { app, mock } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { TestUtil } from '../../TestUtil';
+import { HookManageService } from '../../../app/core/service/HookManageService';
+import { HookType } from '../../../app/common/enum/Hook';
+import { UserRepository } from '../../../app/repository/UserRepository';
+import { PACKAGE_VERSION_ADDED } from '../../../app/core/event';
+import { Change } from '../../../app/core/entity/Change';
+import { ChangeRepository } from '../../../app/repository/ChangeRepository';
+import { Task, TriggerHookTask } from '../../../app/core/entity/Task';
+import { HookEvent } from '../../../app/core/entity/HookEvent';
+import { CreateHookTriggerService } from '../../../app/core/service/CreateHookTriggerService';
+import { TaskRepository } from '../../../app/repository/TaskRepository';
+import { Hook } from '../../../app/core/entity/Hook';
+import { HookTriggerService } from '../../../app/core/service/HookTriggerService';
+import { RequestOptions } from 'urllib';
+
+describe('test/core/service/HookTriggerService.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let changeRepository: ChangeRepository;
+  let createHookTriggerService: CreateHookTriggerService;
+  let taskRepository: TaskRepository;
+  let hookTriggerService: HookTriggerService;
+  const pkgName = '@cnpmcore/foo';
+  const username = 'mock_username';
+  let userId: string;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    changeRepository = await ctx.getEggObject(ChangeRepository);
+    createHookTriggerService = await ctx.getEggObject(CreateHookTriggerService);
+    taskRepository = await ctx.getEggObject(TaskRepository);
+    const userRepository = await ctx.getEggObject(UserRepository);
+    hookTriggerService = await ctx.getEggObject(HookTriggerService);
+    await TestUtil.createPackage({
+      name: pkgName,
+    }, {
+      name: username,
+    });
+    const user = await userRepository.findUserByName(username);
+    userId = user!.userId;
+  });
+
+  describe('executeTask', () => {
+    let change: Change;
+    let hook: Hook;
+    let callEndpoint: string;
+    let callOptions: RequestOptions;
+
+    beforeEach(async () => {
+      change = Change.create({
+        type: PACKAGE_VERSION_ADDED,
+        targetName: pkgName,
+        data: {
+          version: '1.0.0',
+        },
+      });
+      await changeRepository.addChange(change);
+      hook = await hookManageService.createHook({
+        type: HookType.Package,
+        ownerId: userId,
+        name: pkgName,
+        endpoint: 'http://foo.com',
+        secret: 'mock_secret',
+      });
+      const task = Task.createCreateHookTask(HookEvent.createPublishEvent(pkgName, change.changeId, '1.0.0', 'latest'));
+      await createHookTriggerService.executeTask(task);
+
+      mock(ctx.httpclient, 'request', async (url, options) => {
+        callEndpoint = url;
+        callOptions = options;
+        return {
+          status: 200,
+        };
+      });
+    });
+
+    it('should execute trigger', async () => {
+      const pushTask = await taskRepository.findTaskByBizId(`TriggerHook:${change.changeId}:${hook.hookId}`) as TriggerHookTask;
+      await hookTriggerService.executeTask(pushTask);
+      assert(callEndpoint === hook.endpoint);
+      assert(callOptions);
+      assert(callOptions.method === 'POST');
+      assert(callOptions.headers!['x-npm-signature']);
+      const data = JSON.parse(callOptions.data);
+      assert(data.event === 'package:publish');
+      assert(data.name === pkgName);
+      assert(data.type === 'package');
+      assert(data.version === '1.0.0');
+      assert.deepStrictEqual(data.hookOwner, {
+        username,
+      });
+      assert(data.payload);
+      assert.deepStrictEqual(data.change, {
+        version: '1.0.0',
+        'dist-tag': 'latest',
+      });
+      assert(data.time === pushTask.data.hookEvent.time);
+    });
+  });
+});

test/core/service/PackageSyncerService/executeTask.test.ts

@@ -12,6 +12,9 @@ import { NPMRegistry } from 'app/common/adapter/NPMRegistry';
 import { NFSAdapter } from 'app/common/adapter/NFSAdapter';
 import { getScopeAndName } from 'app/common/PackageUtil';
 import { PackageRepository } from 'app/repository/PackageRepository';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryType } from 'app/common/enum/Registry';
 
 describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
   let ctx: Context;
@@ -19,6 +22,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
   let packageManagerService: PackageManagerService;
   let packageRepository: PackageRepository;
   let npmRegistry: NPMRegistry;
+  let registryManagerService: RegistryManagerService;
 
   beforeEach(async () => {
     ctx = await app.mockModuleContext();
@@ -26,6 +30,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
     packageManagerService = await ctx.getEggObject(PackageManagerService);
     packageRepository = await ctx.getEggObject(PackageRepository);
     npmRegistry = await ctx.getEggObject(NPMRegistry);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
   });
 
   afterEach(async () => {
@@ -62,6 +67,24 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
       assert.equal(abbreviatedManifests.data.name, manifests.data.name);
     });
 
+    it('should resync history version if forceSyncHistory is true', async () => {
+      await packageSyncerService.createTask('foo', { skipDependencies: true });
+      let task = await packageSyncerService.findExecuteTask();
+      assert(task);
+      await packageSyncerService.executeTask(task);
+
+      await packageSyncerService.createTask('foo', { forceSyncHistory: true, skipDependencies: true });
+      task = await packageSyncerService.findExecuteTask();
+      assert(task);
+      await packageSyncerService.executeTask(task);
+      const stream2 = await packageSyncerService.findTaskLog(task);
+      assert(stream2);
+      const log2 = await TestUtil.readStreamToLog(stream2);
+      // console.log(log2);
+      assert(/Remove version 1\.0\.0 for force sync history/.test(log2));
+      assert(/Syncing version 1\.0\.0/.test(log2));
+    });
+
     it('should not sync dependencies where task queue length too high', async () => {
       mock(app.config.cnpmcore, 'taskQueueHighWaterSize', 2);
       await packageSyncerService.createTask('foo', { skipDependencies: false });
@@ -69,6 +92,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
       await packageSyncerService.createTask('foo', { skipDependencies: false });
       await packageSyncerService.createTask('bar', { skipDependencies: false });
       await packageSyncerService.createTask('foobar', { skipDependencies: false });
+      await packageSyncerService.createTask('foobarfoo', { skipDependencies: false });
       const task = await packageSyncerService.findExecuteTask();
       assert(task);
       await packageSyncerService.executeTask(task);
@@ -1310,5 +1334,48 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
       assert(log.includes('][DownloadData] ❌ Get download data error: '));
       assert(log.includes('][DownloadData] ❌❌❌❌❌ 🚮 give up 🚮 ❌❌❌❌❌'));
     });
+
+    describe('should sync from spec registry', async () => {
+      let registry: Registry;
+      beforeEach(async () => {
+        registry = await registryManagerService.createRegistry({
+          name: 'cnpm',
+          changeStream: 'https://replicate.npmjs.com/_changes',
+          host: 'https://custom.npmjs.com',
+          userPrefix: 'cnpm:',
+          type: RegistryType.Npm,
+        });
+      });
+
+      it('should sync from target registry & default registry', async () => {
+        app.mockHttpclient(/https:\/\/custom\.npmjs\.com/, 'GET', () => {
+          throw new Error('mock error');
+        });
+        app.mockHttpclient(/https:\/\/default\.npmjs\.com/, 'GET', () => {
+          throw new Error('mock error');
+        });
+
+        await packageSyncerService.createTask('cnpm-pkg', { registryId: registry.registryId });
+        await packageSyncerService.createTask('npm-pkg');
+
+        // custom registry
+        let task = await packageSyncerService.findExecuteTask();
+        await packageSyncerService.executeTask(task);
+        let stream = await packageSyncerService.findTaskLog(task);
+        assert(stream);
+        let log = await TestUtil.readStreamToLog(stream);
+        assert(log.includes('Syncing from https://custom.npmjs.com/cnpm-pkg'));
+
+        // default registry
+        task = await packageSyncerService.findExecuteTask();
+        mock(app.config.cnpmcore, 'sourceRegistry', 'https://default.npmjs.org');
+        await packageSyncerService.executeTask(task);
+        stream = await packageSyncerService.findTaskLog(task);
+        assert(stream);
+        log = await TestUtil.readStreamToLog(stream);
+        assert(log.includes('Syncing from https://default.npmjs.org/npm-pkg'));
+
+      });
+    });
   });
 });

test/core/service/PackageSyncerService/findExecuteTask.test.ts

@@ -138,9 +138,7 @@ describe('test/core/service/PackageSyncerService/findExecuteTask.test.ts', () =>
       assert(result.processing === 0);
       assert(result.waiting === 0);
 
-      // has two tasks in queue
-      task = await packageSyncerService.findExecuteTask();
-      assert(task);
+      // has one tasks in queue
       task = await packageSyncerService.findExecuteTask();
       assert(task);
       task = await packageSyncerService.findExecuteTask();

test/core/service/PackageSyncerService/getTaskRegistry.test.ts

@@ -0,0 +1,61 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { PackageSyncerService } from 'app/core/service/PackageSyncerService';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryType } from 'app/common/enum/Registry';
+import { Task } from 'app/core/entity/Task';
+
+describe('test/core/service/PackageSyncerService/getTaskRegistry.test.ts', () => {
+  let ctx: Context;
+  let packageSyncerService: PackageSyncerService;
+  let registryManagerService: RegistryManagerService;
+  let registry: Registry;
+  let task: Task;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    packageSyncerService = await ctx.getEggObject(PackageSyncerService);
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    registry = await registryManagerService.createRegistry({
+      name: 'cnpmcore',
+      changeStream: 'https://r.cnpmjs.org/_changes',
+      host: 'https://registry.npmmirror.com',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmcore,
+    });
+
+    task = await packageSyncerService.createTask('@cnpm/banana', {
+      authorIp: '123',
+      authorId: 'ChangesStreamService',
+      registryId: registry.registryId,
+      skipDependencies: true,
+      tips: `Sync cause by changes_stream(${registry.changeStream}) update seq: 1`,
+    });
+
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('getTaskRegistry()', () => {
+    it('should work', async () => {
+      const taskRegistry = await packageSyncerService.initSpecRegistry(task);
+      assert(taskRegistry);
+      assert(taskRegistry.registryId === registry.registryId);
+    });
+    it('should support legacy task', async () => {
+      const task = await packageSyncerService.createTask('@cnpm/bananas', {
+        authorIp: '123',
+        authorId: 'ChangesStreamService',
+        skipDependencies: true,
+        tips: `Sync cause by changes_stream(${registry.changeStream}) update seq: 1`,
+      });
+
+      const taskRegistry = await packageSyncerService.initSpecRegistry(task);
+      assert(taskRegistry === null);
+    });
+  });
+});

test/core/service/RegistryManagerService/index.test.ts

@@ -0,0 +1,160 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import { RegistryType } from 'app/common/enum/Registry';
+import { ScopeManagerService } from 'app/core/service/ScopeManagerService';
+import { Registry } from 'app/core/entity/Registry';
+import { TaskRepository } from 'app/repository/TaskRepository';
+import { TaskType } from 'app/common/enum/Task';
+import { ChangesStreamTaskData } from 'app/core/entity/Task';
+
+describe('test/core/service/RegistryManagerService/index.test.ts', () => {
+  let ctx: Context;
+  let registryManagerService: RegistryManagerService;
+  let scopeManagerService: ScopeManagerService;
+  let taskRepository: TaskRepository;
+
+  before(async () => {
+    ctx = await app.mockModuleContext();
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    scopeManagerService = await ctx.getEggObject(ScopeManagerService);
+    taskRepository = await ctx.getEggObject(TaskRepository);
+  });
+
+  beforeEach(async () => {
+    // create Registry
+    await registryManagerService.createRegistry({
+      name: 'custom',
+      changeStream: 'https://r.cnpmjs.org/_changes',
+      host: 'https://cnpmjs.org',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmcore,
+    });
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('RegistryManagerService', () => {
+
+    describe('query should work', async () => {
+      beforeEach(async () => {
+        // create another
+        await registryManagerService.createRegistry({
+          name: 'custom2',
+          changeStream: 'https://r.cnpmjs.org/_changes',
+          host: 'https://cnpmjs.org',
+          userPrefix: 'ccnpm:',
+          type: RegistryType.Cnpmcore,
+        });
+      });
+
+      it('query success', async () => {
+        // query success
+        const queryRes = await registryManagerService.listRegistries({});
+        assert(queryRes.count === 2);
+        const [ _, registry ] = queryRes.data;
+        assert(_);
+        assert(registry.name === 'custom2');
+      });
+
+      it('pageOptions should work', async () => {
+        // pageOptions should work
+        let queryRes = await registryManagerService.listRegistries({ pageIndex: 0, pageSize: 1 });
+        assert(queryRes.count === 2);
+        assert(queryRes.data.length === 1);
+        const [ firstRegistry ] = queryRes.data;
+        assert(firstRegistry.name === 'custom');
+
+        queryRes = await registryManagerService.listRegistries({ pageIndex: 1, pageSize: 1 });
+        assert(queryRes.count === 2);
+        assert(queryRes.data.length === 1);
+        const [ secondRegistry ] = queryRes.data;
+        assert(secondRegistry.name === 'custom2');
+      });
+
+    });
+
+    it('update work', async () => {
+      let queryRes = await registryManagerService.listRegistries({});
+      const [ registry ] = queryRes.data;
+
+      await registryManagerService.updateRegistry({
+        ...registry,
+        name: 'custom3',
+      });
+
+      queryRes = await registryManagerService.listRegistries({});
+      assert(queryRes.data[0].name === 'custom3');
+
+    });
+
+    it('update should check registry', async () => {
+      const queryRes = await registryManagerService.listRegistries({});
+      assert(queryRes.count === 1);
+      const [ registry ] = queryRes.data;
+      await assert.rejects(
+        registryManagerService.updateRegistry({
+          ...registry,
+          registryId: 'not_exist',
+          name: 'boo',
+        }),
+        /not found/,
+      );
+    });
+
+    it('remove should work', async () => {
+      let queryRes = await registryManagerService.listRegistries({});
+      assert(queryRes.count === 1);
+      await registryManagerService.remove({ registryId: queryRes.data[0].registryId });
+      queryRes = await registryManagerService.listRegistries({});
+      assert(queryRes.count === 0);
+    });
+
+    describe('createSyncChangesStream()', async () => {
+      let registry: Registry;
+      beforeEach(async () => {
+        // create scope
+        [ registry ] = (await registryManagerService.listRegistries({})).data;
+        await scopeManagerService.createScope({ name: '@cnpm', registryId: registry.registryId });
+      });
+
+      it('should work', async () => {
+        // create success
+        await registryManagerService.createSyncChangesStream({ registryId: registry.registryId });
+        const targetName = 'CUSTOM_WORKER';
+        const task = await taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
+        assert(task);
+      });
+
+      it('should preCheck registry', async () => {
+        await assert.rejects(registryManagerService.createSyncChangesStream({ registryId: 'mock_invalid_registry_id' }), /not found/);
+      });
+
+      it('should preCheck scopes', async () => {
+        const newRegistry = await registryManagerService.createRegistry({
+          name: 'custom4',
+          changeStream: 'https://r.cnpmjs.org/_changes',
+          host: 'https://cnpmjs.org',
+          userPrefix: 'cnpm:',
+          type: RegistryType.Cnpmcore,
+        });
+        await assert.rejects(registryManagerService.createSyncChangesStream({ registryId: newRegistry.registryId }), /please create scopes first/);
+      });
+
+      it('should create only once', async () => {
+        // create success
+        await registryManagerService.createSyncChangesStream({ registryId: registry.registryId });
+        await registryManagerService.createSyncChangesStream({ registryId: registry.registryId });
+        await registryManagerService.createSyncChangesStream({ registryId: registry.registryId, since: '100' });
+        const targetName = 'CUSTOM_WORKER';
+        const task = await taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
+        assert((task?.data as ChangesStreamTaskData).since === '');
+      });
+    });
+
+
+  });
+});

test/core/service/ScopeManagerService/index.test.ts

@@ -0,0 +1,47 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { ScopeManagerService } from 'app/core/service/ScopeManagerService';
+
+describe('test/core/service/ScopeManagerService/index.test.ts', () => {
+  let ctx: Context;
+  let scopeManagerService: ScopeManagerService;
+
+  before(async () => {
+    ctx = await app.mockModuleContext();
+    scopeManagerService = await ctx.getEggObject(ScopeManagerService);
+  });
+
+  beforeEach(async () => {
+    // create
+    await scopeManagerService.createScope({
+      name: 'custom',
+      registryId: 'banana',
+    });
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('ScopeManagerService', () => {
+    it('query should work', async () => {
+      const queryRes = await scopeManagerService.listScopes({});
+      assert(queryRes.data[0].name === 'custom');
+    });
+
+    it('query after create work', async () => {
+      // create another
+      await scopeManagerService.createScope({
+        name: 'custom2',
+        registryId: 'banana',
+      });
+
+      const queryRes = await scopeManagerService.listScopes({});
+      const [ _, otherScope ] = queryRes.data;
+      assert(_);
+      assert(otherScope.name === 'custom2');
+
+    });
+  });
+});

test/core/service/TaskService/findExecuteTask.test.ts

@@ -27,23 +27,19 @@ describe('test/core/service/TaskService/findExecuteTask.test.ts', () => {
 
       const newTask = await packageSyncerService.createTask('foo');
       assert(newTask);
+      app.expectLog(/queue size: 1/);
       assert(!newTask.data.taskWorker);
-      // same task but in queue has two
+      // same task not create again
       const newTask2 = await packageSyncerService.createTask('foo');
       assert(newTask2);
       assert(newTask2.taskId === newTask.taskId);
       assert(!newTask2.data.taskWorker);
+      app.expectLog(/queue size: 1/);
 
       // find other task type
       task = await taskService.findExecuteTask(TaskType.SyncBinary);
       assert(!task);
 
-      task = await taskService.findExecuteTask(TaskType.SyncPackage);
-      assert(task);
-      assert(task.targetName === 'foo');
-      assert(task.taskId === newTask.taskId);
-      assert(task.data.taskWorker);
-      assert(task.state === TaskState.Processing);
       task = await taskService.findExecuteTask(TaskType.SyncPackage);
       assert(task);
       assert(task.targetName === 'foo');

test/core/util/ChangesStreamTransform.test.ts

@@ -0,0 +1,133 @@
+import ChangesStreamTransform from 'app/core/util/ChangesStreamTransform';
+import assert = require('assert');
+import { Readable, pipeline, Writable } from 'node:stream';
+import { ChangesStreamChange } from 'app/common/adapter/changesStream/AbstractChangesStream';
+
+describe('test/core/util/ChangesStreamTransform.test.ts', () => {
+  let stream: Readable;
+  let transform: ChangesStreamTransform;
+  beforeEach(async () => {
+    transform = new ChangesStreamTransform();
+  });
+  afterEach(async () => {
+    transform.end();
+    transform.destroy();
+  });
+
+
+  it('should work', async () => {
+    stream = Readable.from('');
+    const res = pipeline(stream, transform, error => {
+      assert(!error);
+    });
+
+    stream.push(`
+    {"results":[
+      {"seq":1,"id":"api.anyfetch.com","changes":[{"rev":"5-a87e847a323ce2503582b68c5f66a8a3"}],"deleted":true},
+      {"seq":2,"id":"backbone.websql.deferred","changes":[{"rev":"4-f5150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+      {"seq":3,"id":"binomal-hash-list","changes":[{"rev":"2-dced04d62bef47954eac61c217ed6fc1"}],"deleted":true},
+      {"seq":4,"id":"concat-file","changes":[{"rev":"5-e463032df555c6af3c47a7c9769904d4"}],"deleted":true},
+      {"seq":7,"id":"iron-core","changes":[{"rev":"4-b3b1f44a33c3a952ff0cbb2cee527f94"}],"deleted":true},
+    `);
+
+    const changes: ChangesStreamChange[] = [];
+    for await (const change of res) {
+      changes.push(change);
+    }
+
+    assert(changes.length === 5);
+    assert.deepEqual(changes.map(_ => _.fullname), [ 'api.anyfetch.com', 'backbone.websql.deferred', 'binomal-hash-list', 'concat-file', 'iron-core' ]);
+  });
+
+  it('should throw when pipe', async () => {
+    let triggered = false;
+    stream = Readable.from('');
+    const res = stream.pipe(transform);
+
+    await assert.rejects(async () => {
+      stream.push('"seq":1,');
+      stream.emit('error', new Error('mock errors'));
+      stream.push('"id":"test1"\n');
+      for await (const _ of res) {
+        triggered = true;
+        assert(_ !== null);
+      }
+    }, /mock errors/);
+
+    assert(triggered === false);
+  });
+
+  it('should work when concurrent', async () => {
+
+    stream = Readable.from('');
+    const res = pipeline(stream, transform, error => {
+      assert(!error);
+    });
+
+    stream.push(`
+      {"results":[
+        {"seq":1,"id":"api.anyfetch.com","changes":[{"rev":"5-a87e847a323ce2503582b68c5f66a8a3"}],"deleted":true},
+        {"seq":2,"id":"backbone.websql.deferred","changes":[{"rev":"4-f5150b238ab62cd890211fb57fc9eca5"}],"deleted":true},
+        {"seq":3,"id":"binomal-hash-list","changes":[{"rev":"2-dced04d62bef47954eac61c217ed6fc1"}],"deleted":true},
+        {"seq":4,"id":"concat-file","changes":[{"rev":"5-e463032df555c6af3c47a7c9769904d4"}],"deleted":true},
+        {"seq":7,"id":"iron-core","changes":[{"rev":"4-b3b1f44a33c3a952ff0cbb2cee527f94"}],"deleted":true},
+    `);
+
+    const changes: ChangesStreamChange[] = [];
+    async function parseMessage() {
+      for await (const change of res) {
+        changes.push(change);
+      }
+    }
+
+    const task = parseMessage();
+    stream.push('{"seq":8,"id":"icon-cone5","changes":[{"rev":"5-a87e847a323ce2503582b68c5f67a8a3"}],"deleted":true},');
+    await task;
+
+    assert(changes.length === 6);
+  });
+
+  it('should work handle backpressure', async () => {
+    let seq = 1;
+    stream = Readable.from('');
+    const transform = new ChangesStreamTransform({ highWaterMark: 1 });
+    let assertDrainTime = 0;
+    let assertWriteTime = 0;
+
+    // 模拟消费流，每 10ms 消费一个 changeObject
+    const assertWrite = new Writable({
+      objectMode: true,
+      highWaterMark: 1,
+      write(_, __, callback) {
+        assertWriteTime++;
+        setTimeout(() => {
+          callback();
+        }, 10);
+      },
+    });
+
+    assertWrite.on('drain', () => {
+      assertDrainTime++;
+    });
+
+    const res = new Promise<void>((resolve, reject) => {
+      pipeline(stream, transform, assertWrite, err => {
+        if (err) {
+          reject(err);
+        }
+        resolve();
+      });
+    });
+
+    for (let i = 0; i < 50; i++) {
+      stream.push(`{"seq":${++seq},"id":"${seq}","changes":[{"rev":"5-a87e847a323ce2503582b68c5f66a8a3"}],"deleted":true},`);
+    }
+
+    await res;
+
+    assert(assertDrainTime === assertWriteTime);
+    assert(assertWriteTime === 50);
+
+  });
+
+});

test/core/util/EntityUtil.test.ts

@@ -0,0 +1,25 @@
+import { EntityUtil } from 'app/core/util/EntityUtil';
+import assert = require('assert');
+
+describe('test/core/util/EntityUtil.test.ts', () => {
+  describe('convertPageOptionsToLimitOption', () => {
+    it('should work', async () => {
+      const res = EntityUtil.convertPageOptionsToLimitOption({ pageIndex: 1, pageSize: 10 });
+      assert(res.limit === 10);
+      assert(res.offset === 10);
+    });
+
+    it('should work for default value', async () => {
+      const res = EntityUtil.convertPageOptionsToLimitOption({});
+      assert(res.limit === 20);
+      assert(res.offset === 0);
+    });
+
+    it('should validate params', async () => {
+      assert.throws(() => {
+        EntityUtil.convertPageOptionsToLimitOption({ pageIndex: 1, pageSize: 101 });
+      }, /max page size is 100, current request is 101/);
+
+    });
+  });
+});

test/infra/QueueAdapter.test.ts

@@ -0,0 +1,31 @@
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import assert from 'assert';
+import { RedisQueueAdapter } from '../../app/infra/QueueAdapter';
+
+describe('test/infra/QueueAdapter.test.ts', () => {
+  let ctx: Context;
+  let queueAdapter: RedisQueueAdapter;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    queueAdapter = await ctx.getEggObject(RedisQueueAdapter);
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  it('should not push duplicate task', async () => {
+    const queueName = 'duplicate_test';
+    const taskId = 'task_id_1';
+    let res = await queueAdapter.push(queueName, taskId);
+    assert(res === true);
+    res = await queueAdapter.push(queueName, taskId);
+    assert(res === false);
+    const length = await queueAdapter.length(queueName);
+    assert(length === 1);
+    const queueTaskId = await queueAdapter.pop(queueName);
+    assert(queueTaskId === taskId);
+  });
+});

test/port/controller/PackageSyncController/createSyncTask.test.ts

@@ -103,6 +103,47 @@ describe('test/port/controller/PackageSyncController/createSyncTask.test.ts', ()
       app.expectLog(', targetName: koa-not-exists,');
     });
 
+    it('should error when invalid registryName', async () => {
+      mock(app.config.cnpmcore, 'alwaysAuth', false);
+      const admin = await TestUtil.createAdmin();
+      const res = await app.httpRequest()
+        .put('/-/package/koa-not-exists/syncs')
+        .set('authorization', admin.authorization)
+        .send({ registryName: 'invalid' })
+        .expect(403);
+
+      assert(res.body.error.includes('Can\'t find target registry'));
+    });
+
+    it('should check the packageEntity registryId', async () => {
+      mock(app.config.cnpmcore, 'alwaysAuth', false);
+      await TestUtil.createPackage({
+        name: '@cnpm/banana',
+        registryId: 'mock_registry_id',
+        isPrivate: false,
+      });
+      const admin = await TestUtil.createAdmin();
+      // create registry
+      await app.httpRequest()
+        .post('/-/registry')
+        .set('authorization', admin.authorization)
+        .send(
+          {
+            name: 'cnpm',
+            host: 'https://r.cnpmjs.org/',
+            changeStream: 'https://r.cnpmjs.org/_changes',
+            type: 'cnpmcore',
+          });
+
+      const res = await app.httpRequest()
+        .put('/-/package/@cnpm/banana/syncs')
+        .set('authorization', admin.authorization)
+        .send({ registryName: 'cnpm' })
+        .expect(403);
+
+      assert(res.body.error.includes('The package is synced from'));
+    });
+
     it('should sync immediately and mock executeTask error when admin user request', async () => {
       mock(app.config.cnpmcore, 'alwaysAuth', false);
       mock.error(PackageSyncerService.prototype, 'executeTask');

test/port/controller/RegistryController/index.test.ts

@@ -0,0 +1,256 @@
+import { TaskType } from 'app/common/enum/Task';
+import { Registry } from 'app/core/entity/Registry';
+import { ChangesStreamTaskData } from 'app/core/entity/Task';
+import { TaskService } from 'app/core/service/TaskService';
+import assert = require('assert');
+import { Context } from 'egg';
+import { app } from 'egg-mock/bootstrap';
+import { TestUtil } from '../../../TestUtil';
+
+describe('test/port/controller/RegistryController/index.test.ts', () => {
+  let ctx: Context;
+  let adminUser: any;
+  let registry: Registry;
+  let taskService: TaskService;
+  before(async () => {
+    ctx = await app.mockModuleContext();
+    taskService = await ctx.getEggObject(TaskService);
+  });
+  beforeEach(async () => {
+    adminUser = await TestUtil.createAdmin();
+    // create success
+    await app.httpRequest()
+      .post('/-/registry')
+      .set('authorization', adminUser.authorization)
+      .send(
+        {
+          name: 'custom3',
+          host: 'https://r.cnpmjs.org/',
+          changeStream: 'https://r.cnpmjs.org/_changes',
+          type: 'cnpmcore',
+        })
+      .expect(200);
+
+    // query success
+    const res = await app.httpRequest()
+      .get('/-/registry')
+      .expect(200);
+
+    registry = res.body.data[0];
+
+    // create scope
+    await app.httpRequest()
+      .post('/-/scope')
+      .set('authorization', adminUser.authorization)
+      .send({
+        name: '@cnpm',
+        registryId: registry.registryId,
+      })
+      .expect(200);
+
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('[POST /-/registry] createRegistry()', () => {
+    it('should 200', async () => {
+      // create success
+      const res = await app.httpRequest()
+        .post('/-/registry')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: 'custom6',
+            host: 'https://r.cnpmjs.org/',
+            changeStream: 'https://r.cnpmjs.org/_changes',
+            type: 'cnpmcore',
+          });
+
+      assert(res.body.ok);
+    });
+
+    it('should verify params', async () => {
+      // create success
+      const res = await app.httpRequest()
+        .post('/-/registry')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: 'custom',
+            type: 'cnpmcore',
+          })
+        .expect(422);
+
+      assert(res.body.error === '[INVALID_PARAM] must have required property \'host\'');
+    });
+
+    it('should 403', async () => {
+      // create forbidden
+      const res = await app.httpRequest()
+        .post('/-/registry')
+        .send(
+          {
+            name: 'custom',
+            host: 'https://r.cnpmjs.org/',
+            changeStream: 'https://r.cnpmjs.org/_changes',
+            type: 'cnpmcore',
+          })
+        .expect(403);
+
+      assert(res.body.error === '[FORBIDDEN] Not allow to access');
+    });
+
+  });
+
+  describe('[GET /-/registry] listRegistries()', () => {
+    it('should 200', async () => {
+      // create success
+      await app.httpRequest()
+        .post('/-/registry')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: 'custom5',
+            host: 'https://r.cnpmjs.org/',
+            changeStream: 'https://r.cnpmjs.org/_changes',
+            type: 'cnpmcore',
+          })
+        .expect(200);
+
+      // query success
+      const res = await app.httpRequest()
+        .get('/-/registry')
+        .expect(200);
+
+      assert(res.body.count === 2);
+      assert(res.body.data[1].name === 'custom5');
+    });
+  });
+
+  describe('[GET /-/registry/:id/scopes] showRegistryScopes()', () => {
+    it('should 200', async () => {
+      // create scope
+      await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send({
+          registryId: registry.registryId,
+          name: '@banana',
+        });
+
+      await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send({
+          registryId: registry.registryId,
+          name: '@apple',
+        });
+
+      await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send({
+          registryId: registry.registryId,
+          name: '@orange',
+        });
+
+      let scopRes = await app.httpRequest()
+        .get(`/-/registry/${registry.registryId}/scopes`)
+        .expect(200);
+      assert(scopRes.body.count === 4);
+      assert(scopRes.body.data.length === 4);
+
+      scopRes = await app.httpRequest()
+        .get(`/-/registry/${registry.registryId}/scopes?pageSize=1`)
+        .expect(200);
+      assert(scopRes.body.count === 4);
+      assert(scopRes.body.data.length === 1);
+
+      scopRes = await app.httpRequest()
+        .get(`/-/registry/${registry.registryId}/scopes?pageSize=2&pageIndex=1`)
+        .expect(200);
+      assert(scopRes.body.count === 4);
+      assert(scopRes.body.data.length === 2);
+
+    });
+    it('should error', async () => {
+      await app.httpRequest()
+        .get('/-/registry/not_exist_id/scopes')
+        .expect(404);
+    });
+  });
+
+  describe('[GET /-/registry/:id] showRegistry()', () => {
+    it('should 200', async () => {
+      const queryRes = await app.httpRequest()
+        .get(`/-/registry/${registry.registryId}`);
+      assert.deepEqual(queryRes.body, registry);
+    });
+
+    it('should error', async () => {
+      await app.httpRequest()
+        .get('/-/registry/not_exist_id')
+        .expect(404);
+    });
+  });
+
+  describe('[DELETE /-/registry] deleteRegistry()', () => {
+    it('should 200', async () => {
+      await app.httpRequest()
+        .delete(`/-/registry/${registry.registryId}`)
+        .set('authorization', adminUser.authorization)
+        .expect(200);
+
+      // query success
+      const queryRes = await app.httpRequest()
+        .get('/-/registry')
+        .set('authorization', adminUser.authorization)
+        .expect(200);
+
+      assert(queryRes.body.count === 0);
+    });
+  });
+
+  describe('[POST /-/registry/:id/sync] createRegistrySyncTask()', () => {
+    it('should 403', async () => {
+      await app.httpRequest()
+        .post(`/-/registry/${registry.registryId}/sync`)
+        .expect(403);
+    });
+
+    it('should error when invalid registryId', async () => {
+      const res = await app.httpRequest()
+        .post('/-/registry/in_valid/sync')
+        .set('authorization', adminUser.authorization)
+        .expect(404);
+      assert(res.body.error.includes('registry not found'));
+    });
+
+    it('should 200', async () => {
+      await app.httpRequest()
+        .post(`/-/registry/${registry.registryId}/sync`)
+        .set('authorization', adminUser.authorization)
+        .expect(200);
+
+      const task = await taskService.findExecuteTask(TaskType.ChangesStream);
+      assert(task?.targetName === 'CUSTOM3_WORKER');
+    });
+
+    it('since params', async () => {
+      await app.httpRequest()
+        .post(`/-/registry/${registry.registryId}/sync`)
+        .set('authorization', adminUser.authorization)
+        .send({
+          since: '9527',
+        })
+        .expect(200);
+
+      const task = await taskService.findExecuteTask(TaskType.ChangesStream);
+      assert(task?.targetName === 'CUSTOM3_WORKER');
+      assert((task?.data as ChangesStreamTaskData).since === '9527');
+    });
+  });
+
+});

test/port/controller/ScopeController/index.test.ts

@@ -0,0 +1,106 @@
+import assert = require('assert');
+import { RegistryType } from 'app/common/enum/Registry';
+import { RegistryManagerService } from 'app/core/service/RegistryManagerService';
+import { Context } from 'egg';
+import { app } from 'egg-mock/bootstrap';
+import { TestUtil } from '../../../TestUtil';
+import { Scope } from 'app/core/entity/Scope';
+
+describe('test/port/controller/ScopeController/index.test.ts', () => {
+  let ctx: Context;
+  let adminUser: any;
+  let registryManagerService: RegistryManagerService;
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    adminUser = await TestUtil.createAdmin();
+    registryManagerService = await ctx.getEggObject(RegistryManagerService);
+    // create registry
+    await registryManagerService.createRegistry({
+      name: 'custom',
+      host: 'https://r.cnpmjs.org/',
+      changeStream: 'https://r.cnpmjs.org/_changes',
+      userPrefix: 'cnpm:',
+      type: RegistryType.Cnpmcore,
+    });
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('[POST /-/scope] createScope()', () => {
+    it('should 200', async () => {
+
+      const queryRes = await registryManagerService.listRegistries({});
+      const [ registry ] = queryRes.data;
+
+      // create success
+      const res = await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: '@cnpm',
+            registryId: registry.registryId,
+          })
+        .expect(200);
+
+      assert(res.body.ok);
+    });
+
+    it('should 400', async () => {
+      const res = await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: '@cnpmbanana',
+            registryId: 'banana',
+          })
+        .expect(400);
+
+      assert(res.body.error === '[BAD_REQUEST] registry banana not found');
+    });
+
+    it('should 403', async () => {
+      // create success
+      const res = await app.httpRequest()
+        .post('/-/scope')
+        .send(
+          {
+            name: '@cnpm',
+          })
+        .expect(403);
+
+      assert(res.body.error === '[FORBIDDEN] Not allow to access');
+    });
+
+  });
+
+  describe('[DELETE /-/scope/:id] deleteScope()', () => {
+    let scope: Scope;
+    beforeEach(async () => {
+      const queryRes = await registryManagerService.listRegistries({});
+      const registry = queryRes.data[0];
+      let res = await app.httpRequest()
+        .post('/-/scope')
+        .set('authorization', adminUser.authorization)
+        .send(
+          {
+            name: '@cnpmjsa',
+            registryId: registry.registryId,
+          });
+      res = await app.httpRequest()
+        .get(`/-/registry/${registry.registryId}/scopes`);
+
+      scope = res.body.data[0];
+    });
+    it('should 200', async () => {
+      const res = await app.httpRequest()
+        .delete(`/-/scope/${scope.scopeId}`)
+        .set('authorization', adminUser.authorization)
+        .expect(200);
+      assert(res.body.ok);
+    });
+  });
+});

test/port/controller/hook/HookController.test.ts

@@ -0,0 +1,157 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { TestUtil } from '../../../TestUtil';
+import { HookManageService } from '../../../../app/core/service/HookManageService';
+import { Hook } from '../../../../app/core/entity/Hook';
+import { UserRepository } from '../../../../app/repository/UserRepository';
+import { HookType } from '../../../../app/common/enum/Hook';
+
+describe('test/port/controller/hook/HookController.test.ts', () => {
+  let ctx: Context;
+  let hookManageService: HookManageService;
+  let user;
+  let userId;
+
+  beforeEach(async () => {
+    user = await TestUtil.createUser();
+    ctx = await app.mockModuleContext();
+    hookManageService = await ctx.getEggObject(HookManageService);
+    const userRepository = await ctx.getEggObject(UserRepository);
+    const userEntity = await userRepository.findUserByName(user.name);
+    userId = userEntity?.userId;
+  });
+
+  describe('POST /-/npm/v1/hooks/hook', () => {
+    it('should work', async () => {
+      const res = await app.httpRequest()
+        .post('/-/npm/v1/hooks/hook')
+        .set('authorization', user.authorization)
+        .set('user-agent', user.ua)
+        .send({
+          type: 'scope',
+          name: '@cnpmcore',
+          endpoint: 'https://example.com/webhook',
+          secret: 'this is certainly very secret',
+        })
+        .expect(200);
+      assert(res.body);
+      assert(res.body.id);
+      assert(res.body.username === user.name);
+      assert(res.body.name === '@cnpmcore');
+      assert(res.body.endpoint === 'https://example.com/webhook');
+      assert(res.body.secret === 'this is certainly very secret');
+      assert(res.body.type === HookType.Scope);
+      assert(res.body.created);
+      assert(res.body.updated);
+      assert(res.body.delivered === false);
+      assert(res.body.last_delivery === null);
+      assert(res.body.response_code === 0);
+      assert(res.body.status === 'active');
+    });
+  });
+
+  describe('PUT /-/npm/v1/hooks/hook/:id', () => {
+    let hook: Hook;
+    beforeEach(async () => {
+      hook = await hookManageService.createHook({
+        type: HookType.Package,
+        ownerId: userId,
+        name: 'foo_package',
+        endpoint: 'http://foo.com',
+        secret: 'mock_secret',
+      });
+    });
+
+    it('should work', async () => {
+      const res = await app.httpRequest()
+        .put(`/-/npm/v1/hooks/hook/${hook.hookId}`)
+        .set('authorization', user.authorization)
+        .set('user-agent', user.ua)
+        .send({
+          endpoint: 'https://new.com/webhook',
+          secret: 'new secret',
+        })
+        .expect(200);
+      assert(res.body.endpoint === 'https://new.com/webhook');
+      assert(res.body.secret === 'new secret');
+    });
+  });
+
+  describe('DELETE /-/npm/v1/hooks/hook/:id', () => {
+    let hook: Hook;
+    beforeEach(async () => {
+      hook = await hookManageService.createHook({
+        type: HookType.Package,
+        ownerId: userId,
+        name: 'foo_package',
+        endpoint: 'http://foo.com',
+        secret: 'mock_secret',
+      });
+    });
+
+    it('should work', async () => {
+      const res = await app.httpRequest()
+        .delete(`/-/npm/v1/hooks/hook/${hook.hookId}`)
+        .set('authorization', user.authorization)
+        .set('user-agent', user.ua)
+        .expect(200);
+      assert(res.body.deleted === true);
+    });
+  });
+
+  describe('GET /-/npm/v1/hooks', () => {
+    beforeEach(async () => {
+      await hookManageService.createHook({
+        type: HookType.Package,
+        ownerId: userId,
+        name: 'foo_package',
+        endpoint: 'http://foo.com',
+        secret: 'mock_secret',
+      });
+    });
+
+    it('should work', async () => {
+      const res = await app.httpRequest()
+        .get('/-/npm/v1/hooks')
+        .set('authorization', user.authorization)
+        .set('user-agent', user.ua)
+        .expect(200);
+      assert(res.body.objects.length === 1);
+    });
+  });
+
+  describe('GET /-/npm/v1/hooks/hook/:id', () => {
+    let hook: Hook;
+    beforeEach(async () => {
+      hook = await hookManageService.createHook({
+        type: HookType.Package,
+        ownerId: userId,
+        name: 'foo_package',
+        endpoint: 'http://foo.com',
+        secret: 'mock_secret',
+      });
+    });
+
+    it('should work', async () => {
+      const res = await app.httpRequest()
+        .get(`/-/npm/v1/hooks/hook/${hook.hookId}`)
+        .set('authorization', user.authorization)
+        .set('user-agent', user.ua)
+        .expect(200);
+      assert(res.body);
+      assert(res.body.id === hook.hookId);
+      assert(res.body.username === user.name);
+      assert(res.body.name === 'foo_package');
+      assert(res.body.endpoint === 'http://foo.com');
+      assert(res.body.secret === 'mock_secret');
+      assert(res.body.type === HookType.Package);
+      assert(res.body.created);
+      assert(res.body.updated);
+      assert(res.body.delivered === false);
+      assert(res.body.last_delivery === null);
+      assert(res.body.response_code === 0);
+      assert(res.body.status === 'active');
+    });
+  });
+});

test/port/controller/package/ShowPackageController.test.ts

@@ -112,6 +112,22 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => {
         .set('If-None-Match', res.headers.etag)
         .expect('vary', 'Origin')
         .expect(304);
+      // ignore sync request
+      res2 = await app.httpRequest()
+        .get(`/${name}?cache=0`)
+        .set('If-None-Match', res.headers.etag)
+        .expect('vary', 'Origin')
+        .expect(200);
+      assert(res2.body.name);
+      assert.equal(res2.headers.etag, res.headers.etag);
+      res2 = await app.httpRequest()
+        .get(`/${name}`)
+        .set('If-None-Match', res.headers.etag)
+        .set('user-agent', 'npm_service.cnpmjs.org/1.0.0')
+        .expect('vary', 'Origin')
+        .expect(200);
+      assert(res2.body.name);
+      assert.equal(res2.headers.etag, res.headers.etag);
 
       mock(app.config.cnpmcore, 'enableCDN', true);
       await app.httpRequest()

test/repository/RegistryRepository.test.ts

@@ -0,0 +1,78 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { RegistryRepository } from 'app/repository/RegistryRepository';
+import { Registry } from 'app/core/entity/Registry';
+import { RegistryType } from 'app/common/enum/Registry';
+
+describe('test/repository/RegistryRepository.test.ts', () => {
+  let ctx: Context;
+
+  let registryRepository: RegistryRepository;
+  let registryModel: Registry;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    registryRepository = await ctx.getEggObject(RegistryRepository);
+    registryModel = await registryRepository.saveRegistry(Registry.create({
+      name: 'cnpmcore',
+      userPrefix: 'cnpm:',
+      changeStream: 'https://r.npmjs.com/_changes',
+      host: 'https://registry.npmjs.org',
+      type: 'cnpmcore' as RegistryType,
+    })) as Registry;
+
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+
+  describe('RegistryRepository', () => {
+    it('create work', async () => {
+      const newRegistry = await registryRepository.saveRegistry(Registry.create({
+        name: 'npm',
+        userPrefix: 'npm:',
+        changeStream: 'https://ra.npmjs.com/_changes',
+        host: 'https://registry.npmjs.org',
+        type: 'cnpmcore' as RegistryType,
+      })) as Registry;
+      assert(newRegistry);
+      assert(newRegistry.type === 'cnpmcore');
+    });
+    it('update work', async () => {
+      const updatedRegistry = await registryRepository.saveRegistry({
+        ...registryModel,
+        registryId: registryModel.registryId,
+        id: registryModel.id,
+        name: 'banana',
+        userPrefix: 'cnpm:',
+        host: 'https://registry.npmjs.org',
+        type: 'cnpmcore' as RegistryType,
+        changeStream: 'https://replicate.npmjs.com/_changes',
+      });
+      assert(updatedRegistry);
+      assert(updatedRegistry.name === 'banana');
+    });
+    it('list work', async () => {
+      const registries = await registryRepository.listRegistries({});
+      assert(registries.count === 1);
+    });
+
+    it('query null', async () => {
+      const queryRes = await registryRepository.findRegistry('orange');
+      assert(queryRes === null);
+    });
+
+    it('query work', async () => {
+      const queryRes = await registryRepository.findRegistry('cnpmcore');
+      assert(queryRes?.name === 'cnpmcore');
+    });
+    it('remove work', async () => {
+      await registryRepository.removeRegistry(registryModel.registryId);
+      const emptyRes = await registryRepository.listRegistries({});
+      assert.deepEqual(emptyRes.data, []);
+    });
+  });
+});

test/repository/ScopeRepository.test.ts

@@ -0,0 +1,73 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { ScopeRepository } from 'app/repository/ScopeRepository';
+import { Scope } from 'app/core/entity/Scope';
+
+describe('test/repository/ScopeRepository.test.ts', () => {
+  let ctx: Context;
+
+  let scopeRepository: ScopeRepository;
+  let cnpmjsScope: Scope;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    scopeRepository = await ctx.getEggObject(ScopeRepository);
+    cnpmjsScope = await scopeRepository.saveScope(Scope.create({
+      name: '@cnpmjs',
+      registryId: '1',
+    })) as Scope;
+  });
+
+  afterEach(async () => {
+    await app.destroyModuleContext(ctx);
+  });
+
+  describe('RegistryRepository', () => {
+    it('create work', async () => {
+      const cnpmScope = await scopeRepository.saveScope(Scope.create({
+        name: '@cnpm',
+        registryId: '1',
+      }));
+      assert(cnpmScope);
+      assert(cnpmjsScope);
+    });
+
+    it('list work', async () => {
+      // list
+      const cnpmScope = await scopeRepository.saveScope(Scope.create({
+        name: '@cnpm',
+        registryId: '1',
+      })) as Scope;
+      const scopeRes = await scopeRepository.listScopes({});
+      assert.deepEqual([ cnpmjsScope.name, cnpmScope.name ], scopeRes.data.map(scope => scope.name));
+    });
+
+    it('update work', async () => {
+      await scopeRepository.saveScope({
+        ...cnpmjsScope,
+        id: cnpmjsScope.id,
+        scopeId: cnpmjsScope.scopeId,
+        name: '@anpm',
+        registryId: '1',
+      });
+      const scopeRes = await scopeRepository.listScopes({});
+      assert(scopeRes.count === 1);
+      assert(scopeRes.data[0].name === '@anpm');
+    });
+
+    it('remove work', async () => {
+      // remove
+      const cnpmScope = await scopeRepository.saveScope(Scope.create({
+        name: '@cnpm',
+        registryId: '1',
+      })) as Scope;
+      await scopeRepository.removeScope(cnpmjsScope.scopeId);
+      const scopesAfterRemove = await scopeRepository.listScopes({});
+      assert.deepEqual(scopesAfterRemove.data.map(scope => scope.name), [ cnpmScope.name ]);
+      await scopeRepository.removeScopeByRegistryId(cnpmjsScope.registryId);
+      const emptyRes = await scopeRepository.listScopes({});
+      assert.deepEqual(emptyRes.data, []);
+    });
+  });
+});

test/repository/TaskRepository.test.ts

@@ -0,0 +1,58 @@
+import assert = require('assert');
+import { app } from 'egg-mock/bootstrap';
+import { Context } from 'egg';
+import { TaskRepository } from 'app/repository/TaskRepository';
+import { Task as TaskModel } from 'app/repository/model/Task';
+import { ChangesStreamTaskData, Task, TaskData } from '../../app/core/entity/Task';
+import { TaskState, TaskType } from '../../app/common/enum/Task';
+import os from 'os';
+import { EasyData, EntityUtil } from '../../app/core/util/EntityUtil';
+
+describe('test/repository/TaskRepository.test.ts', () => {
+  let ctx: Context;
+
+  let taskRepository: TaskRepository;
+
+  beforeEach(async () => {
+    ctx = await app.mockModuleContext();
+    taskRepository = await ctx.getEggObject(TaskRepository);
+    await TaskModel.truncate();
+  });
+
+  afterEach(async () => {
+    await TaskModel.truncate();
+    await app.destroyModuleContext(ctx);
+  });
+
+
+  describe('unique biz id', () => {
+    it('should save succeed if biz id is equal', async () => {
+      const bizId = 'mock_dup_biz_id';
+      const data: EasyData<TaskData<ChangesStreamTaskData>, 'taskId'> = {
+        type: TaskType.ChangesStream,
+        state: TaskState.Waiting,
+        targetName: 'foo',
+        authorId: `pid_${process.pid}`,
+        authorIp: os.hostname(),
+        data: {
+          taskWorker: '',
+          since: '',
+        },
+        bizId,
+      };
+      const newData = EntityUtil.defaultData(data, 'taskId');
+      const task1 = new Task(newData);
+      const task2 = new Task(newData);
+      await Promise.all([
+        taskRepository.saveTask(task1),
+        taskRepository.saveTask(task2),
+      ]);
+      assert(task1.id);
+      assert(task2.id);
+      assert(task1.id === task2.id);
+      assert(task1.taskId);
+      assert(task2.taskId);
+      assert(task1.taskId === task2.taskId);
+    });
+  });
+});