Release 2.10.1

[skip ci]

## [2.10.1](https://github.com/cnpm/npmcore/compare/v2.10.0...v2.10.1) (2023-01-08)

### Bug Fixes

* export _cnpmcore_publish_time on abbreviated manifests ([#374](https://github.com/cnpm/npmcore/issues/374)) ([4bceac5](4bceac5a4c))

.github/workflows/nodejs.yml

@@ -5,15 +5,9 @@ name: Node.js CI
 
 on:
   push:
-    branches:
-      - main
-      - master
+    branches: [ master ]
   pull_request:
-    branches:
-      - main
-      - master
-  schedule:
-    - cron: '0 2 * * *'
+    branches: [ master ]
 
 jobs:
   test-mysql57-fs-nfs:
@@ -28,6 +22,12 @@ jobs:
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
+      redis:
+        # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#example-mapping-redis-ports
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
 
     strategy:
       fail-fast: false
@@ -37,19 +37,13 @@ jobs:
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
         node-version: ${{ matrix.node-version }}
 
-    # https://github.com/marketplace/actions/redis-server-in-github-actions#usage
-    - name: Start Redis
-      uses: supercharge/redis-github-action@1.4.0
-      with:
-        redis-version: 6
-
     - name: Install Dependencies
       run: npm i
 
@@ -57,15 +51,14 @@ jobs:
       run: npm run ci
 
     - name: Code Coverage
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
 
   test-mysql57-oss-nfs:
     runs-on: ${{ matrix.os }}
-    if:
+    if: |
       contains('
-        refs/heads/main
         refs/heads/master
         refs/heads/dev
       ', github.ref)
@@ -80,6 +73,13 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
 
+      redis:
+        # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#example-mapping-redis-ports
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
+
     strategy:
       fail-fast: false
       matrix:
@@ -88,19 +88,13 @@ jobs:
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
         node-version: ${{ matrix.node-version }}
 
-    # https://github.com/marketplace/actions/redis-server-in-github-actions#usage
-    - name: Start Redis
-      uses: supercharge/redis-github-action@1.4.0
-      with:
-        redis-version: 6
-
     - name: Install Dependencies
       run: npm i
 
@@ -108,12 +102,12 @@ jobs:
       run: npm run ci
       env:
         CNPMCORE_NFS_TYPE: oss
-        CNPMCORE_NFS_OSS_BUCKET: cnpmcore-unittest-github
+        CNPMCORE_NFS_OSS_BUCKET: cnpmcore-unittest-github-nodejs-${{ matrix.node-version }}
         CNPMCORE_NFS_OSS_ENDPOINT: https://oss-us-west-1.aliyuncs.com
         CNPMCORE_NFS_OSS_ID: ${{ secrets.CNPMCORE_NFS_OSS_ID }}
         CNPMCORE_NFS_OSS_SECRET: ${{ secrets.CNPMCORE_NFS_OSS_SECRET }}
 
     - name: Code Coverage
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,18 @@
+name: Release
+on:
+  # 合并后自动发布
+  push:
+    branches: [ master, main ]
+
+  # 手动发布
+  workflow_dispatch: {}
+
+jobs:
+  release:
+    name: Node.js
+    uses: artusjs/github-actions/.github/workflows/node-release.yml@v1
+    secrets:
+      NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+      GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
+    with:
+      checkTest: false

.github/workflows/sql-review.yml

@@ -0,0 +1,18 @@
+# https://github.com/marketplace/actions/sql-review
+
+name: SQL Review
+
+on: [pull_request]
+
+jobs:
+  sql-review:
+    runs-on: ubuntu-latest
+    name: SQL Review
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check SQL
+        uses: bytebase/sql-review-action@main
+        with:
+          database-type: MYSQL
+          file-pattern: ^sql/.*\.sql$
+          override-file-path: ./sql-review-override.yml

.github/workflows/stale.yml

@@ -1,27 +0,0 @@
-# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
-#
-# You can adjust the behavior by modifying this file.
-# For more information, see:
-# https://github.com/actions/stale
-name: Mark stale issues and pull requests
-
-on:
-  schedule:
-  - cron: '45 15 * * *'
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-
-    steps:
-    - uses: actions/stale@v3
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: 'Stale issue message'
-        stale-pr-message: 'Stale pull request message'
-        stale-issue-label: 'no-issue-activity'
-        stale-pr-label: 'no-pr-activity'

.gitignore

@@ -13,6 +13,7 @@ config/config.prod.ts
 config/**/*.js
 app/**/*.js
 test/**/*.js
+app.js
 
 .cnpmcore
 .cnpmcore_unittest
@@ -75,7 +76,7 @@ typings/
 
 # Output of 'npm pack'
 *.tgz
-!test/fixtures/*.tgz
+!test/fixtures/**/*.tgz
 
 # Yarn Integrity file
 .yarn-integrity

CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+## [2.10.1](https://github.com/cnpm/npmcore/compare/v2.10.0...v2.10.1) (2023-01-08)
+
+
+### Bug Fixes
+
+* export _cnpmcore_publish_time on abbreviated manifests ([#374](https://github.com/cnpm/npmcore/issues/374)) ([4bceac5](https://github.com/cnpm/npmcore/commit/4bceac5a4c94f8e8624ae1113ad1c5e69a5a2ae1))
+
+## [2.10.0](https://github.com/cnpm/npmcore/compare/v2.9.1...v2.10.0) (2023-01-05)
+
+
+### Features
+
+* unpublish pkg when upstream block ([#372](https://github.com/cnpm/npmcore/issues/372)) ([7e419c1](https://github.com/cnpm/npmcore/commit/7e419c1fb4fe297adea86cb5d9eae4c8e77e2aec))
+
+## [2.9.1](https://github.com/cnpm/npmcore/compare/v2.9.0...v2.9.1) (2022-12-17)
+
+
+### Bug Fixes
+
+* Auto enable npm publish on github action ([3d366dd](https://github.com/cnpm/npmcore/commit/3d366dd996161f8f08ae43bde29b7768f5a5241c))
+* fix tsc:prod ([ca78d00](https://github.com/cnpm/npmcore/commit/ca78d00f28930180a9374c01d2a9b3b47d6e9db3))

DEVELOPER.md

@@ -32,7 +32,7 @@ $ npm install
 $ MYSQL_DATABASE=cnpmcore npm run prepare-database
 
 # 启动 Web 服务
-$ DEBUG_LOCAL_SQL=true npm run dev
+$ npm run dev
 
 # 访问
 curl -v http://127.0.0.1:7001

History.md

@@ -1,4 +1,253 @@
 
+2.9.0 / 2022-12-15
+==================
+
+**features**
+  * [[`c562645`](http://github.com/cnpm/cnpmcore/commit/c562645db7c88f9c3c5787fd450b457574d1cce6)] - feat: suspend task before app close (#365) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.8.1 / 2022-12-05
+==================
+
+**features**
+  * [[`fad30ad`](http://github.com/cnpm/cnpmcore/commit/fad30adc564c931c0bf63828d83bab84105aaef0)] - feat: npm command support npm v6 (#356) (laibao101 <<369632567@qq.com>>)
+
+**fixes**
+  * [[`f961219`](http://github.com/cnpm/cnpmcore/commit/f961219dbe4676156e1766db82379ee40087bcd8)] - fix: Sync save ignore ER_DUP_ENTRY error (#364) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+**others**
+  * [[`7bc0fcc`](http://github.com/cnpm/cnpmcore/commit/7bc0fccaca880efe08228b4109953bd3974d2eb9)] - 🤖 TEST: Fix async function mock (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`84ae9bc`](http://github.com/cnpm/cnpmcore/commit/84ae9bcfa06124255703b926f83fb5e6a6bf9d6b)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
+
+2.8.0 / 2022-11-29
+==================
+
+**others**
+  * [[`d55c680`](http://github.com/cnpm/cnpmcore/commit/d55c680ef906ecb27f7967782ad7d25987cef7d4)] - Event cork (#361) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.7.1 / 2022-11-25
+==================
+
+**fixes**
+  * [[`c6b8aec`](http://github.com/cnpm/cnpmcore/commit/c6b8aecfd0c2b0d454389e931747c431dac5742b)] - fix: request binary error (#360) (Ke Wu <<gemwuu@163.com>>)
+
+2.7.0 / 2022-11-25
+==================
+
+**others**
+  * [[`5738d56`](http://github.com/cnpm/cnpmcore/commit/5738d569ea691c05c3f3b0b74a454a33fefb8fc7)] - refactor: binary sync task use binaryName by default (#358) (Ke Wu <<gemwuu@163.com>>)
+
+2.6.1 / 2022-11-23
+==================
+
+**fixes**
+  * [[`0b35ead`](http://github.com/cnpm/cnpmcore/commit/0b35ead2a0cd73b89d2d961bafec13d7250fe805)] - 🐛 FIX: typo for canvas (fengmk2 <<fengmk2@gmail.com>>)
+
+2.6.0 / 2022-11-23
+==================
+
+**features**
+  * [[`be8387d`](http://github.com/cnpm/cnpmcore/commit/be8387dfa48b9487156542000a93081fa823694a)] - feat: Support canvas sync from different binary (#357) (Ke Wu <<gemwuu@163.com>>)
+
+**fixes**
+  * [[`d6c4cf5`](http://github.com/cnpm/cnpmcore/commit/d6c4cf5029ca6450064fc05696a8624b6c36f0b2)] - fix: duplicate binary task (#354) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.5.2 / 2022-11-11
+==================
+
+**fixes**
+  * [[`7eb209d`](http://github.com/cnpm/cnpmcore/commit/7eb209de1332417db2070846891d78f5afa0cd10)] - fix: create task when waiting (#352) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.5.1 / 2022-11-07
+==================
+
+**others**
+  * [[`e40c502`](http://github.com/cnpm/cnpmcore/commit/e40c5021bb2ba78f8879d19bc477883168560b85)] - 🐛 FIX: Mirror cypress arm64 binary (#351) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.5.0 / 2022-11-04
+==================
+
+**features**
+  * [[`43d77ee`](http://github.com/cnpm/cnpmcore/commit/43d77ee91e52bd74594d9d569b839c1a4b7fbac6)] - feat: long description (#349) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.4.1 / 2022-10-28
+==================
+
+**features**
+  * [[`92350a8`](http://github.com/cnpm/cnpmcore/commit/92350a864313ee42a048d9e83886ef42db3419de)] - 👌 IMPROVE: Show changes stream create task log (#347) (fengmk2 <<fengmk2@gmail.com>>)
+
+**fixes**
+  * [[`28eeeaf`](http://github.com/cnpm/cnpmcore/commit/28eeeafd9870c6b1c5b4f4c23916f6ae73ddda12)] - fix: registry host config (#346) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`cd5bd92`](http://github.com/cnpm/cnpmcore/commit/cd5bd923b8d47bf90b5f077ce04777b38653b850)] - 🐛 FIX: Catch all error on changes stream handler (#344) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.4.0 / 2022-10-25
+==================
+
+**features**
+  * [[`6aa302d`](http://github.com/cnpm/cnpmcore/commit/6aa302d074f2c84f39e2065fa20853b007f6fa3b)] - 📦 NEW: Use oss-cnpm v4 (#340) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`a217fd0`](http://github.com/cnpm/cnpmcore/commit/a217fd07ccad3fe5058881654a13e0c69c758717)] - 👌 IMPROVE: Reduce warning log (#326) (fengmk2 <<fengmk2@gmail.com>>)
+
+**fixes**
+  * [[`b19b0a0`](http://github.com/cnpm/cnpmcore/commit/b19b0a0496e35ac1c6b3de746b9221990ba9dc93)] - fix: Lazy set registryId when executeTask (#341) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+**others**
+  * [[`305175a`](http://github.com/cnpm/cnpmcore/commit/305175ab5fcdc3ad3b60055d45cfcacb23065a80)] - 🤖 TEST: Use enum define on unittest (#333) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`07f2eba`](http://github.com/cnpm/cnpmcore/commit/07f2eba137ba625b2d422677a465920617141b87)] - 🤖 TEST: Mock all binary http requests (#328) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`4b0c7dc`](http://github.com/cnpm/cnpmcore/commit/4b0c7dc6196960d34b2529bfde724e97f1af8444)] - 🤖 TEST: Mock all httpclient request (#327) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.3.1 / 2022-10-06
+==================
+
+**features**
+  * [[`bbc08fd`](http://github.com/cnpm/cnpmcore/commit/bbc08fd26887d55b98b70d1ed210caf81f9d5c22)] - 👌 IMPROVE: syncPackageWorkerMaxConcurrentTasks up to 20 (#322) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`5852f22`](http://github.com/cnpm/cnpmcore/commit/5852f22023525d857ff1ceea205e4315c8079877)] - feat: support sync exist mode (#275) (zhangyuantao <<zhangyuantao@163.com>>)
+
+**fixes**
+  * [[`d79634e`](http://github.com/cnpm/cnpmcore/commit/d79634eea749fef1a420988a8599f156f28ee85a)] - 🐛 FIX: Should sync package when registry id is null (#324) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`24f920d`](http://github.com/cnpm/cnpmcore/commit/24f920d65b31f9eb83c1ecda36adf7f9e2c379c3)] - 🐛 FIX: Should run sync package on all worker (#323) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.3.0 / 2022-09-24
+==================
+
+**others**
+  * [[`bd83a19`](http://github.com/cnpm/cnpmcore/commit/bd83a19eca761c96bcee04e6ae91e68eac3cb6bf)] - 👌 IMPROVE: use urllib3 instead (#302) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`35e7d3a`](http://github.com/cnpm/cnpmcore/commit/35e7d3ad3c78712b507d522a0b72b5a6a5a4ec1c)] - 👌 IMPROVE: Enable phpmyadmin and DEBUG_LOCAL_SQL by default (#320) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.2.0 / 2022-09-22
+==================
+
+**features**
+  * [[`bca0fb3`](http://github.com/cnpm/cnpmcore/commit/bca0fb3c37b9f74f3c41ab181dd3113d9dab4c05)] - feat: only allow pkg sync from registry it belong (#317) (killa <<killa123@126.com>>)
+
+**fixes**
+  * [[`7e9beea`](http://github.com/cnpm/cnpmcore/commit/7e9beead576a41de3aa042b92b788bde5d55f44a)] - fix: only append / if path is not empty and not ends with / (#316) (killa <<killa123@126.com>>)
+  * [[`4fe68cb`](http://github.com/cnpm/cnpmcore/commit/4fe68cbf38f303e797b80b88407f714ec76bfae0)] - fix: fix directory path (#313) (killa <<killa123@126.com>>)
+
+**others**
+  * [[`e72ce35`](http://github.com/cnpm/cnpmcore/commit/e72ce3576f9a3cda095e3feac59eeb1d8c1e8033)] - 🤖 TEST: Skip unstable tests (#318) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`171b11f`](http://github.com/cnpm/cnpmcore/commit/171b11f7bba534c993af4088b00f8545216734a9)] - Revert "fix: fix directory path (#313)" (fengmk2 <<fengmk2@gmail.com>>)
+
+2.1.1 / 2022-09-08
+==================
+
+**fixes**
+  * [[`8fb9dd8`](http://github.com/cnpm/cnpmcore/commit/8fb9dd8cf4800afe3f54aba9ee4c0ae05efb4f1d)] - fix: findExecuteTask only return waiting task (#312) (killa <<killa123@126.com>>)
+
+2.1.0 / 2022-09-05
+==================
+
+**features**
+  * [[`c5d2b49`](http://github.com/cnpm/cnpmcore/commit/c5d2b49ab3a0ce0d67f6e7cc19e0be867c92d04c)] - feat: auto get next valid task (#311) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+2.0.0 / 2022-09-05
+==================
+
+**others**
+  * [[`fc4baff`](http://github.com/cnpm/cnpmcore/commit/fc4baff226540e7cfee9adc069e17a59f4050a43)] - chore: refactor schedule with @Schedule (#309) (killa <<killa123@126.com>>)
+
+1.11.6 / 2022-09-04
+==================
+
+**fixes**
+  * [[`768f951`](http://github.com/cnpm/cnpmcore/commit/768f951b6f2509f14c30a70d86a6719107d963a4)] - fix: cnpmjsorg changesstream limit (#310) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+1.11.5 / 2022-09-02
+==================
+
+**fixes**
+  * [[`f673ab8`](http://github.com/cnpm/cnpmcore/commit/f673ab8ba1545909ff6b8e445364646511930891)] - fix: execute state check (#308) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+**others**
+  * [[`091420a`](http://github.com/cnpm/cnpmcore/commit/091420ae2677ecedd1a26a238921321c2a191675)] - 🤖 TEST: Add SQL Review Action (#307) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.11.4 / 2022-08-30
+==================
+
+**fixes**
+  * [[`f9210ca`](http://github.com/cnpm/cnpmcore/commit/f9210ca7e180e19bce08da9ef33e46e990b86ef1)] - fix: changes stream empty (#306) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+1.11.3 / 2022-08-29
+==================
+
+**fixes**
+  * [[`48f228d`](http://github.com/cnpm/cnpmcore/commit/48f228da447d8cde62849fa52cf43bae7754e2e3)] - fix: changes stream updatedAt (#304) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`87045ba`](http://github.com/cnpm/cnpmcore/commit/87045ba8b0e14547c93689600eb7e2c1de2a611b)] - fix: task updatedAt save (#305) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+1.11.2 / 2022-08-28
+==================
+
+**fixes**
+  * [[`4e8700c`](http://github.com/cnpm/cnpmcore/commit/4e8700c4f7c6fb5c4f4d4a2b9a9546096c5d10e2)] - fix: only create createHookTask if hook enable (#299) (killa <<killa123@126.com>>)
+
+**others**
+  * [[`e06c841`](http://github.com/cnpm/cnpmcore/commit/e06c841537113fdb0c00beb22b0a55378c61ce80)] - 🐛 FIX: Should sync public package when registryName not exists (#303) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`f139444`](http://github.com/cnpm/cnpmcore/commit/f139444213403494ebe9bf073df62125413892d9)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`c4a9de5`](http://github.com/cnpm/cnpmcore/commit/c4a9de598dce9a1b82bbcdd91968a15bbc5a4b6b)] - Create SECURITY.md (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`709d65b`](http://github.com/cnpm/cnpmcore/commit/709d65bd0473856c9bfc4416ea2ca375136e354f)] - 🤖 TEST: Use diff bucket on OSS test (#301) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`9576699`](http://github.com/cnpm/cnpmcore/commit/95766990fa9c4c2c43d462f6b151557425b0c741)] - chore: use AsyncGenerator insteadof Transform stream (#300) (killa <<killa123@126.com>>)
+  * [[`3ed5269`](http://github.com/cnpm/cnpmcore/commit/3ed5269f1d22ca3aaca89a90a4fff90f293e2464)] - 📦 NEW: Mirror better-sqlite3 binary (#296) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.11.1 / 2022-08-24
+==================
+
+**fixes**
+  * [[`359a150`](http://github.com/cnpm/cnpmcore/commit/359a150eb450d69e6523b20efcc5c7cfe3efab4d)] - fix: changes stream (#297) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+1.11.0 / 2022-08-23
+==================
+
+**features**
+  * [[`a91c8ac`](http://github.com/cnpm/cnpmcore/commit/a91c8ac4d05dc903780fda516b09364a05a2b1e6)] - feat: sync package from spec regsitry (#293) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`de37008`](http://github.com/cnpm/cnpmcore/commit/de37008261b05845f392d66764cdfe14ae324756)] - feat: changesStream adapter & needSync() method (#292) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`4b506c8`](http://github.com/cnpm/cnpmcore/commit/4b506c8371697ddacdbe99a8ecb330bfc1911ec6)] - feat: init registry & scope (#286) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`41c6e24`](http://github.com/cnpm/cnpmcore/commit/41c6e24c84d546eb9d5515cc0940cc3e4274687b)] - feat: impl trigger Hooks (#289) (killa <<killa123@126.com>>)
+  * [[`79cb826`](http://github.com/cnpm/cnpmcore/commit/79cb82615f04bdb3da3ccbe09bb6a861608b69c5)] - feat: impl migration sql (#290) (killa <<killa123@126.com>>)
+  * [[`4cfa8ed`](http://github.com/cnpm/cnpmcore/commit/4cfa8ed9d687ce7d950d7d20c0ea28221763ba5f)] - feat: impl hooks api (#287) (killa <<killa123@126.com>>)
+  * [[`47d53d2`](http://github.com/cnpm/cnpmcore/commit/47d53d22ad03c02ee9cb9035a38ae205a6d38381)] - feat: add bizId for task (#285) (killa <<killa123@126.com>>)
+  * [[`3b1536b`](http://github.com/cnpm/cnpmcore/commit/3b1536b070b2f9062bc2cc377db96d2f4a160efc)] - feat: add node-webrtc mirror (#274) (Opportunity <<opportunity@live.in>>)
+
+**others**
+  * [[`7106807`](http://github.com/cnpm/cnpmcore/commit/710680742a078b2faf4cb18c3a39c0397308712e)] - 🐛 FIX: Should show queue size on logging (#280) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`3a41b21`](http://github.com/cnpm/cnpmcore/commit/3a41b2161cc99bb2f6f6dd7cbaa7abef25ff4393)] - 🐛 FIX: Handle binary configuration value (#278) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.10.0 / 2022-08-04
+==================
+
+**features**
+  * [[`c2b7d5a`](http://github.com/cnpm/cnpmcore/commit/c2b7d5aa98b5ba8649ec246c616574a22e9a74b8)] - feat: use sort set to impl queue (#277) (killa <<killa123@126.com>>)
+
+1.9.1 / 2022-07-29
+==================
+
+**fixes**
+  * [[`c54aa21`](http://github.com/cnpm/cnpmcore/commit/c54aa2165c3938dcbb5a2b3b54e66a0d961cc813)] - fix: check executingCount after task is done (#276) (killa <<killa123@126.com>>)
+
+**others**
+  * [[`3268d03`](http://github.com/cnpm/cnpmcore/commit/3268d030b620825c8c2e6331e1745c1788066c61)] - 🤖 TEST: show package not use cache if isSync (#273) (fengmk2 <<fengmk2@gmail.com>>)
+
+1.9.0 / 2022-07-25
+==================
+
+**features**
+  * [[`af6a75a`](http://github.com/cnpm/cnpmcore/commit/af6a75af32ea04c90fda82be3a56c99ec77e5807)] - feat: add forceSyncHistory options (#271) (killa <<killa123@126.com>>)
+
+1.8.0 / 2022-07-21
+==================
+
+**features**
+  * [[`b49a38c`](http://github.com/cnpm/cnpmcore/commit/b49a38c77e044c978e6de32a9d3e257cc90ea7c1)] - feat: use Model with inject (#269) (killa <<killa123@126.com>>)
+
+1.7.1 / 2022-07-20
+==================
+
+**fixes**
+  * [[`52fca55`](http://github.com/cnpm/cnpmcore/commit/52fca55aa883865f0ae70bfc1ff274c313b8f76a)] - fix: show package not use cache if isSync (#268) (killa <<killa123@126.com>>)
+
+1.7.0 / 2022-07-12
+==================
+
+**others**
+  * [[`4f7ce8b`](http://github.com/cnpm/cnpmcore/commit/4f7ce8b4b2a5806a225ce67228388e14388b7059)] - deps: upgrade leoric to 2.x (#262) (killa <<killa123@126.com>>)
+
 1.6.0 / 2022-07-11
 ==================
 

README.md

@@ -24,12 +24,12 @@ See [DEVELOPER.md](DEVELOPER.md)
 
 ## Contributors
 
-|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|[<img src="https://avatars.githubusercontent.com/u/13284978?v=4" width="100px;"/><br/><sub><b>Beace</b></sub>](https://github.com/Beace)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars.githubusercontent.com/u/17879221?v=4" width="100px;"/><br/><sub><b>laibao101</b></sub>](https://github.com/laibao101)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/5574625?v=4" width="100px;"/><br/><sub><b>elrrrrrrr</b></sub>](https://github.com/elrrrrrrr)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|[<img src="https://avatars.githubusercontent.com/u/4635838?v=4" width="100px;"/><br/><sub><b>gemwuu</b></sub>](https://github.com/gemwuu)<br/>|[<img src="https://avatars.githubusercontent.com/u/17879221?v=4" width="100px;"/><br/><sub><b>laibao101</b></sub>](https://github.com/laibao101)<br/>|
 | :---: | :---: | :---: | :---: | :---: | :---: |
-|[<img src="https://avatars.githubusercontent.com/u/8198408?v=4" width="100px;"/><br/><sub><b>BlackHole1</b></sub>](https://github.com/BlackHole1)<br/>|[<img src="https://avatars.githubusercontent.com/u/1814071?v=4" width="100px;"/><br/><sub><b>xiekw2010</b></sub>](https://github.com/xiekw2010)<br/>|[<img src="https://avatars.githubusercontent.com/u/958063?v=4" width="100px;"/><br/><sub><b>thonatos</b></sub>](https://github.com/thonatos)<br/>|[<img src="https://avatars.githubusercontent.com/u/11039003?v=4" width="100px;"/><br/><sub><b>chenpx976</b></sub>](https://github.com/chenpx976)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|[<img src="https://avatars.githubusercontent.com/u/1119126?v=4" width="100px;"/><br/><sub><b>looksgood</b></sub>](https://github.com/looksgood)<br/>|
-[<img src="https://avatars.githubusercontent.com/u/3478550?v=4" width="100px;"/><br/><sub><b>coolyuantao</b></sub>](https://github.com/coolyuantao)<br/>
+|[<img src="https://avatars.githubusercontent.com/u/13284978?v=4" width="100px;"/><br/><sub><b>Beace</b></sub>](https://github.com/Beace)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars.githubusercontent.com/u/3478550?v=4" width="100px;"/><br/><sub><b>coolyuantao</b></sub>](https://github.com/coolyuantao)<br/>|[<img src="https://avatars.githubusercontent.com/u/8198408?v=4" width="100px;"/><br/><sub><b>BlackHole1</b></sub>](https://github.com/BlackHole1)<br/>|[<img src="https://avatars.githubusercontent.com/u/1814071?v=4" width="100px;"/><br/><sub><b>xiekw2010</b></sub>](https://github.com/xiekw2010)<br/>|[<img src="https://avatars.githubusercontent.com/u/13471233?v=4" width="100px;"/><br/><sub><b>OpportunityLiu</b></sub>](https://github.com/OpportunityLiu)<br/>|
+[<img src="https://avatars.githubusercontent.com/u/958063?v=4" width="100px;"/><br/><sub><b>thonatos</b></sub>](https://github.com/thonatos)<br/>|[<img src="https://avatars.githubusercontent.com/u/11039003?v=4" width="100px;"/><br/><sub><b>chenpx976</b></sub>](https://github.com/chenpx976)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|[<img src="https://avatars.githubusercontent.com/u/1119126?v=4" width="100px;"/><br/><sub><b>looksgood</b></sub>](https://github.com/looksgood)<br/>
 
-This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sat Jul 09 2022 08:59:28 GMT+0800`.
+This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sat Dec 03 2022 14:59:50 GMT+0800`.
 
 <!-- GITCONTRIBUTOR_END -->
 

SECURITY.md

@@ -0,0 +1,41 @@
+# Security Policy
+
+## Supported Versions
+
+Currently being supported with security updates.
+
+| Version  | Supported          |
+| -------- | ------------------ |
+| >= 1.0.0 | :white_check_mark: |
+
+## Reporting a Vulnerability
+
+The cnpmcore OSS team and community take all security vulnerabilities seriously.
+Thank you for improving the security of our open source software.
+We appreciate your efforts and responsible disclosure and will make every effort to acknowledge your contributions.
+
+Report security vulnerabilities by emailing the cnpmcore security team at:
+
+```
+fengmk2+cnpmcoresecurity@gmail.com
+killa123@126.com
+```
+
+The lead maintainer will acknowledge your email within 48 hours,
+and will send a more detailed response within 72 hours indicating the next steps in handling your report.
+After the initial reply to your report,
+the security team will endeavor to keep you informed of the progress towards a fix and full announcement,
+and may ask for additional information or guidance.
+
+Report security vulnerabilities in third-party modules to the person or team maintaining the module.
+
+## Disclosure Policy
+
+When the security team receives a security bug report, they will assign it
+to a primary handler. This person will coordinate the fix and release
+process, involving the following steps:
+
+  * Confirm the problem and determine the affected versions.
+  * Audit code to find any potential similar problems.
+  * Prepare fixes for all releases still under maintenance. These fixes
+    will be released as fast as possible to NPM.

app.ts

@@ -1,7 +1,6 @@
 import path from 'path';
 import { readFile } from 'fs/promises';
 import { Application } from 'egg';
-
 declare module 'egg' {
   interface Application {
     binaryHTML: string;
@@ -23,4 +22,14 @@ export default class CnpmcoreAppHook {
     const text = await readFile(filepath, 'utf-8');
     this.app.binaryHTML = text.replace('{{registry}}', this.app.config.cnpmcore.registry);
   }
+
+  // 应用退出时执行
+  // 需要暂停当前执行的 changesStream task
+  async beforeClose() {
+    await this.app.runInAnonymousContextScope(async ctx => {
+      await ctx.beginModuleScope(async () => {
+        await ctx.module.cnpmcoreCore.changesStreamService.suspendTaskWhenExit();
+      });
+    });
+  }
 }

app/common/FileUtil.ts

@@ -31,7 +31,8 @@ export async function downloadToTempfile(httpclient: EggContextHttpClient,
     retries--;
     if (retries > 0) {
       // sleep 1s ~ 4s in random
-      await setTimeout(1000 + Math.random() * 4000);
+      const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000;
+      await setTimeout(delay);
     }
   }
   throw lastError;

app/common/LogUtil.ts

@@ -0,0 +1,3 @@
+export function isoNow() {
+  return new Date().toISOString();
+}

app/common/SyncUtil.ts

@@ -6,7 +6,7 @@ export function isSyncWorkerRequest(ctx: EggContext) {
   if (!isSyncWorkerRequest) {
     const ua = ctx.headers['user-agent'] || '';
     // old sync client will request with these user-agent
-    if (ua.indexOf('npm_service.cnpmjs.org/') !== -1) {
+    if (ua.includes('npm_service.cnpmjs.org/')) {
       isSyncWorkerRequest = true;
     }
   }

app/common/adapter/NPMRegistry.ts

@@ -9,7 +9,7 @@ import {
   EggContextHttpClient,
   EggAppConfig,
 } from 'egg';
-import { HttpMethod } from 'urllib';
+import { HttpMethod } from 'urllib/src/Request';
 
 const INSTANCE_NAME = 'npmRegistry';
 
@@ -25,9 +25,14 @@ export class NPMRegistry {
   @Inject()
   private config: EggAppConfig;
   private timeout = 10000;
+  public registryHost: string;
 
   get registry(): string {
-    return this.config.cnpmcore.sourceRegistry;
+    return this.registryHost || this.config.cnpmcore.sourceRegistry;
+  }
+
+  public setRegistryHost(registryHost = '') {
+    this.registryHost = registryHost;
   }
 
   public async getFullManifests(fullname: string, retries = 3) {
@@ -47,7 +52,8 @@ export class NPMRegistry {
       retries--;
       if (retries > 0) {
         // sleep 1s ~ 4s in random
-        await setTimeout(1000 + Math.random() * 4000);
+        const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000;
+        await setTimeout(delay);
       }
     }
     throw lastError;

app/common/adapter/QueueAdapter.ts

@@ -1,32 +0,0 @@
-import {
-  AccessLevel,
-  Inject,
-  ContextProto,
-} from '@eggjs/tegg';
-import { Redis } from 'ioredis';
-
-@ContextProto({
-  accessLevel: AccessLevel.PUBLIC,
-})
-export class QueueAdapter {
-  @Inject()
-  private readonly redis: Redis;
-
-  private getQueueName(key: string) {
-    return `CNPMCORE_Q_${key}`;
-  }
-
-  async push<T>(key: string, item: T) {
-    return await this.redis.lpush(this.getQueueName(key), JSON.stringify(item));
-  }
-
-  async pop<T>(key: string) {
-    const json = await this.redis.rpop(this.getQueueName(key));
-    if (!json) return null;
-    return JSON.parse(json) as T;
-  }
-
-  async length(key: string) {
-    return await this.redis.llen(this.getQueueName(key));
-  }
-}

app/common/adapter/binary/AbstractBinary.ts

@@ -19,11 +19,13 @@ export abstract class AbstractBinary {
   protected httpclient: EggContextHttpClient;
   protected logger: EggLogger;
   protected binaryConfig: BinaryTaskConfig;
+  protected binaryName: string;
 
-  constructor(httpclient: EggContextHttpClient, logger: EggLogger, binaryConfig: BinaryTaskConfig) {
+  constructor(httpclient: EggContextHttpClient, logger: EggLogger, binaryConfig: BinaryTaskConfig, binaryName: string) {
     this.httpclient = httpclient;
     this.logger = logger;
     this.binaryConfig = binaryConfig;
+    this.binaryName = binaryName;
   }
 
   abstract fetch(dir: string, params?: any): Promise<FetchResult | undefined>;

app/common/adapter/binary/ApiBinary.ts

@@ -4,13 +4,13 @@ import { BinaryTaskConfig } from '../../../../config/binaries';
 
 export class ApiBinary extends AbstractBinary {
   private apiUrl: string;
-  constructor(httpclient: EggContextHttpClient, logger: EggLogger, binaryConfig: BinaryTaskConfig, apiUrl: string) {
-    super(httpclient, logger, binaryConfig);
+  constructor(httpclient: EggContextHttpClient, logger: EggLogger, binaryConfig: BinaryTaskConfig, apiUrl: string, binaryName: string) {
+    super(httpclient, logger, binaryConfig, binaryName);
     this.apiUrl = apiUrl;
   }
 
   async fetch(dir: string): Promise<FetchResult | undefined> {
-    const url = `${this.apiUrl}/${this.binaryConfig.category}${dir}`;
+    const url = `${this.apiUrl}/${this.binaryName}${dir}`;
     const data = await this.requestJSON(url);
     if (!Array.isArray(data)) {
       this.logger.warn('[ApiBinary.fetch:response-data-not-array] data: %j', data);

app/common/adapter/binary/CypressBinary.ts

@@ -31,10 +31,11 @@ export class CypressBinary extends AbstractBinary {
         // "https://cdn.cypress.io/desktop/4.0.0/darwin-x64/cypress.zip"
         // "https://cdn.cypress.io/desktop/4.0.0/linux-x64/cypress.zip"
         // "https://cdn.cypress.io/desktop/4.0.0/win32-x64/cypress.zip"
+        // "https://cdn.cypress.io/desktop/9.2.0/darwin-arm64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/darwin-x64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/linux-x64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/win32-x64/cypress.zip"
-        const platforms = [ 'darwin-x64', 'linux-x64', 'win32-x64' ];
+        const platforms = [ 'darwin-x64', 'darwin-arm64', 'linux-x64', 'win32-x64' ];
         for (const platform of platforms) {
           this.dirItems[subDir].push({
             name: `${platform}/`,

app/common/adapter/binary/GithubBinary.ts

@@ -14,6 +14,11 @@ export class GithubBinary extends AbstractBinary {
         const url = `https://api.github.com/repos/${this.binaryConfig.repo}/releases?per_page=100&page=${i + 1}`;
         const data = await this.requestJSON(url);
         if (!Array.isArray(data)) {
+          // {"message":"API rate limit exceeded for 47.57.239.54. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}
+          if (typeof data?.message === 'string' && data.message.includes('rate limit')) {
+            this.logger.info('[GithubBinary.fetch:hit-rate-limit] skip sync this time, data: %j, url: %s', data, url);
+            return;
+          }
           this.logger.warn('[GithubBinary.fetch:response-data-not-array] data: %j, url: %s', data, url);
           return;
         }

app/common/adapter/binary/ImageminBinary.ts

@@ -8,7 +8,7 @@ export class ImageminBinary extends AbstractBinary {
   async fetch(dir: string): Promise<FetchResult | undefined> {
     if (!this.dirItems) {
       this.dirItems = {};
-      const npmPackageName = this.binaryConfig.options?.npmPackageName ?? this.binaryConfig.category;
+      const npmPackageName = this.binaryConfig.options?.npmPackageName ?? this.binaryName;
       const pkgUrl = `https://registry.npmjs.com/${npmPackageName}`;
       const data = await this.requestJSON(pkgUrl);
       this.dirItems = {};

app/common/adapter/binary/NodePreGypBinary.ts

@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { AbstractBinary, FetchResult, BinaryItem } from './AbstractBinary';
 
 export class NodePreGypBinary extends AbstractBinary {
@@ -9,7 +10,7 @@ export class NodePreGypBinary extends AbstractBinary {
   async fetch(dir: string): Promise<FetchResult | undefined> {
     if (!this.dirItems) {
       this.dirItems = {};
-      const pkgUrl = `https://registry.npmjs.com/${this.binaryConfig.category}`;
+      const pkgUrl = `https://registry.npmjs.com/${this.binaryName}`;
       const data = await this.requestJSON(pkgUrl);
       this.dirItems = {};
       this.dirItems['/'] = [];
@@ -32,7 +33,7 @@ export class NodePreGypBinary extends AbstractBinary {
 
         let currentDir = this.dirItems['/'];
         let versionPrefix = '';
-        const remotePath = pkgVersion.binary.remote_path;
+        let remotePath = pkgVersion.binary.remote_path;
         const napiVersions = pkgVersion.binary.napi_versions ?? [];
         if (this.binaryConfig.options?.requiredNapiVersions && napiVersions.length === 0) continue;
         if (remotePath?.includes('{version}')) {
@@ -76,7 +77,7 @@ export class NodePreGypBinary extends AbstractBinary {
                     date,
                     size: '-',
                     isDir: false,
-                    url: `${this.binaryConfig.distUrl}/${this.binaryConfig.category}${versionPrefix}/${name}`,
+                    url: `${this.binaryConfig.distUrl}/${this.binaryName}${versionPrefix}/${name}`,
                     ignoreDownloadStatuses: [ 404 ],
                   });
                 }
@@ -98,7 +99,7 @@ export class NodePreGypBinary extends AbstractBinary {
                   date,
                   size: '-',
                   isDir: false,
-                  url: `${this.binaryConfig.distUrl}/${this.binaryConfig.category}${versionPrefix}/${name}`,
+                  url: `${this.binaryConfig.distUrl}/${this.binaryName}${versionPrefix}/${name}`,
                   ignoreDownloadStatuses: [ 404 ],
                 });
               }
@@ -148,17 +149,31 @@ export class NodePreGypBinary extends AbstractBinary {
           //   "package_name": "{platform}-{arch}.tar.gz",
           //   "module_path": "bin"
           // },
+          // handle {configuration}
+          // "binary": {
+          //   "module_name": "wrtc",
+          //   "module_path": "./build/{configuration}/",
+          //   "remote_path": "./{module_name}/v{version}/{configuration}/",
+          //   "package_name": "{platform}-{arch}.tar.gz",
+          //   "host": "https://node-webrtc.s3.amazonaws.com"
+          // },
           for (const platform of nodePlatforms) {
             const archs = nodeArchs[platform];
             for (const arch of archs) {
-              const name = binaryFile.replace('{platform}', platform)
+              const binaryFileName = binaryFile.replace('{platform}', platform)
                 .replace('{arch}', arch);
+              remotePath = remotePath.replace('{module_name}', moduleName)
+                .replace('{name}', this.binaryName)
+                .replace('{version}', version)
+                .replace('{configuration}', 'Release');
+              const binaryFilePath = join('/', remotePath, binaryFileName);
+              const remoteUrl = `${this.binaryConfig.distUrl}${binaryFilePath}`;
               currentDir.push({
-                name,
+                name: binaryFileName,
                 date,
                 size: '-',
                 isDir: false,
-                url: `${this.binaryConfig.distUrl}/${this.binaryConfig.category}${versionPrefix}/${name}`,
+                url: remoteUrl,
                 ignoreDownloadStatuses: [ 404 ],
               });
             }

app/common/adapter/binary/PuppeteerBinary.ts

@@ -27,7 +27,7 @@ export class PuppeteerBinary extends AbstractBinary {
       //   chromium: '768783',
       //   firefox: 'latest',
       // };
-      const unpkgURL = 'https://unpkg.com/puppeteer@latest/lib/cjs/puppeteer/revisions.js';
+      const unpkgURL = 'https://unpkg.com/puppeteer-core@latest/lib/cjs/puppeteer/revisions.js';
       const text = await this.requestXml(unpkgURL);
       const m = /chromium:\s+\'(\d+)\'\,/.exec(text);
       if (m && !chromiumRevisions.has(m[1])) {

app/common/adapter/changesStream/AbstractChangesStream.ts

@@ -0,0 +1,40 @@
+import {
+  ImplDecorator,
+  Inject,
+  QualifierImplDecoratorUtil,
+} from '@eggjs/tegg';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import {
+  EggHttpClient,
+  EggLogger,
+} from 'egg';
+
+export const CHANGE_STREAM_ATTRIBUTE = 'CHANGE_STREAM_ATTRIBUTE';
+export type ChangesStreamChange = {
+  seq: string;
+  fullname: string;
+};
+
+export abstract class AbstractChangeStream {
+  @Inject()
+  protected logger: EggLogger;
+
+  @Inject()
+  protected httpclient: EggHttpClient;
+
+  abstract getInitialSince(registry: Registry): Promise<string>;
+  abstract fetchChanges(registry: Registry, since: string): AsyncGenerator<ChangesStreamChange>;
+
+  getChangesStreamUrl(registry: Registry, since: string, limit?: number): string {
+    const url = new URL(registry.changeStream);
+    url.searchParams.set('since', since);
+    if (limit) {
+      url.searchParams.set('limit', String(limit));
+    }
+    return url.toString();
+  }
+}
+
+export const RegistryChangesStream: ImplDecorator<AbstractChangeStream, typeof RegistryType> =
+  QualifierImplDecoratorUtil.generatorDecorator(AbstractChangeStream, CHANGE_STREAM_ATTRIBUTE);

app/common/adapter/changesStream/CnpmcoreChangesStream.ts

@@ -0,0 +1,52 @@
+import { ContextProto } from '@eggjs/tegg';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { E500 } from 'egg-errors';
+import { AbstractChangeStream, RegistryChangesStream } from './AbstractChangesStream';
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Cnpmcore)
+export class CnpmcoreChangesStream extends AbstractChangeStream {
+
+  async getInitialSince(registry: Registry): Promise<string> {
+    const db = (new URL(registry.changeStream)).origin;
+    const { status, data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 10000,
+      dataType: 'json',
+    });
+    if (!data.update_seq) {
+      throw new E500(`get getInitialSince failed: ${data.update_seq}`);
+    }
+    const since = String(data.update_seq - 10);
+    this.logger.warn('[NpmChangesStream.getInitialSince:firstSeq] GET %s status: %s, data: %j, since: %s',
+      registry.name, status, data, since);
+    return since;
+  }
+
+  async* fetchChanges(registry: Registry, since: string) {
+    const db = this.getChangesStreamUrl(registry, since);
+    // json mode
+    const { data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 30000,
+      dataType: 'json',
+      gzip: true,
+    });
+
+    if (data.results?.length > 0) {
+      for (const change of data.results) {
+        const seq = String(change.seq);
+        const fullname = change.id;
+        // cnpmcore 默认返回 >= 需要做特殊判断
+        if (seq && fullname && seq !== since) {
+          const change = {
+            fullname,
+            seq,
+          };
+          yield change;
+        }
+      }
+    }
+  }
+}

app/common/adapter/changesStream/CnpmjsorgChangesStream.ts

@@ -0,0 +1,65 @@
+import { ContextProto } from '@eggjs/tegg';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { E500 } from 'egg-errors';
+import { AbstractChangeStream, RegistryChangesStream } from './AbstractChangesStream';
+
+const MAX_LIMIT = 10000;
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Cnpmjsorg)
+export class CnpmjsorgChangesStream extends AbstractChangeStream {
+
+  // cnpmjsorg 未实现 update_seq 字段
+  // 默认返回当前时间戳字符串
+  async getInitialSince(registry: Registry): Promise<string> {
+    const since = String((new Date()).getTime());
+    this.logger.warn(`[CnpmjsorgChangesStream.getInitialSince] since: ${since}, skip query ${registry.changeStream}`);
+    return since;
+  }
+
+  private async tryFetch(registry: Registry, since: string, limit = 1000) {
+    if (limit > MAX_LIMIT) {
+      throw new E500(`limit too large, current since: ${since}, limit: ${limit}`);
+    }
+    const db = this.getChangesStreamUrl(registry, since, limit);
+    // json mode
+    const res = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 30000,
+      dataType: 'json',
+      gzip: true,
+    });
+    const { results = [] } = res.data;
+    if (results?.length >= limit) {
+      const [ first ] = results;
+      const last = results[results.length - 1];
+      if (first.gmt_modified === last.gmt_modified) {
+        return await this.tryFetch(registry, since, limit + 1000);
+      }
+    }
+
+    return res;
+  }
+
+  async* fetchChanges(registry: Registry, since: string) {
+    // ref: https://github.com/cnpm/cnpmjs.org/pull/1734
+    // 由于 cnpmjsorg 无法计算准确的 seq
+    // since 是一个时间戳，需要确保一次返回的结果中首尾两个 gmtModified 不相等
+    const { data } = await this.tryFetch(registry, since);
+
+    if (data.results?.length > 0) {
+      for (const change of data.results) {
+        const seq = new Date(change.gmt_modified).getTime() + '';
+        const fullname = change.id;
+        if (seq && fullname && seq !== since) {
+          const change = {
+            fullname,
+            seq,
+          };
+          yield change;
+        }
+      }
+    }
+  }
+}

app/common/adapter/changesStream/NpmChangesStream.ts

@@ -0,0 +1,55 @@
+import { ContextProto } from '@eggjs/tegg';
+import { E500 } from 'egg-errors';
+import { RegistryType } from '../../../common/enum/Registry';
+import { Registry } from '../../../core/entity/Registry';
+import { AbstractChangeStream, ChangesStreamChange, RegistryChangesStream } from './AbstractChangesStream';
+
+@ContextProto()
+@RegistryChangesStream(RegistryType.Npm)
+export class NpmChangesStream extends AbstractChangeStream {
+
+  async getInitialSince(registry: Registry): Promise<string> {
+    const db = (new URL(registry.changeStream)).origin;
+    const { status, data } = await this.httpclient.request(db, {
+      followRedirect: true,
+      timeout: 10000,
+      dataType: 'json',
+    });
+    const since = String(data.update_seq - 10);
+    if (!data.update_seq) {
+      throw new E500(`get getInitialSince failed: ${data.update_seq}`);
+    }
+    this.logger.warn('[NpmChangesStream.getInitialSince] GET %s status: %s, data: %j, since: %s',
+      registry.name, registry.changeStream, status, data, since);
+    return since;
+  }
+
+  async* fetchChanges(registry: Registry, since: string) {
+    const db = this.getChangesStreamUrl(registry, since);
+    const { res } = await this.httpclient.request(db, {
+      streaming: true,
+      timeout: 10000,
+    });
+
+    let buf = '';
+    for await (const chunk of res) {
+      const text = chunk.toString();
+      const lines = text.split('\n');
+
+      for (const line of lines) {
+        const content = buf + line;
+        const match = /"seq":(\d+),"id":"([^"]+)"/g.exec(content);
+        const seq = match?.[1];
+        const fullname = match?.[2];
+        if (seq && fullname) {
+          buf = '';
+          const change: ChangesStreamChange = { fullname, seq };
+          yield change;
+        } else {
+          buf += line;
+        }
+      }
+    }
+  }
+
+}

app/common/enum/Hook.ts

@@ -0,0 +1,19 @@
+export enum HookType {
+  Package = 'package',
+  Scope = 'scope',
+  Owner = 'owner',
+}
+
+export enum HookEventType {
+  Star = 'package:star',
+  Unstar = 'package:unstar',
+  Publish = 'package:publish',
+  Unpublish = 'package:unpublish',
+  Owner = 'package:owner',
+  OwnerRm = 'package:owner-rm',
+  DistTag = 'package:dist-tag',
+  DistTagRm = 'package:dist-tag-rm',
+  Deprecated = 'package:deprecated',
+  Undeprecated = 'package:undeprecated',
+  Change = 'package:change',
+}

app/common/enum/Registry.ts

@@ -0,0 +1,5 @@
+export enum RegistryType {
+  Npm = 'npm',
+  Cnpmcore = 'cnpmcore',
+  Cnpmjsorg = 'cnpmjsorg',
+}

app/common/enum/Task.ts

@@ -2,6 +2,8 @@ export enum TaskType {
   SyncPackage = 'sync_package',
   ChangesStream = 'changes_stream',
   SyncBinary = 'sync_binary',
+  CreateHook = 'create_hook',
+  TriggerHook = 'trigger_hook',
 }
 
 export enum TaskState {

app/common/typing.ts

@@ -35,3 +35,9 @@ export interface NFSClient {
 
   url?(key: string): string;
 }
+
+export interface QueueAdapter {
+  push<T>(key: string, item: T): Promise<boolean>;
+  pop<T>(key: string): Promise<T | null>;
+  length(key: string): Promise<number>;
+}

app/core/entity/Hook.ts

@@ -0,0 +1,61 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+import { HookType } from '../../common/enum/Hook';
+import crypto from 'crypto';
+
+export type CreateHookData = Omit<EasyData<HookData, 'hookId'>, 'enable' | 'latestTaskId'>;
+
+export interface HookData extends EntityData {
+  hookId: string;
+  type: HookType;
+  ownerId: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  latestTaskId?: string;
+  enable: boolean;
+}
+
+export class Hook extends Entity {
+  readonly hookId: string;
+  readonly type: HookType;
+  readonly ownerId: string;
+  readonly name: string;
+  endpoint: string;
+  secret: string;
+  enable: boolean;
+  latestTaskId?: string;
+
+  constructor(data: HookData) {
+    super(data);
+    this.hookId = data.hookId;
+    this.type = data.type;
+    this.ownerId = data.ownerId;
+    this.name = data.name;
+    this.endpoint = data.endpoint;
+    this.secret = data.secret;
+    this.latestTaskId = data.latestTaskId;
+    this.enable = data.enable;
+  }
+
+  static create(data: CreateHookData): Hook {
+    const hookData: EasyData<HookData, 'hookId'> = Object.assign({}, data, {
+      enable: true,
+      latestTaskId: undefined,
+    });
+    const newData = EntityUtil.defaultData(hookData, 'hookId');
+    return new Hook(newData);
+  }
+
+  // payload 可能会特别大，如果做多次 stringify 浪费太多 cpu
+  signPayload(payload: object): { digest, payloadStr } {
+    const payloadStr = JSON.stringify(payload);
+    const digest = crypto.createHmac('sha256', this.secret)
+      .update(JSON.stringify(payload))
+      .digest('hex');
+    return {
+      digest,
+      payloadStr,
+    };
+  }
+}

app/core/entity/HookEvent.ts

@@ -0,0 +1,93 @@
+import { HookEventType } from '../../common/enum/Hook';
+
+export interface PublishChangePayload {
+  'dist-tag'?: string;
+  version: string;
+}
+
+export interface UnpublishChangePayload {
+  'dist-tag'?: string;
+  version?: string;
+}
+
+export interface DistTagChangePayload {
+  'dist-tag': string;
+}
+
+export interface PackageOwnerPayload {
+  maintainer: string;
+}
+
+export interface DeprecatedChangePayload {
+  deprecated: string;
+}
+
+export class HookEvent<T = object> {
+  readonly changeId: string;
+  readonly event: HookEventType;
+  readonly fullname: string;
+  readonly type: 'package';
+  readonly version: '1.0.0';
+  readonly change: T;
+  readonly time: number;
+
+  constructor(event: HookEventType, changeId: string, fullname: string, change: T) {
+    this.changeId = changeId;
+    this.event = event;
+    this.fullname = fullname;
+    this.type = 'package';
+    this.version = '1.0.0';
+    this.change = change;
+    this.time = Date.now();
+  }
+
+  static createPublishEvent(fullname: string, changeId: string, version: string, distTag?: string): HookEvent<PublishChangePayload> {
+    return new HookEvent(HookEventType.Publish, changeId, fullname, {
+      'dist-tag': distTag,
+      version,
+    });
+  }
+
+  static createUnpublishEvent(fullname: string, changeId: string, version?: string, distTag?: string): HookEvent<UnpublishChangePayload> {
+    return new HookEvent(HookEventType.Unpublish, changeId, fullname, {
+      'dist-tag': distTag,
+      version,
+    });
+  }
+
+  static createOwnerEvent(fullname: string, changeId: string, maintainer: string): HookEvent<PackageOwnerPayload> {
+    return new HookEvent(HookEventType.Owner, changeId, fullname, {
+      maintainer,
+    });
+  }
+
+  static createOwnerRmEvent(fullname: string, changeId: string, maintainer: string): HookEvent<PackageOwnerPayload> {
+    return new HookEvent(HookEventType.OwnerRm, changeId, fullname, {
+      maintainer,
+    });
+  }
+
+  static createDistTagEvent(fullname: string, changeId: string, distTag: string): HookEvent<DistTagChangePayload> {
+    return new HookEvent(HookEventType.DistTag, changeId, fullname, {
+      'dist-tag': distTag,
+    });
+  }
+
+  static createDistTagRmEvent(fullname: string, changeId: string, distTag: string): HookEvent<DistTagChangePayload> {
+    return new HookEvent(HookEventType.DistTagRm, changeId, fullname, {
+      'dist-tag': distTag,
+    });
+  }
+
+  static createDeprecatedEvent(fullname: string, changeId: string, deprecated: string): HookEvent<DeprecatedChangePayload> {
+    return new HookEvent(HookEventType.Deprecated, changeId, fullname, {
+      deprecated,
+    });
+  }
+
+  static createUndeprecatedEvent(fullname: string, changeId: string, deprecated: string): HookEvent<DeprecatedChangePayload> {
+    return new HookEvent(HookEventType.Undeprecated, changeId, fullname, {
+      deprecated,
+    });
+  }
+}

app/core/entity/Package.ts

@@ -11,6 +11,7 @@ interface PackageData extends EntityData {
   description: string;
   abbreviatedsDist?: Dist;
   manifestsDist?: Dist;
+  registryId?: string;
 }
 
 export enum DIST_NAMES {
@@ -36,6 +37,7 @@ export class Package extends Entity {
   description: string;
   abbreviatedsDist?: Dist;
   manifestsDist?: Dist;
+  registryId?: string;
 
   constructor(data: PackageData) {
     super(data);
@@ -46,6 +48,7 @@ export class Package extends Entity {
     this.description = data.description;
     this.abbreviatedsDist = data.abbreviatedsDist;
     this.manifestsDist = data.manifestsDist;
+    this.registryId = data.registryId;
   }
 
   static create(data: EasyData<PackageData, 'packageId'>): Package {

app/core/entity/Registry.ts

@@ -0,0 +1,38 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+import type { RegistryType } from '../../common/enum/Registry';
+
+interface RegistryData extends EntityData {
+  name: string;
+  registryId: string;
+  host: string;
+  changeStream: string;
+  userPrefix: string;
+  type: RegistryType;
+}
+
+export type CreateRegistryData = Omit<EasyData<RegistryData, 'registryId'>, 'id'>;
+
+export class Registry extends Entity {
+  name: string;
+  registryId: string;
+  host: string;
+  changeStream: string;
+  userPrefix: string;
+  type: RegistryType;
+
+  constructor(data: RegistryData) {
+    super(data);
+    this.name = data.name;
+    this.registryId = data.registryId;
+    this.host = data.host;
+    this.changeStream = data.changeStream;
+    this.userPrefix = data.userPrefix;
+    this.type = data.type;
+  }
+
+  public static create(data: CreateRegistryData): Registry {
+    const newData = EntityUtil.defaultData(data, 'registryId');
+    return new Registry(newData);
+  }
+}

app/core/entity/Scope.ts

@@ -0,0 +1,28 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+
+interface ScopeData extends EntityData {
+  name: string;
+  scopeId: string;
+  registryId: string;
+}
+
+export type CreateScopeData = Omit<EasyData<ScopeData, 'scopeId'>, 'id'>;
+
+export class Scope extends Entity {
+  name: string;
+  registryId: string;
+  scopeId: string;
+
+  constructor(data: ScopeData) {
+    super(data);
+    this.name = data.name;
+    this.registryId = data.registryId;
+    this.scopeId = data.scopeId;
+  }
+
+  static create(data: CreateScopeData): Scope {
+    const newData = EntityUtil.defaultData(data, 'scopeId');
+    return new Scope(newData);
+  }
+}

app/core/entity/Task.ts

@@ -4,19 +4,28 @@ import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
 import { TaskType, TaskState } from '../../common/enum/Task';
 import dayjs from '../../common/dayjs';
+import { HookEvent } from './HookEvent';
 
-interface TaskData extends EntityData {
+export const HOST_NAME = os.hostname();
+export const PID = process.pid;
+
+export interface TaskBaseData {
+  taskWorker: string;
+}
+
+export interface TaskData<T = TaskBaseData> extends EntityData {
   taskId: string;
   type: TaskType;
   state: TaskState;
   targetName: string;
   authorId: string;
   authorIp: string;
-  data: any;
+  data: T;
   logPath?: string;
   logStorePosition?: string;
   attempts?: number;
   error?: string;
+  bizId?: string;
 }
 
 export type SyncPackageTaskOptions = {
@@ -25,22 +34,62 @@ export type SyncPackageTaskOptions = {
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
+  // force sync history version
+  forceSyncHistory?: boolean;
+  registryId?: string;
 };
 
-export class Task extends Entity {
+export interface CreateHookTaskData extends TaskBaseData {
+  hookEvent: HookEvent;
+}
+
+export interface TriggerHookTaskData extends TaskBaseData {
+  hookEvent: HookEvent;
+  hookId: string;
+  responseStatus?: number;
+}
+
+export interface CreateSyncPackageTaskData extends TaskBaseData {
+  tips?: string;
+  skipDependencies?: boolean;
+  syncDownloadData?: boolean;
+  forceSyncHistory?: boolean;
+}
+
+export interface ChangesStreamTaskData extends TaskBaseData {
+  since: string;
+  last_package?: string,
+  last_package_created?: Date,
+  task_count?: number,
+  registryId?: string,
+}
+
+export interface TaskUpdateCondition {
+  taskId: string;
+  attempts: number;
+}
+
+export type CreateHookTask = Task<CreateHookTaskData>;
+export type TriggerHookTask = Task<TriggerHookTaskData>;
+export type CreateSyncPackageTask = Task<CreateSyncPackageTaskData>;
+export type ChangesStreamTask = Task<ChangesStreamTaskData>;
+
+export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
   taskId: string;
   type: TaskType;
   state: TaskState;
   targetName: string;
+  taskWorker: string;
   authorId: string;
   authorIp: string;
-  data: any;
+  data: T;
   logPath: string;
   logStorePosition: string;
   attempts: number;
   error: string;
+  bizId?: string;
 
-  constructor(data: TaskData) {
+  constructor(data: TaskData<T>) {
     super(data);
     this.taskId = data.taskId;
     this.type = data.type;
@@ -53,6 +102,7 @@ export class Task extends Entity {
     this.logStorePosition = data.logStorePosition ?? '';
     this.attempts = data.attempts ?? 0;
     this.error = data.error ?? '';
+    this.bizId = data.bizId;
   }
 
   public resetLogPath() {
@@ -61,15 +111,15 @@ export class Task extends Entity {
   }
 
   public setExecuteWorker() {
-    this.data.taskWorker = `${os.hostname()}:${process.pid}`;
+    this.data.taskWorker = `${HOST_NAME}:${PID}`;
   }
 
-  private static create(data: EasyData<TaskData, 'taskId'>): Task {
+  private static create<T extends TaskBaseData>(data: EasyData<TaskData<T>, 'taskId'>): Task<T> {
     const newData = EntityUtil.defaultData(data, 'taskId');
     return new Task(newData);
   }
 
-  public static createSyncPackage(fullname: string, options?: SyncPackageTaskOptions): Task {
+  public static createSyncPackage(fullname: string, options?: SyncPackageTaskOptions): CreateSyncPackageTask {
     const data = {
       type: TaskType.SyncPackage,
       state: TaskState.Waiting,
@@ -80,8 +130,10 @@ export class Task extends Entity {
         // task execute worker
         taskWorker: '',
         tips: options?.tips,
+        registryId: options?.registryId ?? '',
         skipDependencies: options?.skipDependencies,
         syncDownloadData: options?.syncDownloadData,
+        forceSyncHistory: options?.forceSyncHistory,
       },
     };
     const task = this.create(data);
@@ -89,20 +141,72 @@ export class Task extends Entity {
     return task;
   }
 
-  public static createChangesStream(targetName: string): Task {
+  public static createChangesStream(targetName: string, registryId = '', since = ''): ChangesStreamTask {
     const data = {
       type: TaskType.ChangesStream,
       state: TaskState.Waiting,
       targetName,
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
+      data: {
+        // task execute worker
+        taskWorker: '',
+        registryId,
+        since,
+      },
+    };
+    return this.create(data) as ChangesStreamTask;
+  }
+
+  public updateSyncData({ lastSince, taskCount, lastPackage }: SyncInfo) {
+    const syncData = this.data as unknown as ChangesStreamTaskData;
+    // 更新任务记录信息
+    syncData.since = lastSince;
+    syncData.task_count = (syncData.task_count || 0) + taskCount;
+
+    if (taskCount > 0) {
+      syncData.last_package = lastPackage;
+      syncData.last_package_created = new Date();
+    }
+  }
+
+  public static createCreateHookTask(hookEvent: HookEvent): CreateHookTask {
+    const data = {
+      type: TaskType.CreateHook,
+      state: TaskState.Waiting,
+      targetName: hookEvent.fullname,
       authorId: `pid_${process.pid}`,
       authorIp: os.hostname(),
+      bizId: `CreateHook:${hookEvent.changeId}`,
+      data: {
+        // task execute worker
+        taskWorker: '',
+        hookEvent,
+      },
+    };
+    const task = this.create(data);
+    task.logPath = `/packages/${hookEvent.fullname}/hooks/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
+  }
+
+  public static createTriggerHookTask(hookEvent: HookEvent, hookId: string): TriggerHookTask {
+    const data = {
+      type: TaskType.TriggerHook,
+      state: TaskState.Waiting,
+      targetName: hookEvent.fullname,
+      authorId: `pid_${process.pid}`,
+      bizId: `TriggerHook:${hookEvent.changeId}:${hookId}`,
+      authorIp: os.hostname(),
       data: {
         // task execute worker
         taskWorker: '',
-        since: '',
+        hookEvent,
+        hookId,
       },
     };
-    return this.create(data);
+    const task = this.create(data);
+    task.logPath = `/packages/${hookEvent.fullname}/hooks/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
   }
 
   public static createSyncBinary(targetName: string, lastData: any): Task {
@@ -110,8 +214,9 @@ export class Task extends Entity {
       type: TaskType.SyncBinary,
       state: TaskState.Waiting,
       targetName,
-      authorId: `pid_${process.pid}`,
-      authorIp: os.hostname(),
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
+      bizId: `SyncBinary:${targetName}`,
       data: {
         // task execute worker
         taskWorker: '',
@@ -122,4 +227,21 @@ export class Task extends Entity {
     task.logPath = `/binaries/${targetName}/syncs/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
     return task;
   }
+
+  start(): TaskUpdateCondition {
+    const condition = {
+      taskId: this.taskId,
+      attempts: this.attempts,
+    };
+    this.setExecuteWorker();
+    this.state = TaskState.Processing;
+    this.attempts += 1;
+    return condition;
+  }
 }
+
+export type SyncInfo = {
+  lastSince: string;
+  taskCount: number;
+  lastPackage?: string;
+};

app/core/event/ChangesStream.ts

@@ -1,3 +1,4 @@
+import { EggAppConfig } from 'egg';
 import { Event, Inject } from '@eggjs/tegg';
 import {
   PACKAGE_UNPUBLISHED,
@@ -8,83 +9,141 @@ import {
   PACKAGE_TAG_REMOVED,
   PACKAGE_MAINTAINER_CHANGED,
   PACKAGE_MAINTAINER_REMOVED,
-  PACKAGE_META_CHANGED,
+  PACKAGE_META_CHANGED, PackageMetaChange,
 } from './index';
 import { ChangeRepository } from '../../repository/ChangeRepository';
 import { Change } from '../entity/Change';
+import { HookEvent } from '../entity/HookEvent';
+import { Task } from '../entity/Task';
+import { User } from '../entity/User';
+import { TaskService } from '../service/TaskService';
 
 class ChangesStreamEvent {
   @Inject()
   private readonly changeRepository: ChangeRepository;
 
-  protected async addChange(type: string, fullname: string, data: object) {
-    await this.changeRepository.addChange(Change.create({
+  @Inject()
+  protected readonly taskService: TaskService;
+
+  @Inject()
+  protected readonly config: EggAppConfig;
+
+  protected get hookEnable() {
+    return this.config.hookEnable;
+  }
+
+  protected async addChange(type: string, fullname: string, data: object): Promise<Change> {
+    const change = Change.create({
       type,
       targetName: fullname,
       data,
-    }));
+    });
+    await this.changeRepository.addChange(change);
+    return change;
   }
 }
 
 @Event(PACKAGE_UNPUBLISHED)
 export class PackageUnpublished extends ChangesStreamEvent {
   async handle(fullname: string) {
-    await this.addChange(PACKAGE_UNPUBLISHED, fullname, {});
+    const change = await this.addChange(PACKAGE_UNPUBLISHED, fullname, {});
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(fullname, change.changeId));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
 export class PackageVersionAdded extends ChangesStreamEvent {
-  async handle(fullname: string, version: string) {
-    await this.addChange(PACKAGE_VERSION_ADDED, fullname, { version });
+  async handle(fullname: string, version: string, tag?: string) {
+    const change = await this.addChange(PACKAGE_VERSION_ADDED, fullname, { version });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createPublishEvent(fullname, change.changeId, version, tag));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_VERSION_REMOVED)
 export class PackageVersionRemoved extends ChangesStreamEvent {
-  async handle(fullname: string, version: string) {
-    await this.addChange(PACKAGE_VERSION_REMOVED, fullname, { version });
+  async handle(fullname: string, version: string, tag?: string) {
+    const change = await this.addChange(PACKAGE_VERSION_REMOVED, fullname, { version });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createUnpublishEvent(fullname, change.changeId, version, tag));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_TAG_ADDED)
 export class PackageTagAdded extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_ADDED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_ADDED, fullname, { tag });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createDistTagEvent(fullname, change.changeId, tag));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_TAG_CHANGED)
 export class PackageTagChanged extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_CHANGED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_CHANGED, fullname, { tag });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createDistTagEvent(fullname, change.changeId, tag));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_TAG_REMOVED)
 export class PackageTagRemoved extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
-    await this.addChange(PACKAGE_TAG_REMOVED, fullname, { tag });
+    const change = await this.addChange(PACKAGE_TAG_REMOVED, fullname, { tag });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createDistTagRmEvent(fullname, change.changeId, tag));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_MAINTAINER_CHANGED)
 export class PackageMaintainerChanged extends ChangesStreamEvent {
-  async handle(fullname: string) {
-    await this.addChange(PACKAGE_MAINTAINER_CHANGED, fullname, {});
+  async handle(fullname: string, maintainers: User[]) {
+    const change = await this.addChange(PACKAGE_MAINTAINER_CHANGED, fullname, {});
+    // TODO 应该比较差值，而不是全量推送
+    if (this.hookEnable) {
+      for (const maintainer of maintainers) {
+        const task = Task.createCreateHookTask(HookEvent.createOwnerEvent(fullname, change.changeId, maintainer.name));
+        await this.taskService.createTask(task, true);
+      }
+    }
   }
 }
 
 @Event(PACKAGE_MAINTAINER_REMOVED)
 export class PackageMaintainerRemoved extends ChangesStreamEvent {
   async handle(fullname: string, maintainer: string) {
-    await this.addChange(PACKAGE_MAINTAINER_REMOVED, fullname, { maintainer });
+    const change = await this.addChange(PACKAGE_MAINTAINER_REMOVED, fullname, { maintainer });
+    if (this.hookEnable) {
+      const task = Task.createCreateHookTask(HookEvent.createOwnerRmEvent(fullname, change.changeId, maintainer));
+      await this.taskService.createTask(task, true);
+    }
   }
 }
 
 @Event(PACKAGE_META_CHANGED)
 export class PackageMetaChanged extends ChangesStreamEvent {
-  async handle(fullname: string, meta: object) {
-    await this.addChange(PACKAGE_META_CHANGED, fullname, { ...meta });
+  async handle(fullname: string, meta: PackageMetaChange) {
+    const change = await this.addChange(PACKAGE_META_CHANGED, fullname, { ...meta });
+    const { deprecateds } = meta;
+    if (this.hookEnable) {
+      for (const deprecated of deprecateds || []) {
+        const task = Task.createCreateHookTask(HookEvent.createDeprecatedEvent(fullname, change.changeId, deprecated.version));
+        await this.taskService.createTask(task, true);
+      }
+    }
   }
 }

app/core/event/index.ts

@@ -1,4 +1,5 @@
 import '@eggjs/tegg';
+import { User } from '../entity/User';
 
 export const PACKAGE_UNPUBLISHED = 'PACKAGE_UNPUBLISHED';
 export const PACKAGE_BLOCKED = 'PACKAGE_BLOCKED';
@@ -12,18 +13,28 @@ export const PACKAGE_MAINTAINER_CHANGED = 'PACKAGE_MAINTAINER_CHANGED';
 export const PACKAGE_MAINTAINER_REMOVED = 'PACKAGE_MAINTAINER_REMOVED';
 export const PACKAGE_META_CHANGED = 'PACKAGE_META_CHANGED';
 
+export interface PackageDeprecated {
+  version: string;
+  deprecated: string;
+}
+
+export interface PackageMetaChange {
+  deprecateds?: Array<PackageDeprecated>;
+}
+
+
 declare module '@eggjs/tegg' {
   interface Events {
     [PACKAGE_UNPUBLISHED]: (fullname: string) => Promise<void>;
     [PACKAGE_BLOCKED]: (fullname: string) => Promise<void>;
     [PACKAGE_UNBLOCKED]: (fullname: string) => Promise<void>;
-    [PACKAGE_VERSION_ADDED]: (fullname: string, version: string) => Promise<void>;
-    [PACKAGE_VERSION_REMOVED]: (fullname: string, version: string) => Promise<void>;
+    [PACKAGE_VERSION_ADDED]: (fullname: string, version: string, tag?: string) => Promise<void>;
+    [PACKAGE_VERSION_REMOVED]: (fullname: string, version: string, tag?: string) => Promise<void>;
     [PACKAGE_TAG_ADDED]: (fullname: string, tag: string) => Promise<void>;
     [PACKAGE_TAG_CHANGED]: (fullname: string, tag: string) => Promise<void>;
     [PACKAGE_TAG_REMOVED]: (fullname: string, tag: string) => Promise<void>;
-    [PACKAGE_MAINTAINER_CHANGED]: (fullname: string) => Promise<void>;
+    [PACKAGE_MAINTAINER_CHANGED]: (fullname: string, maintainers: User[]) => Promise<void>;
     [PACKAGE_MAINTAINER_REMOVED]: (fullname: string, maintainer: string) => Promise<void>;
-    [PACKAGE_META_CHANGED]: (fullname: string, meta: object) => Promise<void>;
+    [PACKAGE_META_CHANGED]: (fullname: string, meta: PackageMetaChange) => Promise<void>;
   }
 }

app/core/service/BinarySyncerService.ts

@@ -30,6 +30,7 @@ import { ElectronBinary } from '../../common/adapter/binary/ElectronBinary';
 import { NodePreGypBinary } from '../../common/adapter/binary/NodePreGypBinary';
 import { ImageminBinary } from '../../common/adapter/binary/ImageminBinary';
 import { PlaywrightBinary } from '../../common/adapter/binary/PlaywrightBinary';
+import { TaskRepository } from 'app/repository/TaskRepository';
 
 const BinaryClasses = {
   [SyncerClass.NodeBinary]: NodeBinary,
@@ -58,6 +59,8 @@ export class BinarySyncerService extends AbstractService {
   @Inject()
   private readonly taskService: TaskService;
   @Inject()
+  private readonly taskRepository: TaskRepository;
+  @Inject()
   private readonly httpclient: EggContextHttpClient;
   @Inject()
   private readonly nfsAdapter: NFSAdapter;
@@ -71,15 +74,51 @@ export class BinarySyncerService extends AbstractService {
   }
 
   public async listRootBinaries(binaryName: string) {
-    return await this.binaryRepository.listBinaries(binaryName, '/');
+    // 通常 binaryName 和 category 是一样的，但是有些特殊的 binaryName 会有多个 category，比如 canvas
+    // 所以查询 canvas 的时候，需要将 binaryName 和 category 的数据都查出来
+    const {
+      category,
+    } = binaries[binaryName];
+    const reqs = [
+      this.binaryRepository.listBinaries(binaryName, '/'),
+    ];
+    if (category && category !== binaryName) {
+      reqs.push(this.binaryRepository.listBinaries(category, '/'));
+    }
+
+    const [
+      rootBinary,
+      categoryBinary,
+    ] = await Promise.all(reqs);
+
+    const versions = rootBinary.map(b => b.name);
+    categoryBinary?.forEach(b => {
+      const version = b.name;
+      // 只将没有的版本添加进去
+      if (!versions.includes(version)) {
+        rootBinary.push(b);
+      }
+    });
+
+    return rootBinary;
   }
 
   public async downloadBinary(binary: Binary) {
     return await this.nfsAdapter.getDownloadUrlOrStream(binary.storePath);
   }
 
+  // SyncBinary 由定时任务每台单机定时触发，手动去重
+  // 添加 bizId 在 db 防止重复，记录 id 错误
   public async createTask(binaryName: string, lastData?: any) {
-    return await this.taskService.createTask(Task.createSyncBinary(binaryName, lastData), false);
+    const existsTask = await this.taskRepository.findTaskByTargetName(binaryName, TaskType.SyncBinary);
+    if (existsTask) {
+      return existsTask;
+    }
+    try {
+      return await this.taskService.createTask(Task.createSyncBinary(binaryName, lastData), false);
+    } catch (e) {
+      this.logger.error('[BinarySyncerService.createTask] binaryName: %s, error: %s', binaryName, e);
+    }
   }
 
   public async findTask(taskId: string) {
@@ -174,7 +213,7 @@ export class BinarySyncerService extends AbstractService {
             logs = [];
           } catch (err: any) {
             if (err.name === 'DownloadNotFoundError') {
-              this.logger.warn('Not found %s, skip it', item.sourceUrl);
+              this.logger.info('Not found %s, skip it', item.sourceUrl);
               logs.push(`[${isoNow()}][${dir}] 🧪️ [${parentIndex}${index}] Download ${item.sourceUrl} not found, skip it`);
             } else {
               this.logger.error('Download binary %s %s', item.sourceUrl, err);
@@ -250,15 +289,13 @@ export class BinarySyncerService extends AbstractService {
 
   private createBinaryInstance(binaryName: string): AbstractBinary | undefined {
     const config = this.config.cnpmcore;
+    const binaryConfig = binaries[binaryName];
+
     if (config.sourceRegistryIsCNpm) {
-      const binaryConfig = binaries[binaryName];
       const syncBinaryFromAPISource = config.syncBinaryFromAPISource || `${config.sourceRegistry}/-/binary`;
-      return new ApiBinary(this.httpclient, this.logger, binaryConfig, syncBinaryFromAPISource);
-    }
-    for (const binaryConfig of Object.values(binaries)) {
-      if (binaryConfig.category === binaryName) {
-        return new BinaryClasses[binaryConfig.syncer](this.httpclient, this.logger, binaryConfig);
-      }
+      return new ApiBinary(this.httpclient, this.logger, binaryConfig, syncBinaryFromAPISource, binaryName);
     }
+
+    return new BinaryClasses[binaryConfig.syncer](this.httpclient, this.logger, binaryConfig, binaryName);
   }
 }

app/core/service/ChangesStreamService.ts

@@ -3,17 +3,23 @@ import { setTimeout } from 'timers/promises';
 import {
   AccessLevel,
   ContextProto,
+  EggObjectFactory,
   Inject,
 } from '@eggjs/tegg';
-import {
-  EggContextHttpClient,
-} from 'egg';
-import { TaskType } from '../../common/enum/Task';
+import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { Task } from '../entity/Task';
-import { PackageSyncerService } from './PackageSyncerService';
+import { HOST_NAME, ChangesStreamTask, Task } from '../entity/Task';
+import { PackageSyncerService, RegistryNotMatchError } from './PackageSyncerService';
 import { TaskService } from './TaskService';
+import { RegistryManagerService } from './RegistryManagerService';
+import { RegistryType } from '../../common/enum/Registry';
+import { E500 } from 'egg-errors';
+import { Registry } from '../entity/Registry';
+import { AbstractChangeStream } from '../../common/adapter/changesStream/AbstractChangesStream';
+import { getScopeAndName } from '../../common/PackageUtil';
+import { ScopeManagerService } from './ScopeManagerService';
+import { PackageRepository } from '../../repository/PackageRepository';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -22,50 +28,69 @@ export class ChangesStreamService extends AbstractService {
   @Inject()
   private readonly taskRepository: TaskRepository;
   @Inject()
-  private readonly httpclient: EggContextHttpClient;
-  @Inject()
   private readonly packageSyncerService: PackageSyncerService;
   @Inject()
   private readonly taskService: TaskService;
+  @Inject()
+  private readonly registryManagerService : RegistryManagerService;
+  @Inject()
+  private readonly scopeManagerService : ScopeManagerService;
+  @Inject()
+  private readonly eggObjectFactory: EggObjectFactory;
+  @Inject()
+  private readonly packageRepository: PackageRepository;
 
-  public async findExecuteTask() {
+  // 出于向下兼容考虑, changes_stream 类型 Task 分为
+  // GLOBAL_WORKER: 默认的同步源
+  // `{registryName}_WORKER`: 自定义 scope 的同步源
+  public async findExecuteTask(): Promise<ChangesStreamTask | null> {
     const targetName = 'GLOBAL_WORKER';
-    const existsTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
-    if (!existsTask) {
+    const globalRegistryTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.ChangesStream);
+    // 如果没有配置默认同步源，先进行初始化
+    if (!globalRegistryTask) {
       await this.taskService.createTask(Task.createChangesStream(targetName), false);
     }
-    return await this.taskService.findExecuteTask(TaskType.ChangesStream);
+    // 自定义 scope 由 admin 手动创建
+    // 根据 TaskType.ChangesStream 从队列中获取
+    return await this.taskService.findExecuteTask(TaskType.ChangesStream) as ChangesStreamTask;
   }
 
-  public async executeTask(task: Task) {
+  public async suspendTaskWhenExit() {
+    this.logger.info('[ChangesStreamService.suspendTaskWhenExit:start]');
+    if (this.config.cnpmcore.enableChangesStream) {
+      // 防止继续获取新的任务
+      this.config.cnpmcore.enableChangesStream = false;
+      const authorIp = os.hostname();
+      // 暂停当前机器所有的 changesStream 任务
+      const tasks = await this.taskRepository.findTaskByAuthorIpAndType(authorIp, TaskType.ChangesStream);
+      for (const task of tasks) {
+        if (task.state === TaskState.Processing) {
+          this.logger.info('[ChangesStreamService.suspendTaskWhenExit:suspend] taskId: %s', task.taskId);
+          // 1. 更新任务状态为 waiting
+          // 2. 重新推入任务队列供其他机器执行
+          await this.taskService.retryTask(task);
+        }
+      }
+    }
+    this.logger.info('[ChangesStreamService.suspendTaskWhenExit:finish]');
+  }
+
+  public async executeTask(task: ChangesStreamTask) {
     task.authorIp = os.hostname();
     task.authorId = `pid_${process.pid}`;
     await this.taskRepository.saveTask(task);
 
-    const changesStreamRegistry: string = this.config.cnpmcore.changesStreamRegistry;
-    // https://github.com/npm/registry-follower-tutorial
-    // default "update_seq": 7138885,
+    // 初始化 changeStream 任务
+    // since 默认从 1 开始
     try {
       let since: string = task.data.since;
-      // get update_seq from ${changesStreamRegistry} on the first time
       if (!since) {
-        const { status, data } = await this.httpclient.request(changesStreamRegistry, {
-          followRedirect: true,
-          timeout: 10000,
-          dataType: 'json',
-        });
-        if (data.update_seq) {
-          since = String(data.update_seq - 10);
-        } else {
-          since = '7139538';
-        }
-        this.logger.warn('[ChangesStreamService.executeTask:firstSeq] GET %s status: %s, data: %j, since: %s',
-          changesStreamRegistry, status, data, since);
+        since = await this.getInitialSince(task);
       }
       // allow disable changesStream dynamic
       while (since && this.config.cnpmcore.enableChangesStream) {
-        const { lastSince, taskCount } = await this.handleChanges(since, task);
-        this.logger.warn('[ChangesStreamService.executeTask:changes] since: %s => %s, %d new tasks, taskId: %s, updatedAt: %j',
+        const { lastSince, taskCount } = await this.executeSync(since, task);
+        this.logger.info('[ChangesStreamService.executeTask:changes] since: %s => %s, %d new tasks, taskId: %s, updatedAt: %j',
           since, lastSince, taskCount, task.taskId, task.updatedAt);
         since = lastSince;
         if (taskCount === 0 && this.config.env === 'unittest') {
@@ -81,101 +106,137 @@ export class ChangesStreamService extends AbstractService {
     }
   }
 
-  private async handleChanges(since: string, task: Task) {
-    const changesStreamRegistry: string = this.config.cnpmcore.changesStreamRegistry;
-    const changesStreamRegistryMode: string = this.config.cnpmcore.changesStreamRegistryMode;
-    const db = `${changesStreamRegistry}/_changes?since=${since}`;
-    let lastSince = since;
-    let taskCount = 0;
-    if (changesStreamRegistryMode === 'streaming') {
-      const { res } = await this.httpclient.request(db, {
-        streaming: true,
-        timeout: 10000,
-      });
-      for await (const chunk of res) {
-        const text: string = chunk.toString();
-        // {"seq":7138879,"id":"@danydodson/prettier-config","changes":[{"rev":"5-a56057032714af25400d93517773a82a"}]}
-        // console.log('😄%j😄', text);
-        // 😄"{\"seq\":7138738,\"id\":\"wargerm\",\"changes\":[{\"rev\":\"59-f0a0d326db4c62ed480987a04ba3bf8f\"}]}"😄
-        // 😄",\n{\"seq\":7138739,\"id\":\"@laffery/webpack-starter-kit\",\"changes\":[{\"rev\":\"4-84a8dc470a07872f4cdf85cf8ef892a1\"}]},\n{\"seq\":7138741,\"id\":\"venom-bot\",\"changes\":[{\"rev\":\"103-908654b1ad4b0e0fd40b468d75730674\"}]}"😄
-        // 😄",\n{\"seq\":7138743,\"id\":\"react-native-template-pytorch-live\",\"changes\":[{\"rev\":\"40-871c686b200312303ba7c4f7f93e0362\"}]}"😄
-        // 😄",\n{\"seq\":7138745,\"id\":\"ccxt\",\"changes\":[{\"rev\":\"10205-25367c525a0a3bd61be3a72223ce212c\"}]}"😄
-        const matchs = text.matchAll(/"seq":(\d+),"id":"([^"]+)"/gm);
-        let count = 0;
-        let lastPackage = '';
-        for (const match of matchs) {
-          const seq = match[1];
-          const fullname = match[2];
-          if (seq && fullname) {
-            await this.packageSyncerService.createTask(fullname, {
-              authorIp: os.hostname(),
-              authorId: 'ChangesStreamService',
-              skipDependencies: true,
-              tips: `Sync cause by changes_stream(${changesStreamRegistry}) update seq: ${seq}`,
-            });
-            count++;
-            lastSince = seq;
-            lastPackage = fullname;
-          }
-        }
-        if (count > 0) {
-          taskCount += count;
-          task.data = {
-            ...task.data,
-            since: lastSince,
-            last_package: lastPackage,
-            last_package_created: new Date(),
-            task_count: (task.data.task_count || 0) + count,
-          };
-          await this.taskRepository.saveTask(task);
-        }
-      }
-    } else {
-      // json mode
-      // {"results":[{"seq":1988653,"type":"PACKAGE_VERSION_ADDED","id":"dsr-package-mercy-magot-thorp-sward","changes":[{"version":"1.0.1"}]},
-      const { data } = await this.httpclient.request(db, {
-        followRedirect: true,
-        timeout: 30000,
-        dataType: 'json',
-        gzip: true,
-      });
-      if (data.results?.length > 0) {
-        let count = 0;
-        let lastPackage = '';
-        for (const change of data.results) {
-          const seq = change.seq;
-          const fullname = change.id;
-          if (seq && fullname && seq !== since) {
-            await this.packageSyncerService.createTask(fullname, {
-              authorIp: os.hostname(),
-              authorId: 'ChangesStreamService',
-              skipDependencies: true,
-              tips: `Sync cause by changes_stream(${changesStreamRegistry}) update seq: ${seq}, change: ${JSON.stringify(change)}`,
-            });
-            count++;
-            lastSince = seq;
-            lastPackage = fullname;
-          }
-        }
-        if (count > 0) {
-          taskCount += count;
-          task.data = {
-            ...task.data,
-            since: lastSince,
-            last_package: lastPackage,
-            last_package_created: new Date(),
-            task_count: (task.data.task_count || 0) + count,
-          };
-          await this.taskRepository.saveTask(task);
-        }
+  // 优先从 registryId 获取，如果没有的话再返回默认的 registry
+  public async prepareRegistry(task: ChangesStreamTask): Promise<Registry> {
+    const { registryId } = task.data || {};
+    // 如果已有 registryId, 查询 DB 直接获取
+    if (registryId) {
+      const registry = await this.registryManagerService.findByRegistryId(registryId);
+      if (!registry) {
+        this.logger.error('[ChangesStreamService.getRegistry:error] registryId %s not found', registryId);
+        throw new E500(`invalid change stream registry: ${registryId}`);
       }
+      return registry;
     }
 
-    if (taskCount === 0) {
-      // keep update task, make sure updatedAt changed
-      task.updatedAt = new Date();
+    // 从配置文件默认生成
+    const { changesStreamRegistryMode, changesStreamRegistry: changesStreamHost, sourceRegistry: host } = this.config.cnpmcore;
+    const type = changesStreamRegistryMode === 'json' ? RegistryType.Cnpmcore : RegistryType.Npm;
+    const registry = await this.registryManagerService.createRegistry({
+      name: 'default',
+      type,
+      userPrefix: 'npm:',
+      host,
+      changeStream: `${changesStreamHost}/_changes`,
+    });
+    task.data = {
+      ...(task.data || {}),
+      registryId: registry.registryId,
+    };
+    await this.taskRepository.saveTask(task);
+
+    return registry;
+  }
+
+  // 根据 regsitry 判断是否需要添加同步任务
+  // 1. 如果该包已经指定了 registryId 则以 registryId 为准
+  // 1. 该包的 scope 在当前 registry 下
+  // 2. 如果 registry 下没有配置 scope (认为是通用 registry 地址) ，且该包的 scope 不在其他 registry 下
+  public async needSync(registry: Registry, fullname: string): Promise<boolean> {
+    const [ scopeName, name ] = getScopeAndName(fullname);
+    const packageEntity = await this.packageRepository.findPackage(scopeName, name);
+
+    // 如果包不存在，且处在 exist 模式下，则不同步
+    if (this.config.cnpmcore.syncMode === 'exist' && !packageEntity) {
+      return false;
+    }
+
+    if (packageEntity?.registryId) {
+      return registry.registryId === packageEntity.registryId;
+    }
+
+    const scope = await this.scopeManagerService.findByName(scopeName);
+    const inCurrentRegistry = scope && scope?.registryId === registry.registryId;
+    if (inCurrentRegistry) {
+      return true;
+    }
+
+    const registryScopeCount = await this.scopeManagerService.countByRegistryId(registry.registryId);
+    // 当前包没有 scope 信息，且当前 registry 下没有 scope，是通用 registry，需要同步
+    return !scope && !registryScopeCount;
+  }
+  public async getInitialSince(task: ChangesStreamTask): Promise<string> {
+    const registry = await this.prepareRegistry(task);
+    const changesStreamAdapter = await this.eggObjectFactory.getEggObject(AbstractChangeStream, registry.type) as AbstractChangeStream;
+    const since = await changesStreamAdapter.getInitialSince(registry);
+    return since;
+  }
+
+  // 从 changesStream 获取需要同步的数据
+  // 更新任务的 since 和 taskCount 相关字段
+  public async executeSync(since: string, task: ChangesStreamTask) {
+    const registry = await this.prepareRegistry(task);
+    const changesStreamAdapter = await this.eggObjectFactory.getEggObject(AbstractChangeStream, registry.type) as AbstractChangeStream;
+    let taskCount = 0;
+    let lastSince = since;
+
+    // 获取需要同步的数据
+    // 需要根据 scope 和包信息进行过滤
+    const stream = changesStreamAdapter.fetchChanges(registry, since);
+    let lastPackage: string | undefined;
+
+    // 创建同步任务
+    for await (const change of stream) {
+      const { fullname, seq } = change;
+      lastPackage = fullname;
+      lastSince = seq;
+      const valid = await this.needSync(registry, fullname);
+      if (valid) {
+        taskCount++;
+        const tips = `Sync cause by changes_stream(${registry.changeStream}) update seq: ${seq}`;
+        try {
+          const task = await this.packageSyncerService.createTask(fullname, {
+            authorIp: HOST_NAME,
+            authorId: 'ChangesStreamService',
+            registryId: registry.registryId,
+            skipDependencies: true,
+            tips,
+          });
+          this.logger.info('[ChangesStreamService.createTask:success] fullname: %s, task: %s, tips: %s',
+            fullname, task.id, tips);
+        } catch (err) {
+          if (err instanceof RegistryNotMatchError) {
+            this.logger.warn('[ChangesStreamService.executeSync:skip] fullname: %s, error: %s, tips: %s',
+              fullname, err, tips);
+            continue;
+          }
+          // only log error, make sure changes still reading
+          this.logger.error('[ChangesStreamService.executeSync:error] fullname: %s, error: %s, tips: %s',
+            fullname, err, tips);
+          this.logger.error(err);
+          continue;
+        }
+      }
+      // 实时更新 task 信息
+      // 即使不需要同步，防止任务处理累积耗时超过 10min
+      task.updateSyncData({
+        lastSince,
+        lastPackage,
+        taskCount,
+      });
       await this.taskRepository.saveTask(task);
     }
+
+    // 如果 taskCount 为 0 更新一下任务信息
+    if (taskCount === 0) {
+      task.updateSyncData({
+        lastSince,
+        lastPackage,
+        taskCount,
+      });
+      await this.taskRepository.saveTask(task);
+    }
+
     return { lastSince, taskCount };
   }
 }

app/core/service/CreateHookTriggerService.ts

@@ -0,0 +1,78 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { AbstractService } from '../../common/AbstractService';
+import { HookType } from '../../common/enum/Hook';
+import { TaskState } from '../../common/enum/Task';
+import { HookEvent } from '../entity/HookEvent';
+import { CreateHookTask, Task } from '../entity/Task';
+import { HookRepository } from '../../repository/HookRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import pMap from 'p-map';
+import { Hook } from '../entity/Hook';
+import { TaskService } from './TaskService';
+import { isoNow } from '../../common/LogUtil';
+import { getScopeAndName } from '../../common/PackageUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class CreateHookTriggerService extends AbstractService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async executeTask(task: CreateHookTask): Promise<void> {
+    const { hookEvent } = task.data;
+    const [ scope, name ] = getScopeAndName(hookEvent.fullname);
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][Hooks] package ${hookEvent.fullname} not exits`);
+      return;
+    }
+
+    const startLog = [
+      `[${isoNow()}][Hooks] Start Create Trigger for ${pkg.fullname} ${task.data.hookEvent.changeId}`,
+      `[${isoNow()}][Hooks] change content ${JSON.stringify(task.data.hookEvent.change)}`,
+    ];
+    await this.taskService.finishTask(task, TaskState.Processing, startLog.join('\n'));
+
+    try {
+      await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Package} ${pkg.fullname}\n`);
+      await this.createTriggerByMethod(task, HookType.Package, pkg.fullname, hookEvent);
+      await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Scope} ${pkg.scope}\n`);
+      await this.createTriggerByMethod(task, HookType.Scope, pkg.scope, hookEvent);
+
+      const maintainers = await this.packageRepository.listPackageMaintainers(pkg.packageId);
+      for (const maintainer of maintainers) {
+        await this.taskService.appendTaskLog(task, `[${isoNow()}][Hooks] PushHooks to ${HookType.Owner} ${maintainer.name}\n`);
+        await this.createTriggerByMethod(task, HookType.Owner, maintainer.name, hookEvent);
+      }
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][Hooks] create trigger succeed \n`);
+    } catch (e) {
+      e.message = 'create trigger failed: ' + e.message;
+      await this.taskService.finishTask(task, TaskState.Fail, `[${isoNow()}][Hooks] ${e.stack} \n`);
+      return;
+    }
+  }
+
+  private async createTriggerByMethod(task: Task, type: HookType, name: string, hookEvent: HookEvent) {
+    let hooks = await this.hookRepository.listHooksByTypeAndName(type, name);
+    while (hooks.length) {
+      await this.createTriggerTasks(hooks, hookEvent);
+      hooks = await this.hookRepository.listHooksByTypeAndName(type, name, hooks[hooks.length - 1].id);
+      await this.taskService.appendTaskLog(task,
+        `[${isoNow()}][Hooks] PushHooks to ${type} ${name} ${hooks.length} \n`);
+    }
+  }
+
+  private async createTriggerTasks(hooks: Array<Hook>, hookEvent: HookEvent) {
+    await pMap(hooks, async hook => {
+      const triggerHookTask = Task.createTriggerHookTask(hookEvent, hook.hookId);
+      await this.taskService.createTask(triggerHookTask, true);
+    }, { concurrency: 5 });
+  }
+}

app/core/service/EventCorkerAdvice.ts

@@ -0,0 +1,16 @@
+import { ContextEventBus, Inject } from '@eggjs/tegg';
+import { Advice, IAdvice } from '@eggjs/tegg/aop';
+
+@Advice()
+export class EventCorkAdvice implements IAdvice {
+  @Inject()
+  private eventBus: ContextEventBus;
+
+  async beforeCall() {
+    this.eventBus.cork();
+  }
+
+  async afterFinally() {
+    this.eventBus.uncork();
+  }
+}

app/core/service/HookManageService.ts

@@ -0,0 +1,96 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { Hook } from '../entity/Hook';
+import { HookType } from '../../common/enum/Hook';
+import {
+  ForbiddenError,
+  NotFoundError,
+} from 'egg-errors';
+import { HookRepository } from '../../repository/HookRepository';
+import { EggAppConfig } from 'egg';
+
+export interface CreateHookCommand {
+  type: HookType;
+  ownerId: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface UpdateHookCommand {
+  operatorId: string;
+  hookId: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface DeleteHookCommand {
+  operatorId: string;
+  hookId: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookManageService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  get hooksLimit() {
+    return this.config.cnpmcore.hooksLimit;
+  }
+
+  async createHook(cmd: CreateHookCommand): Promise<Hook> {
+    const hooks = await this.hookRepository.listHooksByOwnerId(cmd.ownerId);
+    // FIXME: 会有并发问题，需要有一个用户全局锁去记录
+    if (hooks.length >= this.hooksLimit) {
+      throw new ForbiddenError('hooks limit exceeded');
+    }
+    const hook = Hook.create(cmd);
+    await this.hookRepository.saveHook(hook);
+    return hook;
+  }
+
+  async updateHook(cmd: UpdateHookCommand): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(cmd.hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${cmd.hookId} not found`);
+    }
+    if (hook.ownerId !== cmd.operatorId) {
+      throw new ForbiddenError(`hook ${cmd.hookId} not belong to ${cmd.operatorId}`);
+    }
+    hook.endpoint = cmd.endpoint;
+    hook.secret = cmd.secret;
+    await this.hookRepository.saveHook(hook);
+    return hook;
+  }
+
+  async deleteHook(cmd: DeleteHookCommand): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(cmd.hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${cmd.hookId} not found`);
+    }
+    if (hook.ownerId !== cmd.operatorId) {
+      throw new ForbiddenError(`hook ${cmd.hookId} not belong to ${cmd.operatorId}`);
+    }
+    await this.hookRepository.removeHook(cmd.hookId);
+    return hook;
+  }
+
+  async listHooksByOwnerId(ownerId: string): Promise<Hook[]> {
+    return await this.hookRepository.listHooksByOwnerId(ownerId);
+  }
+
+  async getHookByOwnerId(hookId: string, userId: string): Promise<Hook> {
+    const hook = await this.hookRepository.findHookById(hookId);
+    if (!hook) {
+      throw new NotFoundError(`hook ${hookId} not found`);
+    }
+    if (hook.ownerId !== userId) {
+      throw new ForbiddenError(`hook ${hookId} not belong to ${userId}`);
+    }
+    return hook;
+  }
+}

app/core/service/HookTriggerService.ts

@@ -0,0 +1,111 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { TriggerHookTask } from '../entity/Task';
+import { HookEvent } from '../entity/HookEvent';
+import { HookRepository } from '../../repository/HookRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { DistRepository } from '../../repository/DistRepository';
+import { UserRepository } from '../../repository/UserRepository';
+import { Hook } from '../entity/Hook';
+import { EggContextHttpClient } from 'egg';
+import { isoNow } from '../../common/LogUtil';
+import { TaskState } from '../../common/enum/Task';
+import { TaskService } from './TaskService';
+import { getScopeAndName } from '../../common/PackageUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookTriggerService {
+  @Inject()
+  private readonly hookRepository: HookRepository;
+
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly distRepository: DistRepository;
+
+  @Inject()
+  private readonly userRepository: UserRepository;
+
+  @Inject()
+  private readonly httpclient: EggContextHttpClient;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async executeTask(task: TriggerHookTask) {
+    const { hookId, hookEvent } = task.data;
+    const hook = await this.hookRepository.findHookById(hookId);
+    if (!hook) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] hook ${hookId} not exits`);
+      return;
+    }
+    try {
+      const payload = await this.createTriggerPayload(task, hookEvent, hook);
+      if (!payload) {
+        await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] generate payload failed \n`);
+        return;
+      }
+      const status = await this.doExecuteTrigger(hook, payload);
+      hook.latestTaskId = task.taskId;
+      task.data.responseStatus = status;
+      await this.hookRepository.saveHook(hook);
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] trigger hook succeed ${status} \n`);
+    } catch (e) {
+      e.message = 'trigger hook failed: ' + e.message;
+      task.error = e.message;
+      await this.taskService.finishTask(task, TaskState.Fail, `[${isoNow()}][TriggerHooks] ${e.stack} \n`);
+      return;
+    }
+  }
+
+  async doExecuteTrigger(hook: Hook, payload: object): Promise<number> {
+    const { digest, payloadStr } = hook.signPayload(payload);
+    const url = new URL(hook.endpoint);
+    const res = await this.httpclient.request(hook.endpoint, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'x-npm-signature': `sha256=${digest}`,
+        host: url.host,
+      },
+      // webhook 场景下，由于 endpoint 都不同
+      // 因此几乎不存在连接复用的情况，因此这里不使用 keepAlive
+      agent: false,
+      httpsAgent: false,
+      data: payloadStr,
+    } as any);
+    if (res.status >= 200 && res.status < 300) {
+      return res.status;
+    }
+    throw new Error(`hook response with ${res.status}`);
+  }
+
+  async createTriggerPayload(task: TriggerHookTask, hookEvent: HookEvent, hook: Hook): Promise<object | undefined> {
+    const [ scope, name ] = getScopeAndName(hookEvent.fullname);
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] can not found pkg for ${hookEvent.fullname} \n`);
+      return;
+    }
+    const user = await this.userRepository.findUserByUserId(hook.ownerId);
+    if (!user) {
+      await this.taskService.finishTask(task, TaskState.Success, `[${isoNow()}][TriggerHooks] can not found user for ${hook.ownerId} \n`);
+      return;
+    }
+    const manifest = await this.distRepository.readDistBytesToJSON(pkg!.manifestsDist!);
+    return {
+      event: hookEvent.event,
+      name: pkg.fullname,
+      type: 'package',
+      version: '1.0.0',
+      hookOwner: {
+        username: user.name,
+      },
+      payload: manifest,
+      change: hookEvent.change,
+      time: hookEvent.time,
+    };
+  }
+}

app/core/service/PackageManagerService.ts

@@ -46,6 +46,7 @@ export interface PublishPackageCmd {
   version: string;
   description: string;
   packageJson: any;
+  registryId?: string;
   readme: string;
   // require content or localFile field
   dist: RequireAtLeastOne<{
@@ -64,6 +65,7 @@ export interface PublishPackageCmd {
 
 const TOTAL = '@@TOTAL@@';
 const SCOPE_TOTAL_PREFIX = '@@SCOPE@@:';
+const DESCRIPTION_LIMIT = 1024 * 10;
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -95,6 +97,7 @@ export class PackageManagerService extends AbstractService {
         name: cmd.name,
         isPrivate: cmd.isPrivate,
         description: cmd.description,
+        registryId: cmd.registryId,
       });
     } else {
       // update description
@@ -102,6 +105,15 @@ export class PackageManagerService extends AbstractService {
       if (pkg.description !== cmd.description) {
         pkg.description = cmd.description;
       }
+
+      if (!pkg.registryId && cmd.registryId) {
+        pkg.registryId = cmd.registryId;
+      }
+    }
+
+    // 防止 description 长度超过 db 限制
+    if (pkg.description?.length > DESCRIPTION_LIMIT) {
+      pkg.description = pkg.description.substring(0, DESCRIPTION_LIMIT);
     }
     await this.packageRepository.savePackage(pkg);
     // create maintainer
@@ -117,9 +129,14 @@ export class PackageManagerService extends AbstractService {
       delete cmd.packageJson.readme;
     }
 
+    const publishTime = cmd.publishTime || new Date();
+
     // add _cnpmcore_publish_time field to cmd.packageJson
     if (!cmd.packageJson._cnpmcore_publish_time) {
-      cmd.packageJson._cnpmcore_publish_time = new Date();
+      cmd.packageJson._cnpmcore_publish_time = publishTime;
+    }
+    if (!cmd.packageJson.publish_time) {
+      cmd.packageJson.publish_time = publishTime.getTime();
     }
 
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
@@ -175,6 +192,9 @@ export class PackageManagerService extends AbstractService {
       engines: cmd.packageJson.engines,
       _hasShrinkwrap: cmd.packageJson._hasShrinkwrap,
       hasInstallScript,
+      // https://github.com/cnpm/npminstall/blob/13efc7eec21a61e509226e3772bfb75cd5605612/lib/install_package.js#L176
+      // npminstall require publish time to show the recently update versions
+      publish_time: cmd.packageJson.publish_time,
     });
     const abbreviatedDistBytes = Buffer.from(abbreviated);
     const abbreviatedDistIntegrity = await calculateIntegrity(abbreviatedDistBytes);
@@ -186,7 +206,7 @@ export class PackageManagerService extends AbstractService {
     pkgVersion = PackageVersion.create({
       packageId: pkg.packageId,
       version: cmd.version,
-      publishTime: cmd.publishTime || new Date(),
+      publishTime,
       manifestDist: pkg.createManifest(cmd.version, {
         size: manifestDistBytes.length,
         shasum: manifestDistIntegrity.shasum,
@@ -209,14 +229,21 @@ export class PackageManagerService extends AbstractService {
       this.distRepository.saveDist(pkgVersion.manifestDist, manifestDistBytes),
       this.distRepository.saveDist(pkgVersion.readmeDist, readmeDistBytes),
     ]);
-    await this.packageRepository.createPackageVersion(pkgVersion);
+    try {
+      await this.packageRepository.createPackageVersion(pkgVersion);
+    } catch (e) {
+      if (e.code === 'ER_DUP_ENTRY') {
+        throw new ForbiddenError(`Can't modify pre-existing version: ${pkg.fullname}@${cmd.version}`);
+      }
+      throw e;
+    }
     if (cmd.skipRefreshPackageManifests !== true) {
       await this.refreshPackageChangeVersionsToDists(pkg, [ pkgVersion.version ]);
     }
     if (cmd.tag) {
       await this.savePackageTag(pkg, cmd.tag, cmd.version, true);
     }
-    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version);
+    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, cmd.tag);
     return pkgVersion;
   }
 
@@ -273,7 +300,7 @@ export class PackageManagerService extends AbstractService {
   async replacePackageMaintainers(pkg: Package, maintainers: User[]) {
     await this.packageRepository.replacePackageMaintainers(pkg.packageId, maintainers.map(m => m.userId));
     await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
-    this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname);
+    this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
   }
 
   async savePackageMaintainers(pkg: Package, maintainers: User[]) {
@@ -286,7 +313,7 @@ export class PackageManagerService extends AbstractService {
     }
     if (hasNewRecord) {
       await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
-      this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname);
+      this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
     }
   }
 
@@ -468,6 +495,7 @@ export class PackageManagerService extends AbstractService {
     // all versions removed
     const versions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
     if (versions.length > 0) {
+      let updateTag: string | undefined;
       // make sure latest tag exists
       const latestTag = await this.packageRepository.findPackageTag(pkg.packageId, 'latest');
       if (latestTag?.version === pkgVersion.version) {
@@ -475,12 +503,13 @@ export class PackageManagerService extends AbstractService {
         // https://github.com/npm/libnpmpublish/blob/main/unpublish.js#L62
         const latestVersion = versions.sort(semver.compareLoose).pop();
         if (latestVersion) {
+          updateTag = latestTag.tag;
           await this.savePackageTag(pkg, latestTag.tag, latestVersion, true);
         }
       }
       if (skipRefreshPackageManifests !== true) {
         await this.refreshPackageChangeVersionsToDists(pkg, undefined, [ pkgVersion.version ]);
-        this.eventBus.emit(PACKAGE_VERSION_REMOVED, pkg.fullname, pkgVersion.version);
+        this.eventBus.emit(PACKAGE_VERSION_REMOVED, pkg.fullname, pkgVersion.version, updateTag);
       }
       return;
     }

app/core/service/PackageSyncerService.ts

@@ -4,6 +4,7 @@ import {
   ContextProto,
   Inject,
 } from '@eggjs/tegg';
+import { Pointcut } from '@eggjs/tegg/aop';
 import {
   EggContextHttpClient,
 } from 'egg';
@@ -18,19 +19,26 @@ import { TaskRepository } from '../../repository/TaskRepository';
 import { PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { UserRepository } from '../../repository/UserRepository';
-import { DistRepository } from '../../repository/DistRepository';
-import { Task, SyncPackageTaskOptions } from '../entity/Task';
+import { Task, SyncPackageTaskOptions, CreateSyncPackageTask } from '../entity/Task';
 import { Package } from '../entity/Package';
 import { UserService } from './UserService';
 import { TaskService } from './TaskService';
 import { PackageManagerService } from './PackageManagerService';
 import { CacheService } from './CacheService';
 import { User } from '../entity/User';
+import { RegistryManagerService } from './RegistryManagerService';
+import { Registry } from '../entity/Registry';
+import { BadRequestError } from 'egg-errors';
+import { ScopeManagerService } from './ScopeManagerService';
+import { EventCorkAdvice } from './EventCorkerAdvice';
 
 function isoNow() {
   return new Date().toISOString();
 }
 
+export class RegistryNotMatchError extends BadRequestError {
+}
+
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
 })
@@ -56,9 +64,19 @@ export class PackageSyncerService extends AbstractService {
   @Inject()
   private readonly httpclient: EggContextHttpClient;
   @Inject()
-  private readonly distRepository: DistRepository;
+  private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
 
   public async createTask(fullname: string, options?: SyncPackageTaskOptions) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    // sync task request registry is not same as package registry
+    if (pkg && pkg.registryId && options?.registryId) {
+      if (pkg.registryId !== options.registryId) {
+        throw new RegistryNotMatchError(`package ${fullname} is not in registry ${options.registryId}`);
+      }
+    }
     return await this.taskService.createTask(Task.createSyncPackage(fullname, options), true);
   }
 
@@ -71,7 +89,7 @@ export class PackageSyncerService extends AbstractService {
   }
 
   public async findExecuteTask() {
-    return await this.taskService.findExecuteTask(TaskType.SyncPackage);
+    return await this.taskService.findExecuteTask(TaskType.SyncPackage) as CreateSyncPackageTask;
   }
 
   public get allowSyncDownloadData() {
@@ -160,7 +178,8 @@ export class PackageSyncerService extends AbstractService {
     let useTime = Date.now() - startTime;
     while (useTime < maxTimeout) {
       // sleep 1s ~ 6s in random
-      await setTimeout(1000 + Math.random() * 5000);
+      const delay = process.env.NODE_ENV === 'test' ? 100 : 1000 + Math.random() * 5000;
+      await setTimeout(delay);
       try {
         const { data, status, url } = await this.npmRegistry.getSyncTask(fullname, logId, offset);
         useTime = Date.now() - startTime;
@@ -190,10 +209,54 @@ export class PackageSyncerService extends AbstractService {
     await this.taskService.appendTaskLog(task, logs.join('\n'));
   }
 
+  // 初始化对应的 Registry
+  // 1. 优先从 pkg.registryId 获取 (registryId 一经设置 不应改变)
+  // 1. 其次从 task.data.registryId (创建单包同步任务时传入)
+  // 2. 接着根据 scope 进行计算 (作为子包依赖同步时候，无 registryId)
+  // 3. 最后返回 default registryId (可能 default registry 也不存在)
+  public async initSpecRegistry(task: Task, pkg: Package | null = null): Promise<Registry | null> {
+    const registryId = pkg?.registryId || (task.data as SyncPackageTaskOptions).registryId;
+    let targetHost: string = this.config.cnpmcore.sourceRegistry;
+    let registry: Registry | null = null;
+
+    // 当前任务作为 deps 引入时，不会配置 registryId
+    // 历史 Task 可能没有配置 registryId
+    if (registryId) {
+      registry = await this.registryManagerService.findByRegistryId(registryId);
+    } else if (pkg?.scope) {
+      const scopeModel = await this.scopeManagerService.findByName(pkg?.scope);
+      if (scopeModel?.registryId) {
+        registry = await this.registryManagerService.findByRegistryId(scopeModel?.registryId);
+      }
+    }
+
+    // 采用默认的 registry
+    if (!registry) {
+      registry = await this.registryManagerService.findByRegistryName('default');
+    }
+
+    // 更新 targetHost 地址
+    // defaultRegistry 可能还未创建
+    if (registry?.host) {
+      targetHost = registry.host;
+    }
+    this.npmRegistry.setRegistryHost(targetHost);
+    return registry;
+  }
+
+  // 由于 cnpmcore 将 version 和 tag 作为两个独立的 changes 事件分发
+  // 普通版本发布时，短时间内会有两条相同 task 进行同步
+  // 尽量保证读取和写入都需保证任务幂等，需要确保 changes 在同步任务完成后再触发
+  // 通过 DB 唯一索引来保证任务幂等，插入失败不影响 pkg.manifests 更新
+  // 通过 eventBus.cork/uncork 来暂缓事件触发
+  @Pointcut(EventCorkAdvice)
   public async executeTask(task: Task) {
     const fullname = task.targetName;
-    const { tips, skipDependencies: originSkipDependencies, syncDownloadData } = task.data as SyncPackageTaskOptions;
-    const registry = this.npmRegistry.registry;
+    const [ scope, name ] = getScopeAndName(fullname);
+    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory } = task.data as SyncPackageTaskOptions;
+    let pkg = await this.packageRepository.findPackage(scope, name);
+    const registry = await this.initSpecRegistry(task, pkg);
+    const registryHost = this.npmRegistry.registry;
     let logs: string[] = [];
     if (tips) {
       logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`);
@@ -206,11 +269,23 @@ export class PackageSyncerService extends AbstractService {
     const logUrl = `${this.config.cnpmcore.registry}/-/package/${fullname}/syncs/${task.taskId}/log`;
     this.logger.info('[PackageSyncerService.executeTask:start] taskId: %s, targetName: %s, attempts: %s, taskQueue: %s/%s, syncUpstream: %s, log: %s',
       task.taskId, task.targetName, task.attempts, taskQueueLength, taskQueueHighWaterSize, syncUpstream, logUrl);
-    logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registry}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
+    logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registryHost}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, forceSyncHistory: ${!!forceSyncHistory} attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
     logs.push(`[${isoNow()}] 🚧 log: ${logUrl}`);
 
-    const [ scope, name ] = getScopeAndName(fullname);
-    let pkg = await this.packageRepository.findPackage(scope, name);
+    if (pkg && pkg?.registryId !== registry?.registryId) {
+      if (pkg.registryId) {
+        logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} registry is ${pkg.registryId} not belong to ${registry?.registryId}, skip sync ❌❌❌❌❌`);
+        await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+        this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, invalid registryId',
+          task.taskId, task.targetName);
+        return;
+      }
+      // 多同步源之前没有 registryId
+      // publish() 版本不变时，不会更新 registryId
+      // 在同步前，进行更新操作
+      pkg.registryId = registry?.registryId;
+      await this.packageRepository.savePackage(pkg);
+    }
 
     if (syncDownloadData && pkg) {
       await this.syncDownloadData(task, pkg);
@@ -267,10 +342,13 @@ export class PackageSyncerService extends AbstractService {
     const contentLength = headers['content-length'] || '-';
     logs.push(`[${isoNow()}] HTTP [${status}] content-length: ${contentLength}, timing: ${JSON.stringify(res.timing)}`);
 
-    if (status === 404) {
+    // 404 unpublished
+    // 451 blocked
+    const shouldRemovePkg = status === 404 || status === 451;
+    if (shouldRemovePkg) {
       if (pkg) {
         await this.packageManagerService.unpublishPackage(pkg);
-        logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was unpublished caused by 404 response: ${JSON.stringify(data)}`);
+        logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was unpublished caused by ${status} response: ${JSON.stringify(data)}`);
         logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
         logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
         await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
@@ -322,7 +400,7 @@ export class PackageSyncerService extends AbstractService {
       for (const maintainer of maintainers) {
         if (maintainer.name && maintainer.email) {
           maintainersMap[maintainer.name] = maintainer;
-          const { changed, user } = await this.userService.savePublicUser(maintainer.name, maintainer.email);
+          const { changed, user } = await this.userService.saveUser(registry?.userPrefix, maintainer.name, maintainer.email);
           users.push(user);
           if (changed) {
             changedUserCount++;
@@ -391,7 +469,7 @@ export class PackageSyncerService extends AbstractService {
       const version: string = item.version;
       if (!version) continue;
       let existsItem = existsVersionMap[version];
-      const existsAbbreviatedItem = abbreviatedVersionMap[version];
+      let existsAbbreviatedItem = abbreviatedVersionMap[version];
       const shouldDeleteReadme = !!(existsItem && 'readme' in existsItem);
       if (pkg) {
         if (existsItem) {
@@ -400,16 +478,17 @@ export class PackageSyncerService extends AbstractService {
             updateVersions.push(version);
             logs.push(`[${isoNow()}] 🐛 Remote version ${version} not exists on local abbreviated manifests, need to refresh`);
           }
-        } else {
-          // try to read from db detect if last sync interrupt before refreshPackageManifestsToDists() be called
-          existsItem = await this.distRepository.findPackageVersionManifest(pkg.packageId, version);
-          // only allow existsItem on db to force refresh, to avoid big versions fresh
-          // see https://r.cnpmjs.org/-/package/@npm-torg/public-scoped-free-org-test-package-2/syncs/61fcc7e8c1646e26a845b674/log
-          if (existsItem) {
-            // version not exists on manifests, need to refresh
-            // bugfix: https://github.com/cnpm/cnpmcore/issues/115
-            updateVersions.push(version);
-            logs.push(`[${isoNow()}] 🐛 Remote version ${version} not exists on local manifests, need to refresh`);
+        }
+
+        if (existsItem && forceSyncHistory === true) {
+          const pkgVer = await this.packageRepository.findPackageVersion(pkg.packageId, version);
+          if (pkgVer) {
+            logs.push(`[${isoNow()}] 🚧 [${syncIndex}] Remove version ${version} for force sync history`);
+            await this.packageManagerService.removePackageVersion(pkg, pkgVer, true);
+            existsItem = undefined;
+            existsAbbreviatedItem = undefined;
+            existsVersionMap[version] = undefined;
+            abbreviatedVersionMap[version] = undefined;
           }
         }
       }
@@ -486,17 +565,6 @@ export class PackageSyncerService extends AbstractService {
       if (!pkg) {
         pkg = await this.packageRepository.findPackage(scope, name);
       }
-      if (pkg) {
-        // check again, make sure prefix version not exists
-        const existsPkgVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
-        if (existsPkgVersion) {
-          await rm(localFile, { force: true });
-          logs.push(`[${isoNow()}] 🐛 [${syncIndex}] Synced version ${version} already exists, skip publish it`);
-          await this.taskService.appendTaskLog(task, logs.join('\n'));
-          logs = [];
-          continue;
-        }
-      }
 
       const publishCmd = {
         scope,
@@ -505,6 +573,7 @@ export class PackageSyncerService extends AbstractService {
         description,
         packageJson: item,
         readme,
+        registryId: registry?.registryId,
         dist: {
           localFile,
         },
@@ -513,12 +582,15 @@ export class PackageSyncerService extends AbstractService {
         skipRefreshPackageManifests: true,
       };
       try {
+        // 当 version 记录已经存在时，还需要校验一下 pkg.manifests 是否存在
         const pkgVersion = await this.packageManagerService.publish(publishCmd, users[0]);
         updateVersions.push(pkgVersion.version);
         logs.push(`[${isoNow()}] 🟢 [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
       } catch (err: any) {
         if (err.name === 'ForbiddenError') {
-          logs.push(`[${isoNow()}] 🐛 [${syncIndex}] Synced version ${version} already exists, skip publish error`);
+          logs.push(`[${isoNow()}] 🐛 [${syncIndex}] Synced version ${version} already exists, skip publish, try to set in local manifest`);
+          // 如果 pkg.manifests 不存在，需要补充一下
+          updateVersions.push(version);
         } else {
           err.taskId = task.taskId;
           this.logger.error(err);
@@ -597,6 +669,12 @@ export class PackageSyncerService extends AbstractService {
     let shouldRefreshDistTags = false;
     for (const tag in distTags) {
       const version = distTags[tag];
+      // 新 tag 指向的版本既不在存量数据里，也不在本次同步版本列表里
+      // 例如 latest 对应的 version 写入失败跳过
+      if (!existsVersionMap[version] && !updateVersions.includes(version)) {
+        logs.push(`[${isoNow()}] 🚧 invalid tag(${tag}: ${version}), version is not exists, skip`);
+        continue;
+      }
       const changed = await this.packageManagerService.savePackageTag(pkg, tag, version);
       if (changed) {
         changedTags.push({ action: 'change', tag, version });

app/core/service/RegistryManagerService.ts

@@ -0,0 +1,115 @@
+import {
+  AccessLevel,
+  ContextProto,
+  Inject,
+} from '@eggjs/tegg';
+import { E400, NotFoundError } from 'egg-errors';
+import { RegistryRepository } from '../../repository/RegistryRepository';
+import { AbstractService } from '../../common/AbstractService';
+import { Registry } from '../entity/Registry';
+import { PageOptions, PageResult } from '../util/EntityUtil';
+import { ScopeManagerService } from './ScopeManagerService';
+import { TaskService } from './TaskService';
+import { Task } from '../entity/Task';
+
+export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name'> {
+  operatorId?: string;
+}
+export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'registryId'> {
+  operatorId?: string;
+}
+export interface RemoveRegistryCmd extends Pick<Registry, 'registryId'> {
+  operatorId?: string;
+}
+
+export interface StartSyncCmd {
+  registryId: string;
+  since?: string;
+  operatorId?: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class RegistryManagerService extends AbstractService {
+  @Inject()
+  private readonly registryRepository: RegistryRepository;
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async createSyncChangesStream(startSyncCmd: StartSyncCmd): Promise<void> {
+    const { registryId, operatorId = '-', since } = startSyncCmd;
+    this.logger.info('[RegistryManagerService.startSyncChangesStream:prepare] operatorId: %s, registryId: %s, since: %s', operatorId, registryId, since);
+    const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
+    if (!registry) {
+      throw new NotFoundError(`registry ${registryId} not found`);
+    }
+
+    // 防止和 GLOBAL_WORKER 冲突，只能有一个默认的全局 registry
+    const scopesCount = await this.scopeManagerService.countByRegistryId(registryId);
+    if (scopesCount === 0) {
+      throw new E400(`registry ${registryId} has no scopes, please create scopes first`);
+    }
+
+    // 启动 changeStream
+    const targetName = `${registry.name.toUpperCase()}_WORKER`;
+    await this.taskService.createTask(Task.createChangesStream(targetName, registryId, since), false);
+  }
+
+  async createRegistry(createCmd: CreateRegistryCmd): Promise<Registry> {
+    const { name, changeStream, host, userPrefix, type, operatorId = '-' } = createCmd;
+    this.logger.info('[RegistryManagerService.createRegistry:prepare] operatorId: %s, createCmd: %j', operatorId, createCmd);
+    const registry = Registry.create({
+      name,
+      changeStream,
+      host,
+      userPrefix,
+      type,
+    });
+    await this.registryRepository.saveRegistry(registry);
+    return registry;
+  }
+
+  // 更新部分 registry 信息
+  // 不允许 userPrefix 字段变更
+  async updateRegistry(updateCmd: UpdateRegistryCmd) {
+    const { name, changeStream, host, type, registryId, operatorId = '-' } = updateCmd;
+    this.logger.info('[RegistryManagerService.updateRegistry:prepare] operatorId: %s, updateCmd: %j', operatorId, updateCmd);
+    const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
+    if (!registry) {
+      throw new NotFoundError(`registry ${registryId} not found`);
+    }
+    Object.assign(registry, {
+      name,
+      changeStream,
+      host,
+      type,
+    });
+    await this.registryRepository.saveRegistry(registry);
+  }
+
+  // list all registries with scopes
+  async listRegistries(page: PageOptions): Promise<PageResult<Registry>> {
+    return await this.registryRepository.listRegistries(page);
+  }
+
+  async findByRegistryId(registryId: string): Promise<Registry | null> {
+    return await this.registryRepository.findRegistryByRegistryId(registryId);
+  }
+
+  async findByRegistryName(registryName?: string): Promise<Registry | null> {
+    return await this.registryRepository.findRegistry(registryName);
+  }
+
+  // 删除 Registry 方法
+  // 可选传入 operatorId 作为参数，用于记录操作人员
+  // 同时删除对应的 scope 数据
+  async remove(removeCmd: RemoveRegistryCmd): Promise<void> {
+    const { registryId, operatorId = '-' } = removeCmd;
+    this.logger.info('[RegistryManagerService.remove:prepare] operatorId: %s, registryId: %s', operatorId, registryId);
+    await this.registryRepository.removeRegistry(registryId);
+    await this.scopeManagerService.removeByRegistryId({ registryId, operatorId });
+  }
+}

app/core/service/ScopeManagerService.ts

@@ -0,0 +1,74 @@
+import {
+  AccessLevel,
+  ContextProto,
+  Inject,
+} from '@eggjs/tegg';
+import { ScopeRepository } from '../../repository/ScopeRepository';
+import { AbstractService } from '../../common/AbstractService';
+import { Scope } from '../entity/Scope';
+import { PageOptions, PageResult } from '../util/EntityUtil';
+
+export interface CreateScopeCmd extends Pick<Scope, 'name' | 'registryId'> {
+  operatorId?: string;
+}
+export interface UpdateRegistryCmd extends Pick<Scope, 'name' | 'scopeId' | 'registryId'> {
+  operatorId?: string;
+}
+
+export interface RemoveScopeCmd {
+  scopeId: string;
+  operatorId?: string;
+}
+
+export interface RemoveScopeByRegistryIdCmd {
+  registryId: string;
+  operatorId?: string;
+}
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class ScopeManagerService extends AbstractService {
+  @Inject()
+  private readonly scopeRepository: ScopeRepository;
+
+  async findByName(name: string): Promise<Scope | null> {
+    const scope = await this.scopeRepository.findByName(name);
+    return scope;
+  }
+
+  async countByRegistryId(registryId: string): Promise<number> {
+    const count = await this.scopeRepository.countByRegistryId(registryId);
+    return count;
+  }
+
+  async createScope(createCmd: CreateScopeCmd): Promise<Scope> {
+    const { name, registryId, operatorId } = createCmd;
+    this.logger.info('[ScopeManagerService.CreateScope:prepare] operatorId: %s, createCmd: %s', operatorId, createCmd);
+    const scope = Scope.create({
+      name,
+      registryId,
+    });
+    await this.scopeRepository.saveScope(scope);
+    return scope;
+  }
+
+  async listScopes(page: PageOptions): Promise<PageResult<Scope>> {
+    return await this.scopeRepository.listScopes(page);
+  }
+
+  async listScopesByRegistryId(registryId: string, page: PageOptions): Promise<PageResult<Scope>> {
+    return await this.scopeRepository.listScopesByRegistryId(registryId, page);
+  }
+
+  async removeByRegistryId(removeCmd: RemoveScopeByRegistryIdCmd): Promise<void> {
+    const { registryId, operatorId } = removeCmd;
+    this.logger.info('[ScopeManagerService.remove:prepare] operatorId: %s, registryId: %s', operatorId, registryId);
+    return await this.scopeRepository.removeScopeByRegistryId(registryId);
+  }
+
+  async remove(removeCmd: RemoveScopeCmd): Promise<void> {
+    const { scopeId, operatorId } = removeCmd;
+    this.logger.info('[ScopeManagerService.remove:prepare] operatorId: %s, scopeId: %s', operatorId, scopeId);
+    return await this.scopeRepository.removeScope(scopeId);
+  }
+}

app/core/service/TaskService.ts

@@ -8,7 +8,7 @@ import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
 import { Task } from '../entity/Task';
-import { QueueAdapter } from '../../common/adapter/QueueAdapter';
+import { QueueAdapter } from '../../common/typing';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -28,21 +28,27 @@ export class TaskService extends AbstractService {
   public async createTask(task: Task, addTaskQueueOnExists: boolean) {
     const existsTask = await this.taskRepository.findTaskByTargetName(task.targetName, task.type);
     if (existsTask) {
-      if (addTaskQueueOnExists && existsTask.state === TaskState.Waiting) {
-        const queueLength = await this.getTaskQueueLength(task.type);
-        if (queueLength < this.config.cnpmcore.taskQueueHighWaterSize) {
-          // make sure waiting task in queue
-          await this.queueAdapter.push<string>(task.type, existsTask.taskId);
-          this.logger.info('[TaskService.createTask:exists-to-queue] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
-            task.type, task.targetName, task.taskId, queueLength);
+      // 如果任务还未被触发，就不继续重复创建
+      // 如果任务正在执行，可能任务状态已更新，这种情况需要继续创建
+      if (existsTask.state === TaskState.Waiting) {
+        // 提高任务的优先级
+        if (addTaskQueueOnExists) {
+          const queueLength = await this.getTaskQueueLength(task.type);
+          if (queueLength < this.config.cnpmcore.taskQueueHighWaterSize) {
+            // make sure waiting task in queue
+            await this.queueAdapter.push<string>(task.type, existsTask.taskId);
+            this.logger.info('[TaskService.createTask:exists-to-queue] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
+              task.type, task.targetName, task.taskId, queueLength);
+          }
         }
+        return existsTask;
       }
-      return existsTask;
     }
     await this.taskRepository.saveTask(task);
-    const queueSize = await this.queueAdapter.push<string>(task.type, task.taskId);
+    await this.queueAdapter.push<string>(task.type, task.taskId);
+    const queueLength = await this.getTaskQueueLength(task.type);
     this.logger.info('[TaskService.createTask:new] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
-      task.type, task.targetName, task.taskId, queueSize);
+      task.type, task.targetName, task.taskId, queueLength);
     return task;
   }
 
@@ -51,34 +57,48 @@ export class TaskService extends AbstractService {
       await this.appendLogToNFS(task, appendLog);
     }
     task.state = TaskState.Waiting;
-    // make sure updatedAt changed
-    task.updatedAt = new Date();
     await this.taskRepository.saveTask(task);
-    const queueSize = await this.queueAdapter.push<string>(task.type, task.taskId);
+    await this.queueAdapter.push<string>(task.type, task.taskId);
+    const queueLength = await this.getTaskQueueLength(task.type);
     this.logger.info('[TaskService.retryTask:save] taskType: %s, targetName: %s, taskId: %s, queue size: %s',
-      task.type, task.targetName, task.taskId, queueSize);
+      task.type, task.targetName, task.taskId, queueLength);
   }
 
   public async findTask(taskId: string) {
     return await this.taskRepository.findTask(taskId);
   }
 
+  public async findTasks(taskIdList: Array<string>) {
+    return await this.taskRepository.findTasks(taskIdList);
+  }
+
   public async findTaskLog(task: Task) {
     return await this.nfsAdapter.getDownloadUrlOrStream(task.logPath);
   }
 
   public async findExecuteTask(taskType: TaskType) {
-    const taskId = await this.queueAdapter.pop<string>(taskType);
-    if (taskId) {
-      const task = await this.taskRepository.findTask(taskId);
-      if (task) {
-        task.setExecuteWorker();
-        task.state = TaskState.Processing;
-        task.attempts += 1;
-        await this.taskRepository.saveTask(task);
-        return task;
+    let taskId = await this.queueAdapter.pop<string>(taskType);
+    let task: Task | null;
+
+    while (taskId) {
+      task = await this.taskRepository.findTask(taskId);
+
+      // 任务已删除或任务已执行
+      // 继续取下一个任务
+      if (task === null || task?.state !== TaskState.Waiting) {
+        taskId = await this.queueAdapter.pop<string>(taskType);
+        continue;
       }
+
+      const condition = task.start();
+      const saveSucceed = await this.taskRepository.idempotentSaveTask(task, condition);
+      if (!saveSucceed) {
+        taskId = await this.queueAdapter.pop<string>(taskType);
+        continue;
+      }
+      return task;
     }
+
     return null;
   }
 
@@ -99,7 +119,7 @@ export class TaskService extends AbstractService {
         task.resetLogPath();
       }
       await this.retryTask(task);
-      this.logger.warn(
+      this.logger.info(
         '[TaskService.retryExecuteTimeoutTasks:retry] taskType: %s, targetName: %s, taskId: %s, attempts %s will retry again',
         task.type, task.targetName, task.taskId, task.attempts);
     }
@@ -119,7 +139,6 @@ export class TaskService extends AbstractService {
 
   public async appendTaskLog(task: Task, appendLog: string) {
     await this.appendLogToNFS(task, appendLog);
-    task.updatedAt = new Date();
     await this.taskRepository.saveTask(task);
   }
 

app/core/service/UserService.ts

@@ -70,8 +70,8 @@ export class UserService extends AbstractService {
     return { user: userEntity, token };
   }
 
-  async savePublicUser(name: string, email: string): Promise<{ changed: boolean, user: UserEntity }> {
-    const storeName = name.startsWith('name:') ? name : `npm:${name}`;
+  async saveUser(userPrefix = 'npm:', name: string, email: string): Promise<{ changed: boolean, user: UserEntity }> {
+    const storeName = name.startsWith('name:') ? name : `${userPrefix}${name}`;
     let user = await this.userRepository.findUserByName(storeName);
     if (!user) {
       const passwordSalt = crypto.randomBytes(20).toString('hex');

app/core/typing.ts

@@ -0,0 +1,5 @@
+import { ChangesStreamService } from './service/ChangesStreamService';
+
+export interface ContextCnpmcore {
+  changesStreamService: ChangesStreamService;
+}

app/core/util/EntityUtil.ts

@@ -1,10 +1,24 @@
 import { EntityData } from '../entity/Entity';
 import ObjectID from 'bson-objectid';
+import { E400 } from 'egg-errors';
 
 type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
 
 export type EasyData<T extends EntityData, Id extends keyof T> = PartialBy<T, 'createdAt' | 'updatedAt' | Id>;
 
+const MAX_PAGE_SIZE = 100 as const;
+export interface PageOptions {
+  pageSize?: number;
+  pageIndex?: number;
+}
+export interface PageResult<T> {
+  count: number;
+  data: Array<T>
+}
+export interface PageLimitOptions {
+  offset: number;
+  limit: number;
+}
 
 export class EntityUtil {
   static defaultData<T extends EntityData, Id extends keyof T>(data: EasyData<T, Id>, id: Id): T {
@@ -17,4 +31,15 @@ export class EntityUtil {
   static createId(): string {
     return new ObjectID().toHexString();
   }
+
+  static convertPageOptionsToLimitOption(page: PageOptions): PageLimitOptions {
+    const { pageIndex = 0, pageSize = 20 } = page;
+    if (pageSize > MAX_PAGE_SIZE) {
+      throw new E400(`max page size is 100, current request is ${pageSize}`);
+    }
+    return {
+      offset: pageIndex * pageSize,
+      limit: pageSize,
+    };
+  }
 }

app/infra/QueueAdapter.ts

@@ -0,0 +1,47 @@
+import {
+  AccessLevel,
+  Inject,
+  ContextProto,
+} from '@eggjs/tegg';
+import { Redis } from 'ioredis';
+import { QueueAdapter } from '../common/typing';
+
+/**
+ * Use sort set to keep queue in order and keep same value only insert once
+ */
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+  name: 'queueAdapter',
+})
+export class RedisQueueAdapter implements QueueAdapter {
+  @Inject()
+  private readonly redis: Redis;
+
+  private getQueueName(key: string) {
+    return `CNPMCORE_Q_V2_${key}`;
+  }
+
+  private getQueueScoreName(key: string) {
+    return `CNPMCORE_Q_S_V2_${key}`;
+  }
+
+  /**
+   * If queue has the same item, return false
+   * If queue not has the same item, return true
+   */
+  async push<T>(key: string, item: T): Promise<boolean> {
+    const score = await this.redis.incr(this.getQueueScoreName(key));
+    const res = await this.redis.zadd(this.getQueueName(key), score, JSON.stringify(item));
+    return res !== 0;
+  }
+
+  async pop<T>(key: string) {
+    const [ json ] = await this.redis.zpopmin(this.getQueueName(key));
+    if (!json) return null;
+    return JSON.parse(json) as T;
+  }
+
+  async length(key: string) {
+    return await this.redis.zcount(this.getQueueName(key), '-inf', '+inf');
+  }
+}

app/port/UserRoleManager.ts

@@ -14,7 +14,7 @@ import { Token as TokenEntity } from '../core/entity/Token';
 import { sha512 } from '../common/UserUtil';
 
 // https://docs.npmjs.com/creating-and-viewing-access-tokens#creating-tokens-on-the-website
-type TokenRole = 'read' | 'publish' | 'setting';
+export type TokenRole = 'read' | 'publish' | 'setting';
 
 @ContextProto({
   // only inject on port module

app/port/binary.html

@@ -1,9 +1,9 @@
-<!DOCTYPE html> 
-<html> 
-  <head> 
+<!DOCTYPE html>
+<html>
+  <head>
     <title>CNPM Binaries Mirror</title>
-  </head> 
-  <body> 
+  </head>
+  <body>
     <script>
       // Forked from https://chromedriver.storage.googleapis.com/index.html
       // Split a string in 2 parts. The first is the leading number, if any,
@@ -52,11 +52,11 @@
         // the lowest.
         if (isNaN(numA) == false) return -1
         if (isNaN(numB) == false) return 1
-       
-        // They are both strings. 
+
+        // They are both strings.
         return (a < b) ? -1 : (a > b ? 1 : 0)
       }
- 
+
       // Helper function to retrieve the value of a GET query parameter.
       // Greatly inspired from http://alturl.com/8rj7a
       function getParameter(parameterName) {
@@ -66,26 +66,26 @@
         if (queryString.length <= 0) {
           return '';
         }
-        
+
         // Find the beginning of the string
         begin = queryString.indexOf(parameterName);
- 
+
         // If the parameter name is not found, skip it, otherwise return the
         // value.
         if (begin == -1) {
           return '';
         }
-         
+
         // Add the length (integer) to the beginning.
         begin += parameterName.length;
- 
+
         // Multiple parameters are separated by the '&' sign.
         end = queryString.indexOf ('&', begin);
-        
+
         if (end == -1) {
           end = queryString.length;
         }
-        
+
         // Return the string.
         return escape(unescape(queryString.substring(begin, end)));
       }
@@ -94,7 +94,7 @@
       function displayList(items, root, path) {
         // Display the header
         document.write('<h1>Index of /' + path + '</h1>');
- 
+
         // Start the table for the results.
         document.write('<table style="border-spacing:15px 0px;">');
 
@@ -103,18 +103,18 @@
         if (sortOrder != 'desc') {
           sortLink += '&sort=desc';
         }
- 
+
         // Display the table header.
         document.write('<tr><th><img src="https://gw.alipayobjects.com/mdn/rms_fa382b/afts/img/A*v6fRRLopV_0AAAAAAAAAAAAAARQnAQ" alt="[ICO]"></th>');
         document.write('<th><a href="' + sortLink + '">Name</a></th>');
         document.write('<th>Last modified</th>');
         document.write('<th>Size</th>');
         document.write('<tr><th colspan="5"><hr></th></tr>');
- 
+
         // Display the 'go back' button.
         if (path != '') {
           var backpath = location.pathname;
- 
+
           // If there is more than one section delimited by '/' in the current
           // path we truncate the last section and append the rest to backpath.
           var delimiter = path.lastIndexOf('/');
@@ -125,15 +125,15 @@
               backpath += path.substr(0, delimiter+1);
             }
           }
- 
+
           document.write('<tr><td valign="top"><img src="https://gw.alipayobjects.com/mdn/rms_fa382b/afts/img/A*3QmJSqp2zpUAAAAAAAAAAAAAARQnAQ" alt="[DIR]"></td>');
           document.write('<td><a href="');
           document.write(backpath);
           document.write('">Parent Directory</a></td>');
           document.write('<td>&nbsp;</td>');
-          document.write('<td align="right">  - </td></tr>'); 
+          document.write('<td align="right">  - </td></tr>');
         }
- 
+
         // Set up the variables.
         var directories = new Array();
         var files = new Array();
@@ -146,7 +146,7 @@
             directories.push(item);
           }
         }
- 
+
         files.sort(alphanumCompare);
         directories.sort(alphanumCompare);
 
@@ -155,13 +155,18 @@
           files.reverse();
           directories.reverse();
         }
-      
+
         // Display the directories.
         for (var i = 0; i < directories.length; i++) {
           var lnk = location.pathname.substr(0, location.pathname.indexOf('?'));
           var item = directories[i];
-          lnk += '?path=' + path + item.name;
-      
+          if (path && !path.endsWith('/')) {
+            lnk += '?path=' + path + '/' + item.name;
+          } else {
+            lnk += '?path=' + path + item.name;
+          }
+
+
           document.write('<tr>');
           document.write('<td valign="top"><img src="https://gw.alipayobjects.com/mdn/rms_fa382b/afts/img/A*ct35SJLile8AAAAAAAAAAAAAARQnAQ" alt="[DIR]"></td>');
           document.write('<td><a href="' + lnk + '">' +
@@ -170,7 +175,7 @@
           document.write('<td align="right">-</td>');
           document.write('</tr>');
         }
-      
+
         // Display the files.
         for (var i = 0; i < files.length; i++) {
           var item = files[i];
@@ -189,16 +194,16 @@
           if (sizeUnit !== '') {
             size = size.toFixed(2) + sizeUnit;
           }
-          var lastModified = item.date;          
+          var lastModified = item.date;
           // Remove the entries we don't want to show.
           if (filename == '') {
             continue;
           }
-        
+
           if (filename.indexOf('$folder$') >= 0) {
             continue;
           }
-        
+
           // Display the row.
           document.write('<tr>');
           document.write('<td valign="top"><img src="https://gw.alipayobjects.com/mdn/rms_fa382b/afts/img/A*FKvWRo-vns4AAAAAAAAAAAAAARQnAQ" alt="[DIR]"></td>');
@@ -208,13 +213,13 @@
           document.write('<td align="right">' + size + '</td>');
           document.write('</tr>');
         }
- 
+
         // Close the table.
         document.write('<tr><th colspan="5"><hr></th></tr>');
         document.write('</table>');
         document.title = 'CNPM Binaries Mirror';
       }
- 
+
       function fetchAndDisplay() {
         var path = getParameter('path');
         var lastSlash = location.pathname.lastIndexOf("/");
@@ -238,6 +243,6 @@
         }
       }
       fetchAndDisplay();
-    </script> 
-  </body> 
+    </script>
+  </body>
 </html>

app/port/controller/AbstractController.ts

@@ -42,8 +42,8 @@ export abstract class AbstractController extends MiddlewareController {
     return this.config.cnpmcore.sourceRegistry;
   }
 
-  protected get enableSyncAll() {
-    return this.config.cnpmcore.syncMode === 'all';
+  protected get enableSync() {
+    return this.config.cnpmcore.syncMode === 'all' || this.config.cnpmcore.syncMode === 'exist';
   }
 
   protected isPrivateScope(scope: string) {
@@ -57,7 +57,7 @@ export abstract class AbstractController extends MiddlewareController {
     // dont sync private scope
     if (!this.isPrivateScope(scope)) {
       // syncMode = none, redirect public package to source registry
-      if (!this.enableSyncAll) {
+      if (!this.enableSync) {
         err.redirectToSourceRegistry = this.sourceRegistry;
       }
     }

app/port/controller/BinarySyncController.ts

@@ -33,9 +33,10 @@ export class BinarySyncController extends AbstractController {
     method: HTTPMethodEnum.GET,
   })
   async listBinaries() {
-    return Object.values(binaries).map(binaryConfig => {
+    return Object.entries(binaries).map(([ binaryName, binaryConfig ]) => {
       return {
-        name: `${binaryConfig.category}/`,
+        name: `${binaryName}/`,
+        category: `${binaryConfig.category}/`,
         description: binaryConfig.description,
         distUrl: binaryConfig.distUrl,
         repoUrl: /^https?:\/\//.test(binaryConfig.repo) ? binaryConfig.repo : `https://github.com/${binaryConfig.repo}`,
@@ -59,7 +60,18 @@ export class BinarySyncController extends AbstractController {
     const parsed = path.parse(subpath);
     const parent = parsed.dir === '/' ? '/' : `${parsed.dir}/`;
     const name = subpath.endsWith('/') ? `${parsed.base}/` : parsed.base;
-    const binary = await this.binarySyncerService.findBinary(binaryName, parent, name);
+    // 首先查询 binary === category 的情况
+    let binary = await this.binarySyncerService.findBinary(binaryName, parent, name);
+    if (!binary) {
+      // 查询不到再去查询 mergeCategory 的情况
+      const category = binaries?.[binaryName]?.category;
+      if (category) {
+        // canvas/v2.6.1/canvas-v2.6.1-node-v57-linux-glibc-x64.tar.gz
+        // -> node-canvas-prebuilt/v2.6.1/node-canvas-prebuilt-v2.6.1-node-v57-linux-glibc-x64.tar.gz
+        binary = await this.binarySyncerService.findBinary(category, parent, name.replace(new RegExp(`^${binaryName}-`), `${category}-`));
+      }
+    }
+
     if (!binary) {
       throw new NotFoundError(`Binary "${binaryName}${subpath}" not found`);
     }

app/port/controller/HookController.ts

@@ -0,0 +1,128 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  Inject,
+} from '@eggjs/tegg';
+import { HookManageService } from '../../core/service/HookManageService';
+import { TaskService } from '../../core/service/TaskService';
+import { UserRoleManager } from '../UserRoleManager';
+import { HookType } from '../../common/enum/Hook';
+import { TriggerHookTask } from '../../core/entity/Task';
+import { HookConvertor } from './convertor/HookConvertor';
+import { CreateHookRequestRule, UpdateHookRequestRule } from '../typebox';
+
+export interface CreateHookRequest {
+  type: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+}
+
+export interface UpdateHookRequest {
+  endpoint: string;
+  secret: string;
+}
+
+@HTTPController({
+  path: '/-/npm',
+})
+export class HookController {
+  @Inject()
+  private readonly hookManageService: HookManageService;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  @Inject()
+  private readonly userRoleManager: UserRoleManager;
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook',
+    method: HTTPMethodEnum.POST,
+  })
+  async createHook(@Context() ctx: EggContext, @HTTPBody() req: CreateHookRequest) {
+    ctx.tValidate(CreateHookRequestRule, req);
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.createHook({
+      ownerId: user.userId,
+      type: req.type as HookType,
+      name: req.name,
+      endpoint: req.endpoint,
+      secret: req.secret,
+    });
+    return HookConvertor.convertToHookVo(hook, user);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.PUT,
+  })
+  async updateHook(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() req: UpdateHookRequest) {
+    ctx.tValidate(UpdateHookRequestRule, req);
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.updateHook({
+      operatorId: user.userId,
+      hookId: id,
+      endpoint: req.endpoint,
+      secret: req.secret,
+    });
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToHookVo(hook, user, task);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  async deleteHook(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const hook = await this.hookManageService.deleteHook({
+      operatorId: user.userId,
+      hookId: id,
+    });
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToDeleteHookVo(hook, user, task);
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks',
+    method: HTTPMethodEnum.GET,
+  })
+  async listHooks(@Context() ctx: EggContext) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const hooks = await this.hookManageService.listHooksByOwnerId(user.userId);
+    const tasks = await this.taskService.findTasks(hooks.map(t => t.latestTaskId).filter((t): t is string => !!t));
+    const res = hooks.map(hook => {
+      const task = tasks.find(t => t.taskId === hook.latestTaskId) as TriggerHookTask;
+      return HookConvertor.convertToHookVo(hook, user, task);
+    });
+    return {
+      objects: res,
+    };
+  }
+
+  @HTTPMethod({
+    path: '/v1/hooks/hook/:id',
+    method: HTTPMethodEnum.GET,
+  })
+  async getHook(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const user = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const hook = await this.hookManageService.getHookByOwnerId(id, user.userId);
+    let task: TriggerHookTask | null = null;
+    if (hook.latestTaskId) {
+      task = await this.taskService.findTask(hook.latestTaskId) as TriggerHookTask;
+    }
+    return HookConvertor.convertToHookVo(hook, user, task);
+  }
+}

app/port/controller/PackageSyncController.ts

@@ -8,12 +8,14 @@ import {
   EggContext,
   Inject,
   HTTPQuery,
+  BackgroundTaskHelper,
 } from '@eggjs/tegg';
 import { ForbiddenError, NotFoundError } from 'egg-errors';
 import { AbstractController } from './AbstractController';
 import { FULLNAME_REG_STRING, getScopeAndName } from '../../common/PackageUtil';
 import { Task } from '../../core/entity/Task';
 import { PackageSyncerService } from '../../core/service/PackageSyncerService';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
 import { TaskState } from '../../common/enum/Task';
 import { SyncPackageTaskRule, SyncPackageTaskType } from '../typebox';
 
@@ -22,6 +24,12 @@ export class PackageSyncController extends AbstractController {
   @Inject()
   private packageSyncerService: PackageSyncerService;
 
+  @Inject()
+  private backgroundTaskHelper: BackgroundTaskHelper;
+
+  @Inject()
+  private registryManagerService: RegistryManagerService;
+
   private async _executeTaskAsync(task: Task) {
     const startTime = Date.now();
     this.logger.info('[PackageSyncController:executeTask:start] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
@@ -46,26 +54,38 @@ export class PackageSyncController extends AbstractController {
     method: HTTPMethodEnum.PUT,
   })
   async createSyncTask(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPBody() data: SyncPackageTaskType) {
-    if (!this.enableSyncAll) {
+    if (!this.enableSync) {
       throw new ForbiddenError('Not allow to sync package');
     }
     const tips = data.tips || `Sync cause by "${ctx.href}", parent traceId: ${ctx.tracer.traceId}`;
+    const isAdmin = await this.userRoleManager.isAdmin(ctx);
+
     const params = {
       fullname,
       tips,
       skipDependencies: !!data.skipDependencies,
       syncDownloadData: !!data.syncDownloadData,
       force: !!data.force,
+      // only admin allow to sync history version
+      forceSyncHistory: !!data.forceSyncHistory && isAdmin,
     };
     ctx.tValidate(SyncPackageTaskRule, params);
     const [ scope, name ] = getScopeAndName(params.fullname);
     const packageEntity = await this.packageRepository.findPackage(scope, name);
-    if (packageEntity?.isPrivate) {
+    const registry = await this.registryManagerService.findByRegistryName(data?.registryName);
+
+    if (!registry && data.registryName) {
+      throw new ForbiddenError(`Can\'t find target registry "${data.registryName}"`);
+    }
+    if (packageEntity?.isPrivate && !registry) {
       throw new ForbiddenError(`Can\'t sync private package "${params.fullname}"`);
     }
     if (params.syncDownloadData && !this.packageSyncerService.allowSyncDownloadData) {
       throw new ForbiddenError('Not allow to sync package download data');
     }
+    if (registry && packageEntity?.registryId && packageEntity.registryId !== registry.registryId) {
+      throw new ForbiddenError(`The package is synced from ${packageEntity.registryId}`);
+    }
     const authorized = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
     const task = await this.packageSyncerService.createTask(params.fullname, {
       authorIp: ctx.ip,
@@ -73,16 +93,21 @@ export class PackageSyncController extends AbstractController {
       tips: params.tips,
       skipDependencies: params.skipDependencies,
       syncDownloadData: params.syncDownloadData,
+      forceSyncHistory: params.forceSyncHistory,
+      registryId: registry?.registryId,
     });
     ctx.logger.info('[PackageSyncController.createSyncTask:success] taskId: %s, fullname: %s',
       task.taskId, fullname);
     if (data.force) {
-      const isAdmin = await this.userRoleManager.isAdmin(ctx);
       if (isAdmin) {
-        // execute task in background
-        this._executeTaskAsync(task);
-        ctx.logger.info('[PackageSyncController.createSyncTask:execute-immediately] taskId: %s',
-          task.taskId);
+        // set background task timeout to 5min
+        this.backgroundTaskHelper.timeout = 1000 * 60 * 5;
+        this.backgroundTaskHelper.run(async () => {
+          ctx.logger.info('[PackageSyncController.createSyncTask:execute-immediately] taskId: %s',
+            task.taskId);
+          // execute task in background
+          await this._executeTaskAsync(task);
+        });
       }
     }
     ctx.status = 201;
@@ -153,6 +178,7 @@ export class PackageSyncController extends AbstractController {
       skipDependencies: nodeps === 'true',
       syncDownloadData: false,
       force: false,
+      forceSyncHistory: false,
     };
     const task = await this.createSyncTask(ctx, fullname, options);
     return {

app/port/controller/RegistryController.ts

@@ -0,0 +1,110 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  HTTPQuery,
+  Inject,
+  Middleware,
+} from '@eggjs/tegg';
+import { NotFoundError } from 'egg-errors';
+import { AbstractController } from './AbstractController';
+import { Static } from 'egg-typebox-validate/typebox';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { AdminAccess } from '../middleware/AdminAccess';
+import { ScopeManagerService } from '../../core/service/ScopeManagerService';
+import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions } from '../typebox';
+
+@HTTPController()
+export class RegistryController extends AbstractController {
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+
+  @HTTPMethod({
+    path: '/-/registry',
+    method: HTTPMethodEnum.GET,
+  })
+  async listRegistries(@HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'], @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex']) {
+    const registries = await this.registryManagerService.listRegistries({ pageSize, pageIndex });
+    return registries;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.GET,
+  })
+  async showRegistry(@HTTPParam() id: string) {
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    return registry;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id/scopes',
+    method: HTTPMethodEnum.GET,
+  })
+  async showRegistryScopes(@HTTPParam() id: string, @HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'], @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex']) {
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    const scopes = await this.scopeManagerService.listScopesByRegistryId(id, { pageIndex, pageSize });
+    return scopes;
+  }
+
+  @HTTPMethod({
+    path: '/-/registry',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createRegistry(@Context() ctx: EggContext, @HTTPBody() registryOptions: Static<typeof RegistryCreateOptions>) {
+    ctx.tValidate(RegistryCreateOptions, registryOptions);
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    const { name, changeStream, host, userPrefix = '', type } = registryOptions;
+    await this.registryManagerService.createRegistry({
+      name,
+      changeStream,
+      host,
+      userPrefix,
+      operatorId: authorizedUser.userId,
+      type,
+    });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id/sync',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createRegistrySyncTask(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() registryOptions: Static<typeof RegistryCreateSyncOptions>) {
+    ctx.tValidate(RegistryCreateSyncOptions, registryOptions);
+    const { since } = registryOptions;
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    }
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.registryManagerService.createSyncChangesStream({ registryId: registry.registryId, since, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async removeRegistry(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.registryManagerService.remove({ registryId: id, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+}

app/port/controller/ScopeController.ts

@@ -0,0 +1,63 @@
+import {
+  Context,
+  EggContext,
+  HTTPBody,
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  Inject,
+  Middleware,
+} from '@eggjs/tegg';
+import { E400 } from 'egg-errors';
+import { AbstractController } from './AbstractController';
+import { Static } from 'egg-typebox-validate/typebox';
+import { AdminAccess } from '../middleware/AdminAccess';
+import { ScopeManagerService } from '../../core/service/ScopeManagerService';
+import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { ScopeCreateOptions } from '../typebox';
+
+
+@HTTPController()
+export class ScopeController extends AbstractController {
+  @Inject()
+  private readonly scopeManagerService: ScopeManagerService;
+
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
+
+  @HTTPMethod({
+    path: '/-/scope',
+    method: HTTPMethodEnum.POST,
+  })
+  @Middleware(AdminAccess)
+  async createScope(@Context() ctx: EggContext, @HTTPBody() scopeOptions: Static<typeof ScopeCreateOptions>) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    ctx.tValidate(ScopeCreateOptions, scopeOptions);
+    const { name, registryId } = scopeOptions;
+
+    const registry = await this.registryManagerService.findByRegistryId(registryId);
+    if (!registry) {
+      throw new E400(`registry ${registryId} not found`);
+    }
+
+    await this.scopeManagerService.createScope({
+      name,
+      registryId,
+      operatorId: authorizedUser.userId,
+    });
+    return { ok: true };
+  }
+
+  @HTTPMethod({
+    path: '/-/scope/:id',
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async removeScope(@Context() ctx: EggContext, @HTTPParam() id: string) {
+    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
+    await this.scopeManagerService.remove({ scopeId: id, operatorId: authorizedUser.userId });
+    return { ok: true };
+  }
+
+}

app/port/controller/convertor/HookConvertor.ts

@@ -0,0 +1,61 @@
+import { Hook } from '../../../core/entity/Hook';
+import { TriggerHookTask } from '../../../core/entity/Task';
+import { User } from '../../../core/entity/User';
+import { HookType } from '../../../common/enum/Hook';
+
+export interface HookVo {
+  id: string;
+  username: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  type: HookType;
+  created: Date;
+  updated: Date;
+  delivered: boolean,
+  last_delivery: Date | null,
+  response_code: number,
+  status: 'active',
+}
+
+export interface DeleteHookVo {
+  id: string;
+  username: string;
+  name: string;
+  endpoint: string;
+  secret: string;
+  type: HookType;
+  created: Date;
+  updated: Date;
+  delivered: boolean,
+  last_delivery: Date | null,
+  response_code: number,
+  status: 'active',
+  deleted: boolean,
+}
+
+export class HookConvertor {
+  static convertToHookVo(hook: Hook, user: User, task?: TriggerHookTask | null | undefined): HookVo {
+    return {
+      id: hook.hookId,
+      username: user.name,
+      name: hook.name,
+      endpoint: hook.endpoint,
+      secret: hook.secret,
+      type: hook.type,
+      created: hook.createdAt,
+      updated: hook.updatedAt,
+      delivered: !!task,
+      last_delivery: task?.updatedAt || null,
+      response_code: task?.data.responseStatus || 0,
+      status: 'active',
+    };
+  }
+
+  static convertToDeleteHookVo(hook: Hook, user: User, task?: TriggerHookTask | null): DeleteHookVo {
+    const vo = HookConvertor.convertToHookVo(hook, user, task);
+    return Object.assign(vo, {
+      deleted: true,
+    });
+  }
+}

app/port/controller/package/ShowPackageController.ts

@@ -33,7 +33,7 @@ export class ShowPackageController extends AbstractController {
     const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType;
     // handle cache
     const cacheEtag = await this.cacheService.getPackageEtag(fullname, isFullManifests);
-    if (cacheEtag) {
+    if (!isSync && cacheEtag) {
       let requestEtag = ctx.request.get('if-none-match');
       if (requestEtag.startsWith('W/')) {
         requestEtag = requestEtag.substring(2);

app/port/controller/package/UpdatePackageController.ts

@@ -38,13 +38,13 @@ export class UpdatePackageController extends AbstractController {
     method: HTTPMethodEnum.PUT,
   })
   async update(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPBody() data: Maintainer) {
-    const npmCommand = ctx.get('npm-command');
-    if (npmCommand === 'unpublish') {
+    if (this.isNpmCommandValid(ctx, 'unpublish')) {
       // ignore it
       return { ok: false };
     }
     // only support update maintainer
-    if (npmCommand !== 'owner') {
+    if (!this.isNpmCommandValid(ctx, 'owner')) {
+      const npmCommand = this.getNpmCommand(ctx);
       throw new BadRequestError(`header: npm-command expected "owner", but got "${npmCommand}"`);
     }
     ctx.tValidate(MaintainerDataRule, data);
@@ -61,4 +61,21 @@ export class UpdatePackageController extends AbstractController {
     await this.packageManagerService.replacePackageMaintainers(pkg, users);
     return { ok: true };
   }
+
+  private getNpmCommand(ctx: EggContext) {
+    // npm@6: referer: 'xxx [REDACTED]'
+    // npm@>=7: 'npm-command': 'xxx'
+    let npmCommand = ctx.get('npm-command');
+    if (!npmCommand) {
+      npmCommand = ctx.get('referer').split(' ', 1)[0];
+    }
+
+    return npmCommand;
+  }
+
+  private isNpmCommandValid(ctx: EggContext, expectCommand: string) {
+    const npmCommand = this.getNpmCommand(ctx);
+
+    return npmCommand === expectCommand;
+  }
 }

app/port/middleware/AdminAccess.ts

@@ -0,0 +1,12 @@
+import { EggContext, Next } from '@eggjs/tegg';
+import { ForbiddenError } from 'egg-errors';
+import { UserRoleManager } from '../UserRoleManager';
+
+export async function AdminAccess(ctx: EggContext, next: Next) {
+  const userRoleManager = await ctx.getEggObject(UserRoleManager);
+  const isAdmin = await userRoleManager.isAdmin(ctx);
+  if (!isAdmin) {
+    throw new ForbiddenError('Not allow to access');
+  }
+  await next();
+}

app/port/schedule/ChangesStreamWorker.ts

@@ -0,0 +1,29 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { ChangesStreamService } from '../../core/service/ChangesStreamService';
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    interval: 60000,
+  },
+})
+export class ChangesStreamWorker {
+  @Inject()
+  private readonly changesStreamService: ChangesStreamService;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async subscribe() {
+    if (this.config.cnpmcore.syncMode !== 'all' || !this.config.cnpmcore.enableChangesStream) return;
+    const task = await this.changesStreamService.findExecuteTask();
+    if (!task) return;
+    this.logger.info('[ChangesStreamWorker:start] taskId: %s', task.taskId);
+    await this.changesStreamService.executeTask(task);
+  }
+}

app/port/schedule/CheckRecentlyUpdatedPackages.ts

@@ -0,0 +1,88 @@
+import { EggAppConfig, EggHttpClient, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { PackageSyncerService } from '../../core/service/PackageSyncerService';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { getScopeAndName } from '../../common/PackageUtil';
+
+// https://github.com/cnpm/cnpmcore/issues/9
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    // every 5 mins
+    interval: 60000 * 5,
+  },
+})
+export class CheckRecentlyUpdatedPackages {
+  @Inject()
+  private readonly packageSyncerService: PackageSyncerService;
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  @Inject()
+  private readonly httpclient: EggHttpClient;
+
+  async subscribe() {
+    if (this.config.cnpmcore.syncMode === 'none' || !this.config.cnpmcore.enableCheckRecentlyUpdated) return;
+    const pageSize = 36;
+    const pageCount = this.config.env === 'unittest' ? 2 : 5;
+    for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
+      const offset = pageSize * pageIndex;
+      const pageUrl = `https://www.npmjs.com/browse/updated?offset=${offset}`;
+      let html = '';
+      try {
+        const { status, data } = await this.httpclient.request(pageUrl, {
+          followRedirect: true,
+          timeout: 10000,
+        });
+        this.logger.info('[CheckRecentlyUpdatedPackages.subscribe][%s] request %s status: %s, data size: %s',
+          pageIndex, pageUrl, status, data.length);
+        if (status === 200) {
+          html = data.toString();
+        }
+      } catch (err) {
+        this.logger.info('[CheckRecentlyUpdatedPackages.subscribe:error][%s] request %s error: %s',
+          pageIndex, pageUrl, err);
+        this.logger.error(err);
+        continue;
+      }
+
+      const matchs = /window\.__context__ = ([^<]+?)<\/script>/.exec(html);
+      if (!matchs) continue;
+
+      try {
+        const data = JSON.parse(matchs[1]);
+        const packages = data.context.packages || [];
+        if (Array.isArray(packages)) {
+          this.logger.info('[CheckRecentlyUpdatedPackages.subscribe][%s] parse %d packages on %s',
+            pageIndex, packages.length, pageUrl);
+          for (const pkg of packages) {
+            // skip update when package does not exist
+            if (this.config.cnpmcore.syncMode === 'exist') {
+              const [ scope, name ] = getScopeAndName(pkg.name);
+              const pkgId = await this.packageRepository.findPackageId(scope, name);
+              if (!pkgId) {
+                continue;
+              }
+            }
+            const task = await this.packageSyncerService.createTask(pkg.name, {
+              tips: `Sync cause by recently updated packages ${pageUrl}`,
+            });
+            this.logger.info('[CheckRecentlyUpdatedPackages.subscribe:createTask][%s] taskId: %s, targetName: %s',
+              pageIndex, task.taskId, task.targetName);
+          }
+        }
+      } catch (err) {
+        this.logger.info('[CheckRecentlyUpdatedPackages.subscribe:error][%s] parse %s context json error: %s',
+          pageIndex, pageUrl, err);
+        this.logger.error(err);
+      }
+    }
+  }
+}

app/port/schedule/CleanTempDir.ts

@@ -1,19 +1,25 @@
-import { Subscription } from 'egg';
+import { EggAppConfig, EggLogger } from 'egg';
+import { CronParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
 import { rm, access } from 'fs/promises';
 import path from 'path';
-import dayjs from '../common/dayjs';
+import dayjs from '../../common/dayjs';
 
-export default class CleanTempDir extends Subscription {
-  static get schedule() {
-    return {
-      cron: '0 2 * * *', // run every day at 02:00
-      type: 'worker',
-    };
-  }
+@Schedule<CronParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    cron: '0 2 * * *', // run every day at 02:00
+  },
+})
+export class CleanTempDir {
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
 
   async subscribe() {
-    const { ctx, app } = this;
-    const downloadDir = path.join(app.config.dataDir, 'downloads');
+    const downloadDir = path.join(this.config.dataDir, 'downloads');
     const oldDirs = [
       path.join(downloadDir, dayjs().subtract(1, 'day').format('YYYY/MM/DD')),
       path.join(downloadDir, dayjs().subtract(2, 'day').format('YYYY/MM/DD')),
@@ -32,10 +38,10 @@ export default class CleanTempDir extends Subscription {
         // console.log(err);
         exists = false;
       }
-      ctx.logger.info('[CleanTempDir.subscribe] dir "%s" exists: %s', dir, exists);
+      this.logger.info('[CleanTempDir.subscribe] dir "%s" exists: %s', dir, exists);
       if (exists) {
         await rm(dir, { recursive: true, force: true });
-        ctx.logger.info('[CleanTempDir.subscribe] remove dir "%s"', dir);
+        this.logger.info('[CleanTempDir.subscribe] remove dir "%s"', dir);
       }
     }
   }

app/port/schedule/CreateSyncBinaryTask.ts

@@ -0,0 +1,34 @@
+import { EggAppConfig } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { BinarySyncerService } from '../../core/service/BinarySyncerService';
+import binaries from '../../../config/binaries';
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    // every 5 mins
+    interval: 60000 * 5,
+  },
+})
+export class CreateSyncBinaryTask {
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly binarySyncerService: BinarySyncerService;
+
+  async subscribe() {
+    if (!this.config.cnpmcore.enableSyncBinary) return;
+
+    for (const [ binaryName, binary ] of Object.entries(binaries)) {
+      if (this.config.env === 'unittest' && binaryName !== 'node') continue;
+      if (binary.disable) continue;
+
+      // 默认只同步 binaryName 的二进制，即使有不一致的 category，会在同名的 binaryName 任务中同步
+      // 例如 canvas 只同步 binaryName 为 canvas 的二进制，不同步 category 为 node-canvas-prebuilt 的二进制
+      // node-canvas-prebuilt 的二进制会在 node-canvas-prebuilt 的任务中同步
+      await this.binarySyncerService.createTask(binaryName);
+    }
+  }
+}

app/port/schedule/CreateTriggerHookWorker.ts

@@ -0,0 +1,60 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { TaskService } from '../../core/service/TaskService';
+import { TaskType } from '../../common/enum/Task';
+import { CreateHookTask } from '../../core/entity/Task';
+import { CreateHookTriggerService } from '../../core/service/CreateHookTriggerService';
+
+let executingCount = 0;
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.ALL,
+  scheduleData: {
+    interval: 1000,
+  },
+})
+export class CreateTriggerHookWorker {
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  @Inject()
+  private readonly createHookTriggerService: CreateHookTriggerService;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async subscribe() {
+    if (!this.config.cnpmcore.hookEnable) return;
+    if (executingCount >= this.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks) return;
+
+    executingCount++;
+    try {
+      let task = await this.taskService.findExecuteTask(TaskType.CreateHook) as CreateHookTask;
+      while (task) {
+        const startTime = Date.now();
+        this.logger.info('[CreateTriggerHookWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+          executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+          startTime - task.updatedAt.getTime());
+        await this.createHookTriggerService.executeTask(task);
+        const use = Date.now() - startTime;
+        this.logger.info('[CreateTriggerHookWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
+          executingCount, task.taskId, task.targetName, use);
+        if (executingCount >= this.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks) {
+          this.logger.info('[CreateTriggerHookWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+            executingCount, this.config.cnpmcore.createTriggerHookWorkerMaxConcurrentTasks);
+          break;
+        }
+        // try next task
+        task = await this.taskService.findExecuteTask(TaskType.CreateHook) as CreateHookTask;
+      }
+    } catch (err) {
+      this.logger.error('[TriggerHookWorker:subscribe:executeTask:error][%s] %s', executingCount, err);
+    } finally {
+      executingCount--;
+    }
+  }
+}

app/port/schedule/SavePackageVersionDownloadCounter.ts

@@ -0,0 +1,18 @@
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { PackageManagerService } from '../../core/service/PackageManagerService';
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    interval: 60000,
+  },
+})
+export class SavePackageVersionDownloadCounter {
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
+
+  async subscribe() {
+    await this.packageManagerService.savePackageVersionCounters();
+  }
+}

app/port/schedule/SyncBinaryWorker.ts

@@ -0,0 +1,37 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { BinarySyncerService } from '../../core/service/BinarySyncerService';
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.ALL,
+  scheduleData: {
+    interval: 10000,
+  },
+})
+export class SyncBinaryWorker {
+  @Inject()
+  private readonly binarySyncerService: BinarySyncerService;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async subscribe() {
+    if (!this.config.cnpmcore.enableSyncBinary) return;
+
+    const task = await this.binarySyncerService.findExecuteTask();
+    if (!task) return;
+
+    const startTime = Date.now();
+    this.logger.info('[SyncBinaryWorker:executeTask:start] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+      task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+      startTime - task.updatedAt.getTime());
+    await this.binarySyncerService.executeTask(task);
+    const use = Date.now() - startTime;
+    this.logger.info('[SyncBinaryWorker:executeTask:success] taskId: %s, targetName: %s, use %sms',
+      task.taskId, task.targetName, use);
+  }
+}

app/port/schedule/SyncPackageWorker.ts

@@ -0,0 +1,55 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { PackageSyncerService } from '../../core/service/PackageSyncerService';
+
+
+let executingCount = 0;
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.ALL,
+  scheduleData: {
+    interval: 1000,
+  },
+})
+export class SyncPackageWorker {
+  @Inject()
+  private readonly packageSyncerService: PackageSyncerService;
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async subscribe() {
+    if (this.config.cnpmcore.syncMode !== 'all') return;
+    if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) return;
+
+    executingCount++;
+    try {
+      let task = await this.packageSyncerService.findExecuteTask();
+      while (task) {
+        const startTime = Date.now();
+        this.logger.info('[SyncPackageWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+          executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+          startTime - task.updatedAt.getTime());
+        await this.packageSyncerService.executeTask(task);
+        const use = Date.now() - startTime;
+        this.logger.info('[SyncPackageWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
+          executingCount, task.taskId, task.targetName, use);
+        if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) {
+          this.logger.info('[SyncPackageWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+            executingCount, this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks);
+          break;
+        }
+        // try next task
+        task = await this.packageSyncerService.findExecuteTask();
+      }
+    } catch (err) {
+      this.logger.error('[SyncPackageWorker:subscribe:executeTask:error][%s] %s', executingCount, err);
+    } finally {
+      executingCount--;
+    }
+  }
+}

app/port/schedule/TaskTimeoutHandler.ts

@@ -0,0 +1,31 @@
+import { EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { TaskService } from '../../core/service/TaskService';
+import { CacheAdapter } from '../../common/adapter/CacheAdapter';
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    interval: 60000,
+  },
+}, {
+  immediate: process.env.NODE_ENV !== 'test',
+})
+export class TaskTimeoutHandler {
+  @Inject()
+  private readonly taskService: TaskService;
+
+  @Inject()
+  private readonly cacheAdapter: CacheAdapter;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async subscribe() {
+    await this.cacheAdapter.usingLock('TaskTimeoutHandler', 60, async () => {
+      const result = await this.taskService.retryExecuteTimeoutTasks();
+      this.logger.info('[TaskTimeoutHandler:subscribe] retry execute timeout tasks: %j', result);
+    });
+  }
+}

app/port/schedule/TriggerHookWorker.ts

@@ -0,0 +1,58 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { HookTriggerService } from '../../core/service/HookTriggerService';
+import { TaskService } from '../../core/service/TaskService';
+import { TaskType } from '../../common/enum/Task';
+import { TriggerHookTask } from '../../core/entity/Task';
+
+let executingCount = 0;
+@Schedule<IntervalParams>({
+  type: ScheduleType.ALL,
+  scheduleData: {
+    interval: 1000,
+  },
+})
+export class TriggerHookWorker {
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  @Inject()
+  private readonly hookTriggerService: HookTriggerService;
+
+  @Inject()
+  private readonly taskService: TaskService;
+
+  async subscribe() {
+    if (executingCount >= this.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks) return;
+
+    executingCount++;
+    try {
+      let task = await this.taskService.findExecuteTask(TaskType.TriggerHook) as TriggerHookTask;
+      while (task) {
+        const startTime = Date.now();
+        this.logger.info('[TriggerHookWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
+          executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
+          startTime - task.updatedAt.getTime());
+        await this.hookTriggerService.executeTask(task);
+        const use = Date.now() - startTime;
+        this.logger.info('[TriggerHookWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms',
+          executingCount, task.taskId, task.targetName, use);
+        if (executingCount >= this.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks) {
+          this.logger.info('[TriggerHookWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s',
+            executingCount, this.config.cnpmcore.triggerHookWorkerMaxConcurrentTasks);
+          break;
+        }
+        // try next task
+        task = await this.taskService.findExecuteTask(TaskType.TriggerHook) as TriggerHookTask;
+      }
+    } catch (err) {
+      this.logger.error('[TriggerHookWorker:subscribe:executeTask:error][%s] %s', executingCount, err);
+    } finally {
+      executingCount--;
+    }
+  }
+}

app/port/schedule/UpdateTotalData.ts

@@ -0,0 +1,107 @@
+import { EggLogger } from 'egg';
+import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { TaskRepository } from '../../repository/TaskRepository';
+import { ChangeRepository } from '../../repository/ChangeRepository';
+import { CacheService } from '../../core/service/CacheService';
+import { TaskType } from '../../common/enum/Task';
+import dayjs from '../../common/dayjs';
+
+
+@Schedule<IntervalParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    interval: 60000,
+  },
+}, {
+  // immediate = false on unittest env
+  immediate: process.env.NODE_ENV !== 'test',
+})
+export class UpdateTotalData {
+  @Inject()
+  private readonly logger: EggLogger;
+
+  @Inject()
+  private readonly packageRepository: PackageRepository;
+
+  @Inject()
+  private readonly taskRepository: TaskRepository;
+
+  @Inject()
+  private readonly changeRepository: ChangeRepository;
+
+  @Inject()
+  private readonly packageVersionDownloadRepository: PackageVersionDownloadRepository;
+
+  @Inject()
+  private readonly cacheService: CacheService;
+
+  async subscribe() {
+    const changesStreamTask = await this.taskRepository.findTaskByTargetName('GLOBAL_WORKER', TaskType.ChangesStream);
+    const packageTotal = await this.packageRepository.queryTotal();
+
+    const download = {
+      today: 0,
+      yesterday: 0,
+      samedayLastweek: 0,
+      thisweek: 0,
+      thismonth: 0,
+      thisyear: 0,
+      lastweek: 0,
+      lastmonth: 0,
+      lastyear: 0,
+    };
+    const today = dayjs();
+    const lastYearStartDay = today.subtract(1, 'year').startOf('year');
+    const rows = await this.packageVersionDownloadRepository.query('total', lastYearStartDay.toDate(), today.toDate());
+    if (rows.length > 0) {
+      const todayInt = Number(today.format('YYYYMMDD'));
+      const yesterdayInt = Number(today.subtract(1, 'day').format('YYYYMMDD'));
+      const samedayLastweekInt = Number(today.subtract(1, 'week').startOf('week').format('YYYYMMDD'));
+      const thisWeekStartDayInt = Number(today.startOf('week').format('YYYYMMDD'));
+      const thisWeekEndDayInt = Number(today.endOf('week').format('YYYYMMDD'));
+      const thisMonthStartDayInt = Number(today.startOf('month').format('YYYYMMDD'));
+      const thisMonthEndDayInt = Number(today.endOf('month').format('YYYYMMDD'));
+      const thisYearStartDayInt = Number(today.startOf('year').format('YYYYMMDD'));
+      const thisYearEndDayInt = Number(today.endOf('year').format('YYYYMMDD'));
+      const lastWeekStartDayInt = Number(today.subtract(1, 'week').startOf('week').format('YYYYMMDD'));
+      const lastWeekEndDayInt = Number(today.subtract(1, 'week').endOf('week').format('YYYYMMDD'));
+      const lastMonthStartDayInt = Number(today.subtract(1, 'month').startOf('month').format('YYYYMMDD'));
+      const lastMonthEndDayInt = Number(today.subtract(1, 'month').endOf('month').format('YYYYMMDD'));
+      const lastYearStartDayInt = Number(today.subtract(1, 'year').startOf('year').format('YYYYMMDD'));
+      const lastYearEndDayInt = Number(today.subtract(1, 'year').endOf('year').format('YYYYMMDD'));
+
+      for (const row of rows) {
+        for (let i = 1; i <= 31; i++) {
+          const day = String(i).padStart(2, '0');
+          const field = `d${day}`;
+          const counter = row[field];
+          if (!counter) continue;
+          const dayInt = row.yearMonth * 100 + i;
+          if (dayInt === todayInt) download.today += counter;
+          if (dayInt === yesterdayInt) download.yesterday += counter;
+          if (dayInt === samedayLastweekInt) download.samedayLastweek += counter;
+          if (dayInt >= thisWeekStartDayInt && dayInt <= thisWeekEndDayInt) download.thisweek += counter;
+          if (dayInt >= thisMonthStartDayInt && dayInt <= thisMonthEndDayInt) download.thismonth += counter;
+          if (dayInt >= thisYearStartDayInt && dayInt <= thisYearEndDayInt) download.thisyear += counter;
+          if (dayInt >= lastWeekStartDayInt && dayInt <= lastWeekEndDayInt) download.lastweek += counter;
+          if (dayInt >= lastMonthStartDayInt && dayInt <= lastMonthEndDayInt) download.lastmonth += counter;
+          if (dayInt >= lastYearStartDayInt && dayInt <= lastYearEndDayInt) download.lastyear += counter;
+        }
+      }
+    }
+
+    const lastChange = await this.changeRepository.getLastChange();
+    const totalData = {
+      ...packageTotal,
+      download,
+      changesStream: changesStreamTask && changesStreamTask.data || {},
+      lastChangeId: lastChange && lastChange.id || 0,
+      cacheTime: new Date().toISOString(),
+    };
+    await this.cacheService.saveTotalData(totalData);
+    this.logger.info('[UpdateTotalData.subscribe] total data: %j', totalData);
+  }
+}

app/port/typebox.ts

@@ -1,10 +1,32 @@
 import { Type, Static } from '@sinclair/typebox';
+import { RegistryType } from '../common/enum/Registry';
 import semver from 'semver';
+import { HookType } from '../common/enum/Hook';
 
 export const Name = Type.String({
   transform: [ 'trim' ],
 });
 
+export const Url = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 2048,
+});
+
+export const Secret = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 200,
+});
+
+export const HookName = Type.String({
+  transform: [ 'trim' ],
+  minLength: 1,
+  maxLength: 428,
+});
+
+export const HookTypeType = Type.Enum(HookType);
+
 export const Tag = Type.String({
   format: 'semver-tag',
   transform: [ 'trim' ],
@@ -42,6 +64,10 @@ export const SyncPackageTaskRule = Type.Object({
   syncDownloadData: Type.Boolean(),
   // force sync immediately, only allow by admin
   force: Type.Boolean(),
+  // sync history version
+  forceSyncHistory: Type.Boolean(),
+  // source registry
+  registryName: Type.Optional(Type.String()),
 });
 export type SyncPackageTaskType = Static<typeof SyncPackageTaskRule>;
 
@@ -54,6 +80,18 @@ export const BlockPackageRule = Type.Object({
 });
 export type BlockPackageType = Static<typeof BlockPackageRule>;
 
+export const UpdateHookRequestRule = Type.Object({
+  endpoint: Url,
+  secret: Secret,
+});
+
+export const CreateHookRequestRule = Type.Object({
+  endpoint: Url,
+  secret: Secret,
+  name: HookName,
+  type: HookTypeType,
+});
+
 // https://github.com/xiekw2010/egg-typebox-validate#%E5%A6%82%E4%BD%95%E5%86%99%E8%87%AA%E5%AE%9A%E4%B9%89%E6%A0%A1%E9%AA%8C%E8%A7%84%E5%88%99
 // add custom validate to ajv
 export function patchAjv(ajv: any) {
@@ -70,3 +108,103 @@ export function patchAjv(ajv: any) {
     },
   });
 }
+
+export const QueryPageOptions = Type.Object({
+  pageSize: Type.Optional(Type.Number({
+    transform: [ 'trim' ],
+    minimum: 1,
+    maximum: 100,
+  })),
+  pageIndex: Type.Optional(Type.Number({
+    transform: [ 'trim' ],
+    minimum: 0,
+  })),
+});
+
+export const RegistryCreateSyncOptions = Type.Object({
+  since: Type.Optional(Type.String()),
+});
+
+export const RegistryCreateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  host: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  changeStream: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  userPrefix: Type.Optional(Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  })),
+  type: Type.Enum(RegistryType),
+});
+
+export const RegistryUpdateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  host: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  changeStream: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 4096,
+  }),
+  userPrefix: Type.Optional(Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  })),
+  type: Type.Enum(RegistryType),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});
+
+export const ScopeCreateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});
+
+export const ScopeUpdateOptions = Type.Object({
+  name: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  registryId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+  scopeId: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});

app/repository/BinaryRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Binary as BinaryModel } from './model/Binary';
+import type { Binary as BinaryModel } from './model/Binary';
 import { Binary as BinaryEntity } from '../core/entity/Binary';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,25 +8,28 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class BinaryRepository extends AbstractRepository {
+  @Inject()
+  private readonly Binary: typeof BinaryModel;
+
   async saveBinary(binary: BinaryEntity): Promise<void> {
     if (binary.id) {
-      const model = await BinaryModel.findOne({ id: binary.id });
+      const model = await this.Binary.findOne({ id: binary.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(binary, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(binary, BinaryModel);
+      const model = await ModelConvertor.convertEntityToModel(binary, this.Binary);
       this.logger.info('[BinaryRepository:saveBinary:new] id: %s, binaryId: %s', model.id, model.binaryId);
     }
   }
 
   async findBinary(category: string, parent: string, name: string) {
-    const model = await BinaryModel.findOne({ category, parent, name });
+    const model = await this.Binary.findOne({ category, parent, name });
     if (model) return ModelConvertor.convertModelToEntity(model, BinaryEntity);
     return null;
   }
 
   async listBinaries(category: string, parent: string): Promise<BinaryEntity[]> {
-    const models = await BinaryModel.find({ category, parent });
+    const models = await this.Binary.find({ category, parent });
     return models.map(model => ModelConvertor.convertModelToEntity(model, BinaryEntity));
   }
 }

app/repository/ChangeRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Change as ChangeModel } from './model/Change';
+import type { Change as ChangeModel } from './model/Change';
 import { Change as ChangeEntity } from '../core/entity/Change';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,16 +8,19 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class ChangeRepository extends AbstractRepository {
+  @Inject()
+  private readonly Change: typeof ChangeModel;
+
   async addChange(change: ChangeEntity) {
-    await ModelConvertor.convertEntityToModel(change, ChangeModel);
+    await ModelConvertor.convertEntityToModel(change, this.Change);
   }
 
   async query(since: number, limit: number): Promise<Array<ChangeEntity>> {
-    const models = await ChangeModel.find({ id: { $gte: since } }).order('id', 'asc').limit(limit);
+    const models = await this.Change.find({ id: { $gte: since } }).order('id', 'asc').limit(limit);
     return models.toObject() as ChangeEntity[];
   }
 
   async getLastChange() {
-    return await ChangeModel.findOne().order('id', 'desc').limit(1);
+    return await this.Change.findOne().order('id', 'desc').limit(1);
   }
 }

app/repository/HookRepository.ts

@@ -0,0 +1,66 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { Hook } from '../core/entity/Hook';
+import type { Hook as HookModel } from './model/Hook';
+import { ModelConvertor } from './util/ModelConvertor';
+import { HookType } from '../common/enum/Hook';
+
+export interface UpdateHookCommand {
+  hookId: string;
+  endpoint: string;
+  secret: string;
+}
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HookRepository {
+  @Inject()
+  private readonly Hook: typeof HookModel;
+
+  async saveHook(hook: Hook) {
+    if (hook.id) {
+      const model = await this.Hook.findOne({ id: hook.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(hook, model);
+    } else {
+      await ModelConvertor.convertEntityToModel(hook, this.Hook);
+    }
+  }
+
+  async findHookById(hookId: string): Promise<Hook | undefined> {
+    const model = await this.Hook.findOne({ hookId });
+    if (!model) return;
+    return ModelConvertor.convertModelToEntity(model, Hook);
+  }
+
+  async removeHook(hookId: string): Promise<void> {
+    await this.Hook.remove({ hookId });
+  }
+
+  /**
+   * only endpoint and secret can be updated
+   */
+  async updateHook(cmd: UpdateHookCommand) {
+    this.Hook.update({
+      hookId: cmd.hookId,
+    }, {
+      endpoint: cmd.endpoint,
+      secret: cmd.secret,
+    });
+  }
+
+  async listHooksByOwnerId(ownerId: string) {
+    const hookRows = await this.Hook.find({ ownerId });
+    return hookRows.map(row => ModelConvertor.convertModelToEntity(row, Hook));
+  }
+
+  async listHooksByTypeAndName(type: HookType, name: string, since?: bigint): Promise<Array<Hook>> {
+    let hookRows: Array<HookModel>;
+    if (typeof since !== 'undefined') {
+      hookRows = await this.Hook.find({ type, name, id: { $gt: since } }).limit(100);
+    } else {
+      hookRows = await this.Hook.find({ type, name }).limit(100);
+    }
+    return hookRows.map(row => ModelConvertor.convertModelToEntity(row, Hook));
+  }
+}

app/repository/PackageRepository.ts

@@ -1,17 +1,17 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
-import { Package as PackageModel } from './model/Package';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import type { Package as PackageModel } from './model/Package';
 import { Package as PackageEntity } from '../core/entity/Package';
 import { ModelConvertor } from './util/ModelConvertor';
 import { PackageVersion as PackageVersionEntity } from '../core/entity/PackageVersion';
-import { PackageVersion as PackageVersionModel } from './model/PackageVersion';
+import type { PackageVersion as PackageVersionModel } from './model/PackageVersion';
 import { PackageVersionManifest as PackageVersionManifestEntity } from '../core/entity/PackageVersionManifest';
-import { PackageVersionManifest as PackageVersionManifestModel } from './model/PackageVersionManifest';
-import { Dist as DistModel } from './model/Dist';
+import type { PackageVersionManifest as PackageVersionManifestModel } from './model/PackageVersionManifest';
+import type { Dist as DistModel } from './model/Dist';
 import { Dist as DistEntity } from '../core/entity/Dist';
 import { PackageTag as PackageTagEntity } from '../core/entity/PackageTag';
-import { PackageTag as PackageTagModel } from './model/PackageTag';
-import { Maintainer as MaintainerModel } from './model/Maintainer';
-import { User as UserModel } from './model/User';
+import type { PackageTag as PackageTagModel } from './model/PackageTag';
+import type { Maintainer as MaintainerModel } from './model/Maintainer';
+import type { User as UserModel } from './model/User';
 import { User as UserEntity } from '../core/entity/User';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -19,11 +19,32 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageRepository extends AbstractRepository {
+  @Inject()
+  private readonly Package: typeof PackageModel;
+
+  @Inject()
+  private readonly Dist: typeof DistModel;
+
+  @Inject()
+  private readonly PackageVersion: typeof PackageVersionModel;
+
+  @Inject()
+  private readonly PackageVersionManifest: typeof PackageVersionManifestModel;
+
+  @Inject()
+  private readonly PackageTag: typeof PackageTagModel;
+
+  @Inject()
+  private readonly Maintainer: typeof MaintainerModel;
+
+  @Inject()
+  private readonly User: typeof UserModel;
+
   async findPackage(scope: string, name: string): Promise<PackageEntity | null> {
-    const model = await PackageModel.findOne({ scope, name });
+    const model = await this.Package.findOne({ scope, name });
     if (!model) return null;
-    const manifestsDistModel = model.manifestsDistId ? await DistModel.findOne({ distId: model.manifestsDistId }) : null;
-    const abbreviatedsDistModel = model.abbreviatedsDistId ? await DistModel.findOne({ distId: model.abbreviatedsDistId }) : null;
+    const manifestsDistModel = model.manifestsDistId ? await this.Dist.findOne({ distId: model.manifestsDistId }) : null;
+    const abbreviatedsDistModel = model.abbreviatedsDistId ? await this.Dist.findOne({ distId: model.abbreviatedsDistId }) : null;
     const data = {
       manifestsDist: manifestsDistModel && ModelConvertor.convertModelToEntity(manifestsDistModel, DistEntity),
       abbreviatedsDist: abbreviatedsDistModel && ModelConvertor.convertModelToEntity(abbreviatedsDistModel, DistEntity),
@@ -33,18 +54,18 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageId(scope: string, name: string) {
-    const model = await PackageModel.findOne({ scope, name }).select('packageId');
+    const model = await this.Package.findOne({ scope, name }).select('packageId');
     if (!model) return null;
     return model.packageId;
   }
 
   async savePackage(pkgEntity: PackageEntity): Promise<void> {
     if (pkgEntity.id) {
-      const model = await PackageModel.findOne({ id: pkgEntity.id });
+      const model = await this.Package.findOne({ id: pkgEntity.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(pkgEntity, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(pkgEntity, PackageModel);
+      const model = await ModelConvertor.convertEntityToModel(pkgEntity, this.Package);
       this.logger.info('[PackageRepository:savePackage:new] id: %s, packageId: %s', model.id, model.packageId);
     }
   }
@@ -53,11 +74,11 @@ export class PackageRepository extends AbstractRepository {
     const dist = isFullManifests ? pkgEntity.manifestsDist : pkgEntity.abbreviatedsDist;
     if (!dist) return;
     if (dist.id) {
-      const model = await DistModel.findOne({ id: dist.id });
+      const model = await this.Dist.findOne({ id: dist.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(dist, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(dist, DistModel);
+      const model = await ModelConvertor.convertEntityToModel(dist, this.Dist);
       this.logger.info('[PackageRepository:savePackageDist:new] id: %s, distId: %s, packageId: %s',
         model.id, model.distId, pkgEntity.packageId);
     }
@@ -67,7 +88,7 @@ export class PackageRepository extends AbstractRepository {
   async removePackageDist(pkgEntity: PackageEntity, isFullManifests: boolean): Promise<void> {
     const dist = isFullManifests ? pkgEntity.manifestsDist : pkgEntity.abbreviatedsDist;
     if (!dist) return;
-    const model = await DistModel.findOne({ id: dist.id });
+    const model = await this.Dist.findOne({ id: dist.id });
     if (!model) return;
     await model.remove();
     this.logger.info('[PackageRepository:removePackageDist:remove] id: %s, distId: %s, packageId: %s',
@@ -79,9 +100,9 @@ export class PackageRepository extends AbstractRepository {
   // Package Maintainers
   // return true meaning create new record
   async savePackageMaintainer(packageId: string, userId: string): Promise<undefined | true> {
-    let model = await MaintainerModel.findOne({ packageId, userId });
+    let model = await this.Maintainer.findOne({ packageId, userId });
     if (!model) {
-      model = await MaintainerModel.create({ packageId, userId });
+      model = await this.Maintainer.create({ packageId, userId });
       this.logger.info('[PackageRepository:addPackageMaintainer:new] id: %s, packageId: %s, userId: %s',
         model.id, model.packageId, model.userId);
       return true;
@@ -89,22 +110,22 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageMaintainers(packageId: string): Promise<UserEntity[]> {
-    const models = await MaintainerModel.find({ packageId });
-    const userModels = await UserModel.find({ userId: models.map(m => m.userId) });
+    const models = await this.Maintainer.find({ packageId });
+    const userModels = await this.User.find({ userId: models.map(m => m.userId) });
     return userModels.map(user => ModelConvertor.convertModelToEntity(user, UserEntity));
   }
 
   async replacePackageMaintainers(packageId: string, userIds: string[]): Promise<void> {
-    await MaintainerModel.transaction(async () => {
+    await this.Maintainer.transaction(async ({ connection }) => {
       // delete exists
-      // const removeCount = await MaintainerModel.remove({ packageId }, true, { transaction });
-      const removeCount = await MaintainerModel.remove({ packageId });
+      // const removeCount = await this.Maintainer.remove({ packageId }, true, { transaction });
+      const removeCount = await this.Maintainer.remove({ packageId }, true, { connection });
       this.logger.info('[PackageRepository:replacePackageMaintainers:remove] %d rows, packageId: %s',
         removeCount, packageId);
       // add news
       for (const userId of userIds) {
-        // const model = await MaintainerModel.create({ packageId, userId }, transaction);
-        const model = await MaintainerModel.create({ packageId, userId });
+        // const model = await this.Maintainer.create({ packageId, userId }, transaction);
+        const model = await this.Maintainer.create({ packageId, userId }, { connection });
         this.logger.info('[PackageRepository:replacePackageMaintainers:new] id: %s, packageId: %s, userId: %s',
           model.id, model.packageId, model.userId);
       }
@@ -112,7 +133,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async removePackageMaintainer(packageId: string, userId: string) {
-    const model = await MaintainerModel.findOne({ packageId, userId });
+    const model = await this.Maintainer.findOne({ packageId, userId });
     if (model) {
       await model.remove();
       this.logger.info('[PackageRepository:removePackageMaintainer:remove] id: %s, packageId: %s, userId: %s',
@@ -124,36 +145,36 @@ export class PackageRepository extends AbstractRepository {
 
   // TODO: support paging
   async listPackagesByUserId(userId: string): Promise<PackageEntity[]> {
-    const models = await MaintainerModel.find({ userId });
-    const packageModels = await PackageModel.find({ packageId: models.map(m => m.packageId) });
+    const models = await this.Maintainer.find({ userId });
+    const packageModels = await this.Package.find({ packageId: models.map(m => m.packageId) });
     return packageModels.map(pkg => ModelConvertor.convertModelToEntity(pkg, PackageEntity));
   }
 
   async createPackageVersion(pkgVersionEntity: PackageVersionEntity) {
-    await PackageVersionModel.transaction(async function(transaction) {
+    await this.PackageVersion.transaction(async transaction => {
       await Promise.all([
         // FIXME: transaction is not the options
-        ModelConvertor.convertEntityToModel(pkgVersionEntity, PackageVersionModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.manifestDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.tarDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.readmeDist, DistModel, transaction),
-        ModelConvertor.convertEntityToModel(pkgVersionEntity.abbreviatedDist, DistModel, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity, this.PackageVersion, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.manifestDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.tarDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.readmeDist, this.Dist, transaction),
+        ModelConvertor.convertEntityToModel(pkgVersionEntity.abbreviatedDist, this.Dist, transaction),
       ]);
     });
   }
 
   async savePackageVersion(pkgVersionEntity: PackageVersionEntity) {
     // only abbreviatedDist and manifestDist allow to change, like `deprecated` message
-    let model = await DistModel.findOne({ id: pkgVersionEntity.manifestDist.id });
+    let model = await this.Dist.findOne({ id: pkgVersionEntity.manifestDist.id });
     if (model) {
       await ModelConvertor.saveEntityToModel(pkgVersionEntity.manifestDist, model);
     }
-    model = await DistModel.findOne({ id: pkgVersionEntity.abbreviatedDist.id });
+    model = await this.Dist.findOne({ id: pkgVersionEntity.abbreviatedDist.id });
     if (model) {
       await ModelConvertor.saveEntityToModel(pkgVersionEntity.abbreviatedDist, model);
     }
     if (pkgVersionEntity.id) {
-      const model = await PackageVersionModel.findOne({ id: pkgVersionEntity.id });
+      const model = await this.PackageVersion.findOne({ id: pkgVersionEntity.id });
       if (model) {
         await ModelConvertor.saveEntityToModel(pkgVersionEntity, model);
       }
@@ -161,14 +182,14 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageVersion(packageId: string, version: string): Promise<PackageVersionEntity | null> {
-    const pkgVersionModel = await PackageVersionModel.findOne({ packageId, version });
+    const pkgVersionModel = await this.PackageVersion.findOne({ packageId, version });
     if (!pkgVersionModel) return null;
     return await this.fillPackageVersionEntitiyData(pkgVersionModel);
   }
 
   async listPackageVersions(packageId: string): Promise<PackageVersionEntity[]> {
     // FIXME: read all versions will hit the memory limit
-    const models = await PackageVersionModel.find({ packageId }).order('id desc');
+    const models = await this.PackageVersion.find({ packageId }).order('id desc');
     const entities: PackageVersionEntity[] = [];
     for (const model of models) {
       entities.push(await this.fillPackageVersionEntitiyData(model));
@@ -177,19 +198,19 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageVersionNames(packageId: string): Promise<string[]> {
-    const rows = await PackageVersionModel.find({ packageId }).select('version').order('id desc');
+    const rows = await this.PackageVersion.find({ packageId }).select('version').order('id desc');
     return rows.map(row => row.version);
   }
 
   // only for unittest now
   async removePackageVersions(packageId: string): Promise<void> {
-    const removeCount = await PackageVersionModel.remove({ packageId });
+    const removeCount = await this.PackageVersion.remove({ packageId });
     this.logger.info('[PackageRepository:removePackageVersions:remove] %d rows, packageId: %s',
       removeCount, packageId);
   }
 
   async removePackageVersion(pkgVersion: PackageVersionEntity): Promise<void> {
-    const distRemoveCount = await DistModel.remove({
+    const distRemoveCount = await this.Dist.remove({
       distId: [
         pkgVersion.abbreviatedDist.distId,
         pkgVersion.manifestDist.distId,
@@ -197,32 +218,32 @@ export class PackageRepository extends AbstractRepository {
         pkgVersion.tarDist.distId,
       ],
     });
-    const removeCount = await PackageVersionModel.remove({ packageVersionId: pkgVersion.packageVersionId });
+    const removeCount = await this.PackageVersion.remove({ packageVersionId: pkgVersion.packageVersionId });
     this.logger.info('[PackageRepository:removePackageVersion:remove] %d dist rows, %d rows, packageVersionId: %s',
       distRemoveCount, removeCount, pkgVersion.packageVersionId);
   }
 
   async savePackageVersionManifest(manifestEntity: PackageVersionManifestEntity): Promise<void> {
-    let model = await PackageVersionManifestModel.findOne({ packageVersionId: manifestEntity.packageVersionId });
+    let model = await this.PackageVersionManifest.findOne({ packageVersionId: manifestEntity.packageVersionId });
     if (model) {
       model.manifest = manifestEntity.manifest;
       await model.save();
     } else {
-      model = await ModelConvertor.convertEntityToModel(manifestEntity, PackageVersionManifestModel);
+      model = await ModelConvertor.convertEntityToModel(manifestEntity, this.PackageVersionManifest);
       this.logger.info('[PackageRepository:savePackageVersionManifest:new] id: %s, packageVersionId: %s',
         model.id, model.packageVersionId);
     }
   }
 
   async findPackageVersionManifest(packageVersionId: string) {
-    const model = await PackageVersionManifestModel.findOne({ packageVersionId });
+    const model = await this.PackageVersionManifest.findOne({ packageVersionId });
     if (!model) return null;
-    return ModelConvertor.convertModelToEntity(model, PackageVersionManifestModel);
+    return ModelConvertor.convertModelToEntity(model, this.PackageVersionManifest);
   }
 
   public async queryTotal() {
-    const lastPkg = await PackageModel.findOne().order('id', 'desc');
-    const lastVersion = await PackageVersionModel.findOne().order('id', 'desc');
+    const lastPkg = await this.Package.findOne().order('id', 'desc');
+    const lastVersion = await this.PackageVersion.findOne().order('id', 'desc');
     let packageCount = 0;
     let packageVersionCount = 0;
     let lastPackage = '';
@@ -235,7 +256,7 @@ export class PackageRepository extends AbstractRepository {
     }
 
     if (lastVersion) {
-      const pkg = await PackageModel.findOne({ packageId: lastVersion.packageId });
+      const pkg = await this.Package.findOne({ packageId: lastVersion.packageId });
       if (pkg) {
         const fullname = pkg.scope ? `${pkg.scope}/${pkg.name}` : pkg.name;
         lastPackageVersion = `${fullname}@${lastVersion.version}`;
@@ -257,10 +278,10 @@ export class PackageRepository extends AbstractRepository {
       manifestDistModel,
       abbreviatedDistModel,
     ] = await Promise.all([
-      DistModel.findOne({ distId: model.tarDistId }),
-      DistModel.findOne({ distId: model.readmeDistId }),
-      DistModel.findOne({ distId: model.manifestDistId }),
-      DistModel.findOne({ distId: model.abbreviatedDistId }),
+      this.Dist.findOne({ distId: model.tarDistId }),
+      this.Dist.findOne({ distId: model.readmeDistId }),
+      this.Dist.findOne({ distId: model.manifestDistId }),
+      this.Dist.findOne({ distId: model.abbreviatedDistId }),
     ]);
     const data = {
       tarDist: tarDistModel && ModelConvertor.convertModelToEntity(tarDistModel, DistEntity),
@@ -272,7 +293,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async findPackageTag(packageId: string, tag: string): Promise<PackageTagEntity | null> {
-    const model = await PackageTagModel.findOne({ packageId, tag });
+    const model = await this.PackageTag.findOne({ packageId, tag });
     if (!model) return null;
     const entity = ModelConvertor.convertModelToEntity(model, PackageTagEntity);
     return entity;
@@ -280,18 +301,18 @@ export class PackageRepository extends AbstractRepository {
 
   async savePackageTag(packageTagEntity: PackageTagEntity) {
     if (packageTagEntity.id) {
-      const model = await PackageTagModel.findOne({ id: packageTagEntity.id });
+      const model = await this.PackageTag.findOne({ id: packageTagEntity.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(packageTagEntity, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(packageTagEntity, PackageTagModel);
+      const model = await ModelConvertor.convertEntityToModel(packageTagEntity, this.PackageTag);
       this.logger.info('[PackageRepository:savePackageTag:new] id: %s, packageTagId: %s, tags: %s => %s',
         model.id, model.packageTagId, model.tag, model.version);
     }
   }
 
   async removePackageTag(packageTagEntity: PackageTagEntity) {
-    const model = await PackageTagModel.findOne({ id: packageTagEntity.id });
+    const model = await this.PackageTag.findOne({ id: packageTagEntity.id });
     if (!model) return;
     await model.remove();
     this.logger.info('[PackageRepository:removePackageTag:remove] id: %s, packageTagId: %s, packageId: %s',
@@ -299,7 +320,7 @@ export class PackageRepository extends AbstractRepository {
   }
 
   async listPackageTags(packageId: string): Promise<PackageTagEntity[]> {
-    const models = await PackageTagModel.find({ packageId });
+    const models = await this.PackageTag.find({ packageId });
     const entities: PackageTagEntity[] = [];
     for (const model of models) {
       entities.push(ModelConvertor.convertModelToEntity(model, PackageTagEntity));

app/repository/PackageVersionBlockRepository.ts

@@ -1,6 +1,6 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { PackageVersionBlock as PackageVersionBlockModel } from './model/PackageVersionBlock';
+import type { PackageVersionBlock as PackageVersionBlockModel } from './model/PackageVersionBlock';
 import { PackageVersionBlock as PackageVersionBlockEntity } from '../core/entity/PackageVersionBlock';
 import { AbstractRepository } from './AbstractRepository';
 
@@ -8,13 +8,16 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionBlockRepository extends AbstractRepository {
+  @Inject()
+  private readonly PackageVersionBlock: typeof PackageVersionBlockModel;
+
   async savePackageVersionBlock(block: PackageVersionBlockEntity) {
     if (block.id) {
-      const model = await PackageVersionBlockModel.findOne({ id: block.id });
+      const model = await this.PackageVersionBlock.findOne({ id: block.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(block, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(block, PackageVersionBlockModel);
+      const model = await ModelConvertor.convertEntityToModel(block, this.PackageVersionBlock);
       this.logger.info('[PackageVersionBlockRepository:savePackageVersionBlock:new] id: %s, packageVersionBlockId: %s',
         model.id, model.packageVersionBlockId);
     }
@@ -25,17 +28,17 @@ export class PackageVersionBlockRepository extends AbstractRepository {
   }
 
   async findPackageVersionBlock(packageId: string, version: string) {
-    const model = await PackageVersionBlockModel.findOne({ packageId, version });
+    const model = await this.PackageVersionBlock.findOne({ packageId, version });
     if (model) return ModelConvertor.convertModelToEntity(model, PackageVersionBlockEntity);
     return null;
   }
 
   async listPackageVersionBlocks(packageId: string) {
-    return await PackageVersionBlockModel.find({ packageId });
+    return await this.PackageVersionBlock.find({ packageId });
   }
 
   async removePackageVersionBlock(packageVersionBlockId: string) {
-    const removeCount = await PackageVersionBlockModel.remove({ packageVersionBlockId });
+    const removeCount = await this.PackageVersionBlock.remove({ packageVersionBlockId });
     this.logger.info('[PackageVersionBlockRepository:removePackageVersionBlock:remove] %d rows, packageVersionBlockId: %s',
       removeCount, packageVersionBlockId);
   }

app/repository/PackageVersionDownloadRepository.ts

@@ -1,17 +1,20 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { AbstractRepository } from './AbstractRepository';
-import { PackageVersionDownload as PackageVersionDownloadModel } from './model/PackageVersionDownload';
+import type { PackageVersionDownload as PackageVersionDownloadModel } from './model/PackageVersionDownload';
 
 @ContextProto({
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionDownloadRepository extends AbstractRepository {
+  @Inject()
+  private readonly PackageVersionDownload: typeof PackageVersionDownloadModel;
+
   async plus(packageId: string, version: string, counter: number): Promise<void> {
     const now = new Date();
     const yearMonth = now.getFullYear() * 100 + now.getMonth() + 1;
     const date = new Date().getDate();
     const field = date < 10 ? `d0${date}` : `d${date}`;
-    let model = await PackageVersionDownloadModel.findOne({
+    let model = await this.PackageVersionDownload.findOne({
       packageId,
       version,
       yearMonth,
@@ -23,11 +26,11 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
         version,
         yearMonth,
       };
-      model = await PackageVersionDownloadModel.create(attributes);
+      model = await this.PackageVersionDownload.create(attributes);
       this.logger.info('[PackageVersionDownloadRepository:plus:new] id: %s, packageId: %s, version: %s, yearMonth: %s',
         model.id, model.packageId, model.version, model.yearMonth);
     }
-    await PackageVersionDownloadModel
+    await this.PackageVersionDownload
       .where({ id: model.id })
       .increment(field, counter);
     this.logger.info('[PackageVersionDownloadRepository:plus:increment] id: %s, packageId: %s, version: %s, field: %s%s, plus: %d',
@@ -37,7 +40,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
   async query(packageId: string, start: Date, end: Date) {
     const startYearMonth = start.getFullYear() * 100 + start.getMonth() + 1;
     const endYearMonth = end.getFullYear() * 100 + end.getMonth() + 1;
-    const models = await PackageVersionDownloadModel.find({
+    const models = await this.PackageVersionDownload.find({
       packageId,
       yearMonth: { $gte: startYearMonth, $lte: endYearMonth },
     });
@@ -46,7 +49,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
 
   async saveSyncDataByMonth(packageId: string, yearMonth: number, counters: [string, number][]): Promise<void> {
     const version = '*';
-    let model = await PackageVersionDownloadModel.findOne({
+    let model = await this.PackageVersionDownload.findOne({
       packageId,
       version,
       yearMonth,
@@ -58,7 +61,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
         version,
         yearMonth,
       };
-      model = await PackageVersionDownloadModel.create(attributes);
+      model = await this.PackageVersionDownload.create(attributes);
     }
     for (const [ date, counter ] of counters) {
       const field = `d${date}`;

app/repository/RegistryRepository.ts

@@ -0,0 +1,59 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { ModelConvertor } from './util/ModelConvertor';
+import { Registry, Registry as RegistryEntity } from '../core/entity/Registry';
+import { AbstractRepository } from './AbstractRepository';
+import type { Registry as RegistryModel } from './model/Registry';
+import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class RegistryRepository extends AbstractRepository {
+  @Inject()
+  private readonly Registry: typeof RegistryModel;
+
+  async listRegistries(page: PageOptions): Promise<PageResult<Registry>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Registry.find().count();
+    const models = await this.Registry.find().offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, RegistryEntity)),
+    };
+  }
+
+  async findRegistry(name?: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ name });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
+  async findRegistryByRegistryId(registryId: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ registryId });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
+  async saveRegistry(registry: Registry) {
+    if (registry.id) {
+      const model = await this.Registry.findOne({ id: registry.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(registry, model);
+      return model;
+    }
+    const model = await ModelConvertor.convertEntityToModel(registry, this.Registry);
+    this.logger.info('[RegistryRepository:saveRegistry:new] id: %s, registryId: %s',
+      model.id, model.registryId);
+    return model;
+
+  }
+
+  async removeRegistry(registryId: string): Promise<void> {
+    await this.Registry.remove({ registryId });
+  }
+
+}

app/repository/ScopeRepository.ts

@@ -0,0 +1,67 @@
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
+import { ModelConvertor } from './util/ModelConvertor';
+import { AbstractRepository } from './AbstractRepository';
+import { Scope as ScopeModel } from './model/Scope';
+import { Scope } from '../core/entity/Scope';
+import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil';
+
+@ContextProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class ScopeRepository extends AbstractRepository {
+  @Inject()
+  private readonly Scope: typeof ScopeModel;
+
+  async countByRegistryId(registryId: string): Promise<number> {
+    return await this.Scope.find({ registryId }).count();
+  }
+  async findByName(name: string): Promise<Scope | null> {
+    const model = await this.Scope.findOne({ name });
+    if (!model) {
+      return null;
+    }
+    return ModelConvertor.convertModelToEntity(model, Scope);
+  }
+  async listScopesByRegistryId(registryId: string, page: PageOptions): Promise<PageResult<Scope>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Scope.find({ registryId }).count();
+    const models = await this.Scope.find({ registryId }).offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, Scope)),
+    };
+  }
+
+  async listScopes(page: PageOptions): Promise<PageResult<Scope>> {
+    const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page);
+    const count = await this.Scope.find().count();
+    const models = await this.Scope.find().offset(offset).limit(limit);
+    return {
+      count,
+      data: models.map(model => ModelConvertor.convertModelToEntity(model, Scope)),
+    };
+  }
+
+  async saveScope(scope: Scope) {
+    if (scope.id) {
+      const model = await this.Scope.findOne({ id: scope.id });
+      if (!model) return;
+      await ModelConvertor.saveEntityToModel(scope, model);
+      return model;
+    }
+    const model = await ModelConvertor.convertEntityToModel(scope, this.Scope);
+    this.logger.info('[ScopeRepository:saveScope:new] id: %s, scopeId: %s',
+      model.id, model.scopeId);
+    await model.save();
+    return model;
+  }
+
+  async removeScope(scopeId: string): Promise<void> {
+    await this.Scope.remove({ scopeId });
+  }
+
+  async removeScopeByRegistryId(registryId: string): Promise<void> {
+    await this.Scope.remove({ registryId });
+  }
+
+}

app/repository/TaskRepository.ts

@@ -1,8 +1,9 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import assert from 'assert';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { Task as TaskModel } from './model/Task';
-import { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
-import { Task as TaskEntity } from '../core/entity/Task';
+import type { Task as TaskModel } from './model/Task';
+import type { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
+import { Task as TaskEntity, TaskUpdateCondition } from '../core/entity/Task';
 import { AbstractRepository } from './AbstractRepository';
 import { TaskType, TaskState } from '../../app/common/enum/Task';
 
@@ -10,47 +11,94 @@ import { TaskType, TaskState } from '../../app/common/enum/Task';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class TaskRepository extends AbstractRepository {
+  @Inject()
+  private readonly Task: typeof TaskModel;
+
+  @Inject()
+  private readonly HistoryTask: typeof HistoryTaskModel;
+
   async saveTask(task: TaskEntity): Promise<void> {
     if (task.id) {
-      const model = await TaskModel.findOne({ id: task.id });
+      const model = await this.Task.findOne({ id: task.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(task, model);
     } else {
-      await ModelConvertor.convertEntityToModel(task, TaskModel);
+      try {
+        await ModelConvertor.convertEntityToModel(task, this.Task);
+      } catch (e) {
+        e.message = '[TaskRepository] insert Task failed: ' + e.message;
+        if (e.code === 'ER_DUP_ENTRY') {
+          this.logger.warn(e);
+          const taskModel = await this.Task.findOne({ bizId: task.bizId });
+          // 覆盖 bizId 相同的 id 和 taskId
+          if (taskModel) {
+            task.id = taskModel.id;
+            task.taskId = taskModel.taskId;
+            return;
+          }
+          // taskModel 可能不存在，遇到数据错误
+          // 重新将错误抛出。
+          throw e;
+        }
+        throw e;
+      }
     }
   }
 
+  async idempotentSaveTask(task: TaskEntity, condition: TaskUpdateCondition): Promise<boolean> {
+    assert(task.id, 'task have no save');
+    const changes = ModelConvertor.convertEntityToChanges(task, this.Task);
+    const updateRows = await this.Task.update({
+      taskId: condition.taskId,
+      attempts: condition.attempts,
+    }, changes);
+    return updateRows === 1;
+  }
+
   async saveTaskToHistory(task: TaskEntity): Promise<void> {
-    const model = await TaskModel.findOne({ id: task.id });
+    const model = await this.Task.findOne({ id: task.id });
     if (!model) return;
-    const history = await HistoryTaskModel.findOne({ taskId: task.taskId });
+    const history = await this.HistoryTask.findOne({ taskId: task.taskId });
     if (history) {
       await ModelConvertor.saveEntityToModel(task, history);
     } else {
-      await ModelConvertor.convertEntityToModel(task, HistoryTaskModel);
+      await ModelConvertor.convertEntityToModel(task, this.HistoryTask);
     }
     await model.remove();
   }
 
   async findTask(taskId: string) {
-    const task = await TaskModel.findOne({ taskId });
+    const task = await this.Task.findOne({ taskId });
     if (task) {
       return ModelConvertor.convertModelToEntity(task, TaskEntity);
     }
     // try to read from history
-    const history = await HistoryTaskModel.findOne({ taskId });
+    const history = await this.HistoryTask.findOne({ taskId });
     if (history) {
       return ModelConvertor.convertModelToEntity(history, TaskEntity);
     }
     return null;
   }
 
+  async findTaskByBizId(bizId: string) {
+    const task = await this.Task.findOne({ bizId });
+    if (task) {
+      return ModelConvertor.convertModelToEntity(task, TaskEntity);
+    }
+    return null;
+  }
+
+  async findTasks(taskIds: Array<string>): Promise<Array<TaskEntity>> {
+    const tasks = await this.HistoryTask.find({ taskId: { $in: taskIds } });
+    return tasks.map(task => ModelConvertor.convertModelToEntity(task, TaskEntity));
+  }
+
   async findTaskByTargetName(targetName: string, type: TaskType, state?: TaskState) {
     const where: any = { targetName, type };
     if (state) {
       where.state = state;
     }
-    const task = await TaskModel.findOne(where);
+    const task = await this.Task.findOne(where);
     if (task) {
       return ModelConvertor.convertModelToEntity(task, TaskEntity);
     }
@@ -60,7 +108,7 @@ export class TaskRepository extends AbstractRepository {
   async findTimeoutTasks(taskState: TaskState, timeout: number) {
     const timeoutDate = new Date();
     timeoutDate.setTime(timeoutDate.getTime() - timeout);
-    const models = await TaskModel.find({
+    const models = await this.Task.find({
       state: taskState,
       updatedAt: {
         $lt: timeoutDate,
@@ -68,4 +116,12 @@ export class TaskRepository extends AbstractRepository {
     }).limit(1000);
     return models.map(model => ModelConvertor.convertModelToEntity(model, TaskEntity));
   }
+
+  async findTaskByAuthorIpAndType(authorIp: string, type: TaskType) {
+    const models = await this.Task.find({
+      type,
+      authorIp,
+    }).limit(1000);
+    return models.map(model => ModelConvertor.convertModelToEntity(model, TaskEntity));
+  }
 }

app/repository/UserRepository.ts

@@ -1,7 +1,7 @@
-import { AccessLevel, ContextProto } from '@eggjs/tegg';
+import { AccessLevel, ContextProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
-import { User as UserModel } from './model/User';
-import { Token as TokenModel } from './model/Token';
+import type { User as UserModel } from './model/User';
+import type { Token as TokenModel } from './model/Token';
 import { User as UserEntity } from '../core/entity/User';
 import { Token as TokenEntity } from '../core/entity/Token';
 import { AbstractRepository } from './AbstractRepository';
@@ -10,19 +10,31 @@ import { AbstractRepository } from './AbstractRepository';
   accessLevel: AccessLevel.PUBLIC,
 })
 export class UserRepository extends AbstractRepository {
+  @Inject()
+  private readonly User: typeof UserModel;
+
+  @Inject()
+  private readonly Token: typeof TokenModel;
+
   async saveUser(user: UserEntity): Promise<void> {
     if (user.id) {
-      const model = await UserModel.findOne({ id: user.id });
+      const model = await this.User.findOne({ id: user.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(user, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(user, UserModel);
+      const model = await ModelConvertor.convertEntityToModel(user, this.User);
       this.logger.info('[UserRepository:saveUser:new] id: %s, userId: %s', model.id, model.userId);
     }
   }
 
   async findUserByName(name: string) {
-    const model = await UserModel.findOne({ name });
+    const model = await this.User.findOne({ name });
+    if (!model) return null;
+    return ModelConvertor.convertModelToEntity(model, UserEntity);
+  }
+
+  async findUserByUserId(userId: string) {
+    const model = await this.User.findOne({ userId });
     if (!model) return null;
     return ModelConvertor.convertModelToEntity(model, UserEntity);
   }
@@ -30,7 +42,7 @@ export class UserRepository extends AbstractRepository {
   async findUserAndTokenByTokenKey(tokenKey: string) {
     const token = await this.findTokenByTokenKey(tokenKey);
     if (!token) return null;
-    const userModel = await UserModel.findOne({ userId: token.userId });
+    const userModel = await this.User.findOne({ userId: token.userId });
     if (!userModel) return null;
     return {
       token,
@@ -39,30 +51,30 @@ export class UserRepository extends AbstractRepository {
   }
 
   async findTokenByTokenKey(tokenKey: string) {
-    const model = await TokenModel.findOne({ tokenKey });
+    const model = await this.Token.findOne({ tokenKey });
     if (!model) return null;
     return ModelConvertor.convertModelToEntity(model, TokenEntity);
   }
 
   async saveToken(token: TokenEntity): Promise<void> {
     if (token.id) {
-      const model = await TokenModel.findOne({ id: token.id });
+      const model = await this.Token.findOne({ id: token.id });
       if (!model) return;
       await ModelConvertor.saveEntityToModel(token, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(token, TokenModel);
+      const model = await ModelConvertor.convertEntityToModel(token, this.Token);
       this.logger.info('[UserRepository:saveToken:new] id: %s, tokenId: %s', model.id, model.tokenId);
     }
   }
 
   async removeToken(tokenId: string) {
-    const removeCount = await TokenModel.remove({ tokenId });
+    const removeCount = await this.Token.remove({ tokenId });
     this.logger.info('[UserRepository:removeToken:remove] %d rows, tokenId: %s',
       removeCount, tokenId);
   }
 
   async listTokens(userId: string): Promise<TokenEntity[]> {
-    const models = await TokenModel.find({ userId });
+    const models = await this.Token.find({ userId });
     return models.map(model => ModelConvertor.convertModelToEntity(model, TokenEntity));
   }
 }

app/repository/model/HistoryTask.ts

@@ -1,5 +1,5 @@
 import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
-import { DataTypes, Bone } from 'leoric';
+import { DataTypes, Bone, LENGTH_VARIANTS } from 'leoric';
 import { TaskState, TaskType } from '../../common/enum/Task';
 
 @Model()
@@ -48,6 +48,6 @@ export class HistoryTask extends Bone {
   @Attribute(DataTypes.INTEGER)
   attempts: number;
 
-  @Attribute(DataTypes.TEXT('long'))
+  @Attribute(DataTypes.TEXT(LENGTH_VARIANTS.long))
   error: string;
 }

app/repository/model/Hook.ts

@@ -0,0 +1,47 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { DataTypes, Bone } from 'leoric';
+import { HookType } from '../../common/enum/Hook';
+
+@Model()
+export class Hook extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(24), {
+    unique: true,
+  })
+  hookId: string;
+
+  @Attribute(DataTypes.STRING(20))
+  type: HookType;
+
+  @Attribute(DataTypes.STRING(24))
+  ownerId: string;
+
+  @Attribute(DataTypes.STRING(428))
+  name: string;
+
+  @Attribute(DataTypes.STRING(2048))
+  endpoint: string;
+
+  @Attribute(DataTypes.STRING(200))
+  secret: string;
+
+  @Attribute(DataTypes.STRING(24), {
+    allowNull: true,
+  })
+  latestTaskId: string;
+
+  @Attribute(DataTypes.BOOLEAN)
+  enable: boolean;
+}

app/repository/model/Package.ts

@@ -21,6 +21,9 @@ export class Package extends Bone {
   })
   packageId: string;
 
+  @Attribute(DataTypes.STRING(24))
+  registryId: string;
+
   @Attribute(DataTypes.STRING(214))
   scope: string;
 

app/repository/model/PackageVersionBlock.ts

@@ -1,5 +1,5 @@
 import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
-import { DataTypes, Bone } from 'leoric';
+import { DataTypes, Bone, LENGTH_VARIANTS } from 'leoric';
 
 @Model()
 export class PackageVersionBlock extends Bone {
@@ -26,6 +26,6 @@ export class PackageVersionBlock extends Bone {
   @Attribute(DataTypes.STRING(256))
   version: string;
 
-  @Attribute(DataTypes.TEXT('long'))
+  @Attribute(DataTypes.TEXT(LENGTH_VARIANTS.long))
   reason: string;
 }

app/repository/model/Registry.ts

@@ -0,0 +1,39 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { RegistryType } from '../../common/enum/Registry';
+import { DataTypes, Bone } from 'leoric';
+
+@Model()
+export class Registry extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(24), {
+    unique: true,
+  })
+  registryId: string;
+
+  @Attribute(DataTypes.STRING(256))
+  name: string;
+
+  @Attribute(DataTypes.STRING(4096))
+  host: string;
+
+  @Attribute(DataTypes.STRING(4096), { name: 'change_stream' })
+  changeStream: string;
+
+  @Attribute(DataTypes.STRING(4096), { name: 'user_prefix' })
+  userPrefix: string;
+
+  @Attribute(DataTypes.STRING(256))
+  type: RegistryType;
+
+}

app/repository/model/Scope.ts

@@ -0,0 +1,26 @@
+import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
+import { DataTypes, Bone } from 'leoric';
+
+@Model()
+export class Scope extends Bone {
+  @Attribute(DataTypes.BIGINT, {
+    primary: true,
+    autoIncrement: true,
+  })
+  id: bigint;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_create' })
+  createdAt: Date;
+
+  @Attribute(DataTypes.DATE, { name: 'gmt_modified' })
+  updatedAt: Date;
+
+  @Attribute(DataTypes.STRING(214))
+  name: string;
+
+  @Attribute(DataTypes.STRING(256))
+  registryId: string;
+
+  @Attribute(DataTypes.STRING(256))
+  scopeId: string;
+}

app/repository/model/Task.ts

@@ -1,5 +1,5 @@
 import { Attribute, Model } from '@eggjs/tegg-orm-decorator';
-import { DataTypes, Bone } from 'leoric';
+import { DataTypes, Bone, LENGTH_VARIANTS } from 'leoric';
 import { TaskState, TaskType } from '../../common/enum/Task';
 
 @Model()
@@ -48,6 +48,11 @@ export class Task extends Bone {
   @Attribute(DataTypes.INTEGER)
   attempts: number;
 
-  @Attribute(DataTypes.TEXT('long'))
+  @Attribute(DataTypes.TEXT(LENGTH_VARIANTS.long))
   error: string;
+
+  @Attribute(DataTypes.STRING(48), {
+    unique: true,
+  })
+  bizId: string;
 }

app/repository/util/ModelConvertor.ts

@@ -10,7 +10,7 @@ const ID = 'id';
 
 export class ModelConvertor {
   static async convertEntityToModel<T extends Bone>(entity: object, ModelClazz: EggProtoImplClass<T>, options?): Promise<T> {
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }
@@ -31,11 +31,29 @@ export class ModelConvertor {
     return model as T;
   }
 
+  static convertEntityToChanges<T extends Bone>(entity: object, ModelClazz: EggProtoImplClass<T>) {
+    const changes = {};
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
+    if (!metadata) {
+      throw new Error(`Model ${ModelClazz.name} has no metadata`);
+    }
+    for (const attributeMeta of metadata.attributes) {
+      const modelPropertyName = attributeMeta.propertyName;
+      const entityPropertyName = ModelConvertorUtil.getEntityPropertyName(ModelClazz, modelPropertyName);
+      if (entityPropertyName === CREATED_AT) continue;
+      const attributeValue = _.get(entity, entityPropertyName);
+      changes[modelPropertyName] = attributeValue;
+    }
+    changes[UPDATED_AT] = new Date();
+    entity[UPDATED_AT] = changes[UPDATED_AT];
+    return changes;
+  }
+
   // TODO: options is QueryOptions, should let leoric export it to use
   // Find out which attributes changed and set `updatedAt` to now
   static async saveEntityToModel<T extends Bone>(entity: object, model: T, options?): Promise<boolean> {
     const ModelClazz = model.constructor as EggProtoImplClass<T>;
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }
@@ -46,9 +64,10 @@ export class ModelConvertor {
       const attributeValue = _.get(entity, entityPropertyName);
       model[modelPropertyName] = attributeValue;
     }
-    if (!model.changed()) {
-      return false;
-    }
+
+    // 不允许设置 UPDATED_AT
+    // 通过 leoric 进行更新
+    model[UPDATED_AT] = undefined;
     await model.save(options);
     entity[UPDATED_AT] = model[UPDATED_AT];
     return true;
@@ -57,7 +76,7 @@ export class ModelConvertor {
   static convertModelToEntity<T>(bone: Bone, entityClazz: EggProtoImplClass<T>, data?: object): T {
     data = data || {};
     const ModelClazz = bone.constructor;
-    const metadata = ModelMetadataUtil.getControllerMetadata(ModelClazz);
+    const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }

app/repository/util/ModelConvertorUtil.ts

@@ -12,7 +12,7 @@ export class ModelConvertorUtil {
    * If has no entity property info, use modelProperty as default value
    */
   static getEntityPropertyName(clazz: EggProtoImplClass, modelProperty: string): string {
-    const propertyMap: Map<string, string> | undefined = MetadataUtil.getOwnMetaData(ENTITY_PROPERTY_MAP_ATTRIBUTE, clazz);
+    const propertyMap: Map<string, string> | undefined = MetadataUtil.getMetaData(ENTITY_PROPERTY_MAP_ATTRIBUTE, clazz);
     return propertyMap?.get(modelProperty) ?? modelProperty;
   }
 }

app/schedule/ChangesStreamWorker.ts

@@ -1,26 +0,0 @@
-import { Subscription } from 'egg';
-import { ChangesStreamService } from '../core/service/ChangesStreamService';
-
-const cnpmcoreCore = 'cnpmcoreCore';
-
-export default class ChangesStreamWorker extends Subscription {
-  static get schedule() {
-    return {
-      interval: 60000,
-      type: 'worker',
-    };
-  }
-
-  async subscribe() {
-    const { ctx, app } = this;
-    if (app.config.cnpmcore.syncMode !== 'all' || !app.config.cnpmcore.enableChangesStream) return;
-
-    await ctx.beginModuleScope(async () => {
-      const changesStreamService: ChangesStreamService = ctx.module[cnpmcoreCore].changesStreamService;
-      const task = await changesStreamService.findExecuteTask();
-      if (!task) return;
-      ctx.logger.warn('[ChangesStreamWorker:start] taskId: %s', task.taskId);
-      await changesStreamService.executeTask(task);
-    });
-  }
-}

app/schedule/CheckRecentlyUpdatedPackages.ts

@@ -1,70 +0,0 @@
-import { Subscription } from 'egg';
-import { PackageSyncerService } from '../core/service/PackageSyncerService';
-
-const cnpmcoreCore = 'cnpmcoreCore';
-
-// https://github.com/cnpm/cnpmcore/issues/9
-export default class CheckRecentlyUpdatedPackages extends Subscription {
-  static get schedule() {
-    return {
-      // every 5 mins
-      interval: 60000 * 5,
-      type: 'worker',
-    };
-  }
-
-  async subscribe() {
-    const { ctx, app } = this;
-    if (app.config.cnpmcore.syncMode !== 'all' || !app.config.cnpmcore.enableCheckRecentlyUpdated) return;
-
-    await ctx.beginModuleScope(async () => {
-      const packageSyncerService: PackageSyncerService = ctx.module[cnpmcoreCore].packageSyncerService;
-      const pageSize = 36;
-      const pageCount = app.config.env === 'unittest' ? 2 : 5;
-      for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
-        const offset = pageSize * pageIndex;
-        const pageUrl = `https://www.npmjs.com/browse/updated?offset=${offset}`;
-        let html = '';
-        try {
-          const { status, data } = await ctx.httpclient.request(pageUrl, {
-            followRedirect: true,
-            timeout: 10000,
-          });
-          ctx.logger.info('[CheckRecentlyUpdatedPackages.subscribe][%s] request %s status: %s, data size: %s',
-            pageIndex, pageUrl, status, data.length);
-          if (status === 200) {
-            html = data.toString();
-          }
-        } catch (err) {
-          ctx.logger.info('[CheckRecentlyUpdatedPackages.subscribe:error][%s] request %s error: %s',
-            pageIndex, pageUrl, err);
-          ctx.logger.error(err);
-          continue;
-        }
-
-        const matchs = /window\.__context__ = ([^<]+?)<\/script>/.exec(html);
-        if (!matchs) continue;
-
-        try {
-          const data = JSON.parse(matchs[1]);
-          const packages = data.context.packages || [];
-          if (Array.isArray(packages)) {
-            ctx.logger.info('[CheckRecentlyUpdatedPackages.subscribe][%s] parse %d packages on %s',
-              pageIndex, packages.length, pageUrl);
-            for (const pkg of packages) {
-              const task = await packageSyncerService.createTask(pkg.name, {
-                tips: `Sync cause by recently updated packages ${pageUrl}`,
-              });
-              ctx.logger.info('[CheckRecentlyUpdatedPackages.subscribe:createTask][%s] taskId: %s, targetName: %s',
-                pageIndex, task.taskId, task.targetName);
-            }
-          }
-        } catch (err) {
-          ctx.logger.info('[CheckRecentlyUpdatedPackages.subscribe:error][%s] parse %s context json error: %s',
-            pageIndex, pageUrl, err);
-          ctx.logger.error(err);
-        }
-      }
-    });
-  }
-}

app/schedule/CreateSyncBinaryTask.ts

@@ -1,29 +0,0 @@
-import { Subscription } from 'egg';
-import { BinarySyncerService } from '../core/service/BinarySyncerService';
-import binaries from '../../config/binaries';
-
-const cnpmcoreCore = 'cnpmcoreCore';
-
-export default class CreateSyncBinaryTask extends Subscription {
-  static get schedule() {
-    return {
-      // every 5 mins
-      interval: 60000 * 5,
-      type: 'worker',
-    };
-  }
-
-  async subscribe() {
-    const { ctx, app } = this;
-    if (!app.config.cnpmcore.enableSyncBinary) return;
-
-    await ctx.beginModuleScope(async () => {
-      const binarySyncerService: BinarySyncerService = ctx.module[cnpmcoreCore].binarySyncerService;
-      for (const binary of Object.values(binaries)) {
-        if (app.config.env === 'unittest' && binary.category !== 'node') continue;
-        if (binary.disable) continue;
-        await binarySyncerService.createTask(binary.category);
-      }
-    });
-  }
-}