Release 3.75.1

[skip ci]

## [3.75.1](https://github.com/cnpm/cnpmcore/compare/v3.75.0...v3.75.1) (2025-03-09)

### Bug Fixes

* only get packageId from database ([#768](https://github.com/cnpm/cnpmcore/issues/768)) ([bc068d1](bc068d165c))

.dockerignore

@@ -0,0 +1,6 @@
+logs
+node_modules
+run
+typings
+.cnpmcore*
+coverage

.env.example

@@ -0,0 +1,9 @@
+# CNPMCORE_DATABASE_TYPE=MySQL
+# CNPMCORE_DATABASE_USER=root
+# CNPMCORE_DATABASE_PASSWORD=
+# CNPMCORE_DATABASE_NAME=cnpmcore
+
+# CNPMCORE_DATABASE_TYPE=PostgreSQL
+# CNPMCORE_DATABASE_USER=postgres
+# CNPMCORE_DATABASE_PASSWORD=postgres
+# CNPMCORE_DATABASE_NAME=cnpmcore

.github/workflows/chatgpt-cr.yml

@@ -1,23 +0,0 @@
-name: 🤖 ChatGPT Code Review
-
-permissions:
-  contents: read
-  pull-requests: write
-
-on:
-  pull_request:
-    types: [opened, reopened, synchronize]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: anc95/ChatGPT-CodeReview@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          # Optional
-          LANGUAGE: Chinese
-          MODEL:
-          top_p: 1
-          temperature: 1

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,9 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master ]
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '41 13 * * 3'
+    branches: [ master ]
 
 jobs:
   analyze:

.github/workflows/nodejs.yml

@@ -3,9 +3,74 @@
 
 name: Node.js CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
 
 jobs:
+  test-postgresql-fs-nfs:
+    runs-on: ${{ matrix.os }}
+
+    services:
+      # https://docs.github.com/en/actions/use-cases-and-examples/using-containerized-services/creating-postgresql-service-containers
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+      redis:
+        # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#example-mapping-redis-ports
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.20.0, 18, 20, 22]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout Git Source
+      uses: actions/checkout@v4
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Dependencies
+      run: npm i -g npminstall && npminstall
+
+    - name: Continuous Integration
+      run: npm run ci:postgresql
+      env:
+        # The hostname used to communicate with the PostgreSQL service container
+        POSTGRES_HOST: localhost
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+        # The default PostgreSQL port
+        POSTGRES_PORT: 5432
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
   test-mysql57-fs-nfs:
     runs-on: ${{ matrix.os }}
 
@@ -28,15 +93,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [16, 18, 20]
+        node-version: [18.20.0, 18, 20, 22]
         os: [ubuntu-latest]
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 
@@ -79,15 +144,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [16, 18, 20]
+        node-version: [18.20.0, 18, 20, 22]
         os: [ubuntu-latest]
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 

.github/workflows/release.yml

@@ -1,18 +1,12 @@
 name: Release
 on:
-  # 合并后自动发布
   push:
-    branches: [ master, main ]
-
-  # 手动发布
-  workflow_dispatch: {}
+    branches: [ master ]
 
 jobs:
   release:
     name: Node.js
-    uses: artusjs/github-actions/.github/workflows/node-release.yml@v1
+    uses: cnpm/github-actions/.github/workflows/node-release.yml@master
     secrets:
       NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       GIT_TOKEN: ${{ secrets.GIT_TOKEN }}
-    with:
-      checkTest: false

.gitignore

@@ -120,3 +120,5 @@ dist
 .DS_Store
 run
 !test/ctx_register.js
+
+.egg/

.vscode/launch.json

@@ -17,8 +17,6 @@
       ],
       "console": "integratedTerminal",
       "restart": true,
-      "protocol": "auto",
-      "port": 9229,
       "autoAttachChildProcesses": true
     },
     {
@@ -32,8 +30,6 @@
         "--",
         "--inspect-brk"
       ],
-      "protocol": "auto",
-      "port": 9229,
       "autoAttachChildProcesses": true
     }
   ]

DEVELOPER.md

@@ -2,19 +2,37 @@
 
 ## 环境初始化
 
-本项目的外部服务依赖有：MySQL 数据服务、Redis 缓存服务。
+本项目的外部服务依赖有：MySQL 数据库或 PostgreSQL 数据库、Redis 缓存服务。
+
+生成本地开发环境配置文件：
+
+```bash
+cp .env.example .env
+```
 
 可以通过 Docker 来快速启动本地开发环境：
 
+MySQL 开发环境：
+
 ```bash
-# 启动本地依赖服务
-docker-compose up -d
+# 启动本地依赖服务 - MySQL + Redis
+docker-compose -f docker-compose.yml up -d
 
 # 关闭本地依赖服务
-docker-compose down
+docker-compose -f docker-compose.yml down
 ```
 
-> 手动初始化依赖服务参见[文档](./docs/setup.md)
+PostgreSQL 开发环境：
+
+```bash
+# 启动本地依赖服务 - PostgreSQL + Redis
+docker-compose -f docker-compose-postgres.yml up -d
+
+# 关闭本地依赖服务
+docker-compose -f docker-compose-postgres.yml down
+```
+
+> 手动初始化依赖服务参见[本地开发环境 - MySQL](./docs/setup.md) 或 [本地开发环境 - PostgreSQL](./docs/setup-with-postgresql.md)
 
 ## 本地开发
 
@@ -24,25 +42,69 @@ docker-compose down
 npm install
 ```
 
-### 开发运行
+### 开发运行 - MySQL
 
 ```bash
 # 初始化数据库
-MYSQL_DATABASE=cnpmcore bash ./prepare-database.sh
+CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-mysql.sh
 
 # 启动 Web 服务
 npm run dev
 
 # 访问
 curl -v http://127.0.0.1:7001
+
+# cnpmcore_admin 注册管理员
+npm login --registry=http://127.0.0.1:7001
+```
+
+### 开发运行 - PostgreSQL
+
+```bash
+# 初始化数据库
+CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-postgresql.sh
+
+# 启动 Web 服务
+npm run dev:postgresql
+
+# 访问
+curl -v http://127.0.0.1:7001
+```
+
+### 登录和测试发包
+
+> cnpmcore 默认不开放注册，可以通过 `config.default.ts` 中的 `allowPublicRegistration` 配置开启，否则只有管理员可以登录
+
+
+注册 cnpmcore_admin 管理员
+
+```bash
+npm login --registry=http://127.0.0.1:7001
+
+# 验证登录
+npm whoami --registry=http://127.0.0.1:7001
+```
+
+发包
+
+```bash
+npm publish --registry=http://127.0.0.1:7001
 ```
 
 ### 单元测试
 
+MySQL
+
 ```bash
 npm run test
 ```
 
+PostgreSQL
+
+```bash
+npm run test:postgresql
+```
+
 ## 项目结构
 
 ```txt
@@ -268,9 +330,9 @@ Repository 依赖 Model，然后被 Service 和 Controller 依赖
 
 可能需要涉及3个地方的修改：
 
-1. sql/*.sql
-2. repository/model/*.ts
-3. core/entity/*.ts
+1. `sql/mysql/*.sql`, `sql/postgresql/*.sql`
+2. `repository/model/*.ts`
+3. `core/entity/*.ts`
 
 目前还不会做 Model 到 SQL 的自动转换生成，核心原因有：
 

Dockerfile

@@ -0,0 +1,17 @@
+FROM node:20
+
+# Create app directory
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY . .
+
+RUN npm install -g npminstall --registry=https://registry.npmmirror.com \
+  && npminstall -c \
+  && npm run tsc
+
+ENV NODE_ENV=production \
+  EGG_SERVER_ENV=prod
+
+EXPOSE 7001
+CMD ["npm", "run", "start:foreground"]

History.md

@@ -1,317 +0,0 @@
-
-2.9.0 / 2022-12-15
-==================
-
-**features**
-  * [[`c562645`](http://github.com/cnpm/cnpmcore/commit/c562645db7c88f9c3c5787fd450b457574d1cce6)] - feat: suspend task before app close (#365) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.8.1 / 2022-12-05
-==================
-
-**features**
-  * [[`fad30ad`](http://github.com/cnpm/cnpmcore/commit/fad30adc564c931c0bf63828d83bab84105aaef0)] - feat: npm command support npm v6 (#356) (laibao101 <<369632567@qq.com>>)
-
-**fixes**
-  * [[`f961219`](http://github.com/cnpm/cnpmcore/commit/f961219dbe4676156e1766db82379ee40087bcd8)] - fix: Sync save ignore ER_DUP_ENTRY error (#364) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-**others**
-  * [[`7bc0fcc`](http://github.com/cnpm/cnpmcore/commit/7bc0fccaca880efe08228b4109953bd3974d2eb9)] - 🤖 TEST: Fix async function mock (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`84ae9bc`](http://github.com/cnpm/cnpmcore/commit/84ae9bcfa06124255703b926f83fb5e6a6bf9d6b)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
-
-2.8.0 / 2022-11-29
-==================
-
-**others**
-  * [[`d55c680`](http://github.com/cnpm/cnpmcore/commit/d55c680ef906ecb27f7967782ad7d25987cef7d4)] - Event cork (#361) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.7.1 / 2022-11-25
-==================
-
-**fixes**
-  * [[`c6b8aec`](http://github.com/cnpm/cnpmcore/commit/c6b8aecfd0c2b0d454389e931747c431dac5742b)] - fix: request binary error (#360) (Ke Wu <<gemwuu@163.com>>)
-
-2.7.0 / 2022-11-25
-==================
-
-**others**
-  * [[`5738d56`](http://github.com/cnpm/cnpmcore/commit/5738d569ea691c05c3f3b0b74a454a33fefb8fc7)] - refactor: binary sync task use binaryName by default (#358) (Ke Wu <<gemwuu@163.com>>)
-
-2.6.1 / 2022-11-23
-==================
-
-**fixes**
-  * [[`0b35ead`](http://github.com/cnpm/cnpmcore/commit/0b35ead2a0cd73b89d2d961bafec13d7250fe805)] - 🐛 FIX: typo for canvas (fengmk2 <<fengmk2@gmail.com>>)
-
-2.6.0 / 2022-11-23
-==================
-
-**features**
-  * [[`be8387d`](http://github.com/cnpm/cnpmcore/commit/be8387dfa48b9487156542000a93081fa823694a)] - feat: Support canvas sync from different binary (#357) (Ke Wu <<gemwuu@163.com>>)
-
-**fixes**
-  * [[`d6c4cf5`](http://github.com/cnpm/cnpmcore/commit/d6c4cf5029ca6450064fc05696a8624b6c36f0b2)] - fix: duplicate binary task (#354) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.5.2 / 2022-11-11
-==================
-
-**fixes**
-  * [[`7eb209d`](http://github.com/cnpm/cnpmcore/commit/7eb209de1332417db2070846891d78f5afa0cd10)] - fix: create task when waiting (#352) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.5.1 / 2022-11-07
-==================
-
-**others**
-  * [[`e40c502`](http://github.com/cnpm/cnpmcore/commit/e40c5021bb2ba78f8879d19bc477883168560b85)] - 🐛 FIX: Mirror cypress arm64 binary (#351) (fengmk2 <<fengmk2@gmail.com>>)
-
-2.5.0 / 2022-11-04
-==================
-
-**features**
-  * [[`43d77ee`](http://github.com/cnpm/cnpmcore/commit/43d77ee91e52bd74594d9d569b839c1a4b7fbac6)] - feat: long description (#349) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.4.1 / 2022-10-28
-==================
-
-**features**
-  * [[`92350a8`](http://github.com/cnpm/cnpmcore/commit/92350a864313ee42a048d9e83886ef42db3419de)] - 👌 IMPROVE: Show changes stream create task log (#347) (fengmk2 <<fengmk2@gmail.com>>)
-
-**fixes**
-  * [[`28eeeaf`](http://github.com/cnpm/cnpmcore/commit/28eeeafd9870c6b1c5b4f4c23916f6ae73ddda12)] - fix: registry host config (#346) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-  * [[`cd5bd92`](http://github.com/cnpm/cnpmcore/commit/cd5bd923b8d47bf90b5f077ce04777b38653b850)] - 🐛 FIX: Catch all error on changes stream handler (#344) (fengmk2 <<fengmk2@gmail.com>>)
-
-2.4.0 / 2022-10-25
-==================
-
-**features**
-  * [[`6aa302d`](http://github.com/cnpm/cnpmcore/commit/6aa302d074f2c84f39e2065fa20853b007f6fa3b)] - 📦 NEW: Use oss-cnpm v4 (#340) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`a217fd0`](http://github.com/cnpm/cnpmcore/commit/a217fd07ccad3fe5058881654a13e0c69c758717)] - 👌 IMPROVE: Reduce warning log (#326) (fengmk2 <<fengmk2@gmail.com>>)
-
-**fixes**
-  * [[`b19b0a0`](http://github.com/cnpm/cnpmcore/commit/b19b0a0496e35ac1c6b3de746b9221990ba9dc93)] - fix: Lazy set registryId when executeTask (#341) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-**others**
-  * [[`305175a`](http://github.com/cnpm/cnpmcore/commit/305175ab5fcdc3ad3b60055d45cfcacb23065a80)] - 🤖 TEST: Use enum define on unittest (#333) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`07f2eba`](http://github.com/cnpm/cnpmcore/commit/07f2eba137ba625b2d422677a465920617141b87)] - 🤖 TEST: Mock all binary http requests (#328) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`4b0c7dc`](http://github.com/cnpm/cnpmcore/commit/4b0c7dc6196960d34b2529bfde724e97f1af8444)] - 🤖 TEST: Mock all httpclient request (#327) (fengmk2 <<fengmk2@gmail.com>>)
-
-2.3.1 / 2022-10-06
-==================
-
-**features**
-  * [[`bbc08fd`](http://github.com/cnpm/cnpmcore/commit/bbc08fd26887d55b98b70d1ed210caf81f9d5c22)] - 👌 IMPROVE: syncPackageWorkerMaxConcurrentTasks up to 20 (#322) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`5852f22`](http://github.com/cnpm/cnpmcore/commit/5852f22023525d857ff1ceea205e4315c8079877)] - feat: support sync exist mode (#275) (zhangyuantao <<zhangyuantao@163.com>>)
-
-**fixes**
-  * [[`d79634e`](http://github.com/cnpm/cnpmcore/commit/d79634eea749fef1a420988a8599f156f28ee85a)] - 🐛 FIX: Should sync package when registry id is null (#324) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`24f920d`](http://github.com/cnpm/cnpmcore/commit/24f920d65b31f9eb83c1ecda36adf7f9e2c379c3)] - 🐛 FIX: Should run sync package on all worker (#323) (fengmk2 <<fengmk2@gmail.com>>)
-
-2.3.0 / 2022-09-24
-==================
-
-**others**
-  * [[`bd83a19`](http://github.com/cnpm/cnpmcore/commit/bd83a19eca761c96bcee04e6ae91e68eac3cb6bf)] - 👌 IMPROVE: use urllib3 instead (#302) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`35e7d3a`](http://github.com/cnpm/cnpmcore/commit/35e7d3ad3c78712b507d522a0b72b5a6a5a4ec1c)] - 👌 IMPROVE: Enable phpmyadmin and DEBUG_LOCAL_SQL by default (#320) (fengmk2 <<fengmk2@gmail.com>>)
-
-2.2.0 / 2022-09-22
-==================
-
-**features**
-  * [[`bca0fb3`](http://github.com/cnpm/cnpmcore/commit/bca0fb3c37b9f74f3c41ab181dd3113d9dab4c05)] - feat: only allow pkg sync from registry it belong (#317) (killa <<killa123@126.com>>)
-
-**fixes**
-  * [[`7e9beea`](http://github.com/cnpm/cnpmcore/commit/7e9beead576a41de3aa042b92b788bde5d55f44a)] - fix: only append / if path is not empty and not ends with / (#316) (killa <<killa123@126.com>>)
-  * [[`4fe68cb`](http://github.com/cnpm/cnpmcore/commit/4fe68cbf38f303e797b80b88407f714ec76bfae0)] - fix: fix directory path (#313) (killa <<killa123@126.com>>)
-
-**others**
-  * [[`e72ce35`](http://github.com/cnpm/cnpmcore/commit/e72ce3576f9a3cda095e3feac59eeb1d8c1e8033)] - 🤖 TEST: Skip unstable tests (#318) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`171b11f`](http://github.com/cnpm/cnpmcore/commit/171b11f7bba534c993af4088b00f8545216734a9)] - Revert "fix: fix directory path (#313)" (fengmk2 <<fengmk2@gmail.com>>)
-
-2.1.1 / 2022-09-08
-==================
-
-**fixes**
-  * [[`8fb9dd8`](http://github.com/cnpm/cnpmcore/commit/8fb9dd8cf4800afe3f54aba9ee4c0ae05efb4f1d)] - fix: findExecuteTask only return waiting task (#312) (killa <<killa123@126.com>>)
-
-2.1.0 / 2022-09-05
-==================
-
-**features**
-  * [[`c5d2b49`](http://github.com/cnpm/cnpmcore/commit/c5d2b49ab3a0ce0d67f6e7cc19e0be867c92d04c)] - feat: auto get next valid task (#311) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-2.0.0 / 2022-09-05
-==================
-
-**others**
-  * [[`fc4baff`](http://github.com/cnpm/cnpmcore/commit/fc4baff226540e7cfee9adc069e17a59f4050a43)] - chore: refactor schedule with @Schedule (#309) (killa <<killa123@126.com>>)
-
-1.11.6 / 2022-09-04
-==================
-
-**fixes**
-  * [[`768f951`](http://github.com/cnpm/cnpmcore/commit/768f951b6f2509f14c30a70d86a6719107d963a4)] - fix: cnpmjsorg changesstream limit (#310) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-1.11.5 / 2022-09-02
-==================
-
-**fixes**
-  * [[`f673ab8`](http://github.com/cnpm/cnpmcore/commit/f673ab8ba1545909ff6b8e445364646511930891)] - fix: execute state check (#308) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-**others**
-  * [[`091420a`](http://github.com/cnpm/cnpmcore/commit/091420ae2677ecedd1a26a238921321c2a191675)] - 🤖 TEST: Add SQL Review Action (#307) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.11.4 / 2022-08-30
-==================
-
-**fixes**
-  * [[`f9210ca`](http://github.com/cnpm/cnpmcore/commit/f9210ca7e180e19bce08da9ef33e46e990b86ef1)] - fix: changes stream empty (#306) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-1.11.3 / 2022-08-29
-==================
-
-**fixes**
-  * [[`48f228d`](http://github.com/cnpm/cnpmcore/commit/48f228da447d8cde62849fa52cf43bae7754e2e3)] - fix: changes stream updatedAt (#304) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-  * [[`87045ba`](http://github.com/cnpm/cnpmcore/commit/87045ba8b0e14547c93689600eb7e2c1de2a611b)] - fix: task updatedAt save (#305) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-1.11.2 / 2022-08-28
-==================
-
-**fixes**
-  * [[`4e8700c`](http://github.com/cnpm/cnpmcore/commit/4e8700c4f7c6fb5c4f4d4a2b9a9546096c5d10e2)] - fix: only create createHookTask if hook enable (#299) (killa <<killa123@126.com>>)
-
-**others**
-  * [[`e06c841`](http://github.com/cnpm/cnpmcore/commit/e06c841537113fdb0c00beb22b0a55378c61ce80)] - 🐛 FIX: Should sync public package when registryName not exists (#303) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`f139444`](http://github.com/cnpm/cnpmcore/commit/f139444213403494ebe9bf073df62125413892d9)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`c4a9de5`](http://github.com/cnpm/cnpmcore/commit/c4a9de598dce9a1b82bbcdd91968a15bbc5a4b6b)] - Create SECURITY.md (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`709d65b`](http://github.com/cnpm/cnpmcore/commit/709d65bd0473856c9bfc4416ea2ca375136e354f)] - 🤖 TEST: Use diff bucket on OSS test (#301) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`9576699`](http://github.com/cnpm/cnpmcore/commit/95766990fa9c4c2c43d462f6b151557425b0c741)] - chore: use AsyncGenerator insteadof Transform stream (#300) (killa <<killa123@126.com>>)
-  * [[`3ed5269`](http://github.com/cnpm/cnpmcore/commit/3ed5269f1d22ca3aaca89a90a4fff90f293e2464)] - 📦 NEW: Mirror better-sqlite3 binary (#296) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.11.1 / 2022-08-24
-==================
-
-**fixes**
-  * [[`359a150`](http://github.com/cnpm/cnpmcore/commit/359a150eb450d69e6523b20efcc5c7cfe3efab4d)] - fix: changes stream (#297) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-
-1.11.0 / 2022-08-23
-==================
-
-**features**
-  * [[`a91c8ac`](http://github.com/cnpm/cnpmcore/commit/a91c8ac4d05dc903780fda516b09364a05a2b1e6)] - feat: sync package from spec regsitry (#293) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-  * [[`de37008`](http://github.com/cnpm/cnpmcore/commit/de37008261b05845f392d66764cdfe14ae324756)] - feat: changesStream adapter & needSync() method (#292) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-  * [[`4b506c8`](http://github.com/cnpm/cnpmcore/commit/4b506c8371697ddacdbe99a8ecb330bfc1911ec6)] - feat: init registry & scope (#286) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
-  * [[`41c6e24`](http://github.com/cnpm/cnpmcore/commit/41c6e24c84d546eb9d5515cc0940cc3e4274687b)] - feat: impl trigger Hooks (#289) (killa <<killa123@126.com>>)
-  * [[`79cb826`](http://github.com/cnpm/cnpmcore/commit/79cb82615f04bdb3da3ccbe09bb6a861608b69c5)] - feat: impl migration sql (#290) (killa <<killa123@126.com>>)
-  * [[`4cfa8ed`](http://github.com/cnpm/cnpmcore/commit/4cfa8ed9d687ce7d950d7d20c0ea28221763ba5f)] - feat: impl hooks api (#287) (killa <<killa123@126.com>>)
-  * [[`47d53d2`](http://github.com/cnpm/cnpmcore/commit/47d53d22ad03c02ee9cb9035a38ae205a6d38381)] - feat: add bizId for task (#285) (killa <<killa123@126.com>>)
-  * [[`3b1536b`](http://github.com/cnpm/cnpmcore/commit/3b1536b070b2f9062bc2cc377db96d2f4a160efc)] - feat: add node-webrtc mirror (#274) (Opportunity <<opportunity@live.in>>)
-
-**others**
-  * [[`7106807`](http://github.com/cnpm/cnpmcore/commit/710680742a078b2faf4cb18c3a39c0397308712e)] - 🐛 FIX: Should show queue size on logging (#280) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`3a41b21`](http://github.com/cnpm/cnpmcore/commit/3a41b2161cc99bb2f6f6dd7cbaa7abef25ff4393)] - 🐛 FIX: Handle binary configuration value (#278) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.10.0 / 2022-08-04
-==================
-
-**features**
-  * [[`c2b7d5a`](http://github.com/cnpm/cnpmcore/commit/c2b7d5aa98b5ba8649ec246c616574a22e9a74b8)] - feat: use sort set to impl queue (#277) (killa <<killa123@126.com>>)
-
-1.9.1 / 2022-07-29
-==================
-
-**fixes**
-  * [[`c54aa21`](http://github.com/cnpm/cnpmcore/commit/c54aa2165c3938dcbb5a2b3b54e66a0d961cc813)] - fix: check executingCount after task is done (#276) (killa <<killa123@126.com>>)
-
-**others**
-  * [[`3268d03`](http://github.com/cnpm/cnpmcore/commit/3268d030b620825c8c2e6331e1745c1788066c61)] - 🤖 TEST: show package not use cache if isSync (#273) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.9.0 / 2022-07-25
-==================
-
-**features**
-  * [[`af6a75a`](http://github.com/cnpm/cnpmcore/commit/af6a75af32ea04c90fda82be3a56c99ec77e5807)] - feat: add forceSyncHistory options (#271) (killa <<killa123@126.com>>)
-
-1.8.0 / 2022-07-21
-==================
-
-**features**
-  * [[`b49a38c`](http://github.com/cnpm/cnpmcore/commit/b49a38c77e044c978e6de32a9d3e257cc90ea7c1)] - feat: use Model with inject (#269) (killa <<killa123@126.com>>)
-
-1.7.1 / 2022-07-20
-==================
-
-**fixes**
-  * [[`52fca55`](http://github.com/cnpm/cnpmcore/commit/52fca55aa883865f0ae70bfc1ff274c313b8f76a)] - fix: show package not use cache if isSync (#268) (killa <<killa123@126.com>>)
-
-1.7.0 / 2022-07-12
-==================
-
-**others**
-  * [[`4f7ce8b`](http://github.com/cnpm/cnpmcore/commit/4f7ce8b4b2a5806a225ce67228388e14388b7059)] - deps: upgrade leoric to 2.x (#262) (killa <<killa123@126.com>>)
-
-1.6.0 / 2022-07-11
-==================
-
-**features**
-  * [[`1b9a9c7`](http://github.com/cnpm/cnpmcore/commit/1b9a9c70f66d8393e3b132f18713461a9243db73)] - feat: mirror nydus binaries (#261) (killa <<killa123@126.com>>)
-
-**others**
-  * [[`c1256bf`](http://github.com/cnpm/cnpmcore/commit/c1256bf3807bcc9a5c8be2ec5bf5ca8a5eef112e)] - 🐛 FIX: Ignore 403 status on s3 download fail (#260) (fengmk2 <<fengmk2@gmail.com>>)
-  * [[`d685772`](http://github.com/cnpm/cnpmcore/commit/d6857724307fb0df0c4c118491784b30d19a9a15)] - 🐛 FIX: skia-canvas should use NodePreGypBinary (#259) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.5.0 / 2022-07-09
-==================
-
-**features**
-  * [[`b15b10c`](http://github.com/cnpm/cnpmcore/commit/b15b10c5c6cfb32bcc2b1d94434cdd16871ae565)] - feat(mirror): add skia-canvas mirror (#258) (Beace <<beaceshimin@gmail.com>>)
-
-**others**
-  * [[`2bd6ed0`](http://github.com/cnpm/cnpmcore/commit/2bd6ed0e5dace1d8840c342ecf4c86e8973dc6b7)] - 👌 IMPROVE: use tegg@1.2.0 (fengmk2 <<fengmk2@gmail.com>>)
-
-1.4.0 / 2022-06-28
-==================
-
-**features**
-  * [[`57da0a3`](http://github.com/cnpm/cnpmcore/commit/57da0a3c7e56d6613b57391948949ffea24ec058)] - feat: add configuration enableNopmClientAndVersionCheck (laibao101 <<369632567@qq.com>>)
-
-**others**
-  * [[`bf62932`](http://github.com/cnpm/cnpmcore/commit/bf62932f2e5224de6e34b873bf690a6e887b94b0)] - 🤖 TEST: Fix unstable test cases on OSS env (#254) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.3.2 / 2022-06-27
-==================
-
-**fixes**
-  * [[`c63159d`](http://github.com/cnpm/cnpmcore/commit/c63159d8df804fe711b664606fe42be42010eb38)] - fix: valid npm client with correct pattern (#252) (TZ | 天猪 <<atian25@qq.com>>)
-
-**others**
-  * [[`d578baf`](http://github.com/cnpm/cnpmcore/commit/d578bafff07a0f9d4dd75393492cffc7f5d2660b)] - 🐛 FIX: Ignore exists seq on changes worker (#253) (fengmk2 <<fengmk2@gmail.com>>)
-
-1.3.1 / 2022-06-24
-==================
-
-**fixes**
-  * [[`4ea0ef6`](http://github.com/cnpm/cnpmcore/commit/4ea0ef63b7af9fd4dcc247c2c2ac8e4d579f941a)] - fix: query changes with order by id asc (#251) (killa <<killa123@126.com>>)
-
-1.3.0 / 2022-06-24
-==================
-
-**features**
-  * [[`0948a71`](http://github.com/cnpm/cnpmcore/commit/0948a71a40ac4897d129ef56830665dc028f07c7)] - feat: read enableChangesStream when sync changes stream (#250) (killa <<killa123@126.com>>)
-
-1.2.0 / 2022-06-20
-==================
-
-**others**
-  * [[`c0d8b52`](http://github.com/cnpm/cnpmcore/commit/c0d8b52ea09736ac11b0ef780aec781d172fb94c)] - refactor: move CacheAdapter to ContextProto (#249) (killa <<killa123@126.com>>)
-
-1.1.0 / 2022-06-20
-==================
-
-**features**
-  * [[`66b411e`](http://github.com/cnpm/cnpmcore/commit/66b411ea5bf6192dc9509df408525078e7128a27)] - feat: add type for exports (#248) (killa <<killa123@126.com>>)
-
-1.0.0 / 2022-06-17
-==================
-
-**others**
-  * [[`5cadbf4`](http://github.com/cnpm/cnpmcore/commit/5cadbf4b22bee7d85cd14526f5d6c6e2cd3a2e4b)] - refactor: add infra module (#245) (killa <<killa123@126.com>>),fatal: No names found, cannot describe anything.
-

INTEGRATE.md

@@ -1,4 +1,5 @@
 # 🥚 如何在 [tegg](https://github.com/eggjs/tegg) 中集成 cnpmcore
+
 > 文档中的示例项目可以在 [这里](https://github.com/eggjs/examples/commit/bed580fe053ae573f8b63f6788002ff9c6e7a142) 查看，在开始前请确保已阅读 [DEVELOPER.md](DEVELOPER.md) 中的相关文档，完成本地开发环境搭建。
 
 在生产环境中，我们也可以直接部署 cnpmcore 系统，实现完整的 Registry 镜像功能。
@@ -12,7 +13,8 @@
 ## 🚀 快速开始
 
 ### 🆕 新建一个 tegg 应用
-> 我们以 https://github.com/eggjs/examples/tree/master/hello-tegg 为例
+
+> 我们以 <https://github.com/eggjs/examples/tree/master/hello-tegg> 为例
 
 ```shell
 .
@@ -37,6 +39,7 @@
   ```
 
 1. 修改 `ts-config.json` 配置，这是因为 cnpmcore 使用了 [subPath](https://nodejs.org/api/packages.html#subpath-exports)
+
     ```json
     {
       "extends": "@eggjs/tsconfig",
@@ -50,6 +53,7 @@
     ```
 
 2. 修改 `config/plugin.ts` 文件，开启 cnpmcore 依赖的一些插件
+
     ```typescript
     // 开启如下插件
     {
@@ -76,7 +80,24 @@
     }
     ```
 
-3. 修改 `config.default.ts` 文件，可以直接复制 cnpmcore 中的内容
+3. 修改 `config.default.ts` 文件，可以直接覆盖默认配置
+
+```typescript
+import { SyncMode } from 'cnpmcore/common/constants';
+import { cnpmcoreConfig } from 'cnpmcore/common/config';
+
+export default () => {
+  const config = {};
+  config.cnpmcore = {
+    ...cnpmcoreConfig,
+    enableChangesStream: false,
+    syncMode: SyncMode.all,
+    allowPublicRegistration: true,
+    // 放开注册配置
+  };
+  return config;
+}
+```
 
 ### 🧑‍🤝‍🧑 集成 cnpmcore
 
@@ -90,7 +111,7 @@
     │   └── package.json
     ```
 
-  * 添加 `package.json` ，声明 infra 作为一个 eggModule 单元
+* 添加 `package.json` ，声明 infra 作为一个 eggModule 单元
 
     ```JSON
     {
@@ -101,7 +122,7 @@
     }
     ```
 
-  * 添加 `XXXAdapter.ts` 在对应的 Adapter 中继承 cnpmcore 默认的 Adapter，以 AuthAdapter 为例
+* 添加 `XXXAdapter.ts` 在对应的 Adapter 中继承 cnpmcore 默认的 Adapter，以 AuthAdapter 为例
 
     ```typescript
       import { AccessLevel, SingletonProto } from '@eggjs/tegg';
@@ -145,12 +166,14 @@
 我们以 AuthAdapter 为例，来实现 npm cli 的 SSO 登录的功能。
 
 我们需要实现了 getAuthUrl 和 ensureCurrentUser 这两个方法:
+
   1. getAuthUrl 引导用户访问企业内实际的登录中心。
   2. ensureCurrentUser 当用户完成访问后，需要回调到应用进行鉴权流程。
 我们约定通过 `POST /-/v1/login/sso/:sessionId` 这个路由来进行登录验证。
 当然，你也可以任意修改地址和登录回调，只需保证更新 redis 中的 token 状态即可。
 
 修改 AuthAdapter.ts 文件
+
 ```typescript
 import { AccessLevel, EggContext, SingletonProto } from '@eggjs/tegg';
 import { AuthAdapter } from 'cnpmcore/infra/AuthAdapter';
@@ -185,6 +208,7 @@ export class MyAuthAdapter extends AuthAdapter {
 ```
 
 修改 HelloController 的实现，实际也可以通过登录中心回调、页面确认等方式实现
+
 ```typescript
   // 触发回调接口，会自动完成用户创建
   await this.httpclient.request(`${ctx.origin}/-/v1/login/sso/${name}`, { method: 'POST' });
@@ -195,22 +219,24 @@ export class MyAuthAdapter extends AuthAdapter {
 1. 在命令行输入 `npm login --registry=http://127.0.0.1:7001`
 
     ```shell
-    $ npm login --registry=http://127.0.0.1:7001
-    $ npm notice Log in on http://127.0.0.1:7001/
-    $ Login at:
-    $ http://127.0.0.1:7001/hello?name=e44e8c43-211a-4bcd-ae78-c4cbb1a78ae7
-    $ Press ENTER to open in the browser...
+    npm login --registry=http://127.0.0.1:7001
+    npm notice Log in on http://127.0.0.1:7001/
+    Login at:
+    http://127.0.0.1:7001/hello?name=e44e8c43-211a-4bcd-ae78-c4cbb1a78ae7
+    Press ENTER to open in the browser...
     ```
 
 2. 界面提示回车打开浏览器访问登录中心，也就是我们在 getAuthUrl，返回的 loginUrl 配置
 
 3. 由于我们 mock 了对应实现，界面会直接显示登录成功
+
     ```shell
     Logged in on http://127.0.0.1:7001/.
     ```
 
 4. 在命令行输入 `npm whoami --registry=http://127.0.0.1:7001` 验证
+
     ```shell
-    $ npm whoami --registry=http://127.0.0.1:7001
-    $ hello
+    npm whoami --registry=http://127.0.0.1:7001
+    hello
     ```

README.md

@@ -3,14 +3,13 @@
 [![Node.js CI](https://github.com/cnpm/cnpmcore/actions/workflows/nodejs.yml/badge.svg)](https://github.com/cnpm/cnpmcore/actions/workflows/nodejs.yml)
 [![codecov](https://codecov.io/gh/cnpm/cnpmcore/branch/main/graph/badge.svg)](https://codecov.io/gh/cnpm/cnpmcore)
 [![CodeQL](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml)
-[![emoji-log](https://cdn.rawgit.com/ahmadawais/stuff/ca97874/emoji-log/non-flat-round.svg)](https://github.com/ahmadawais/Emoji-Log/)
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore?ref=badge_shield)
 
-Reimplementation based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
+Reimplement based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
 
 ## Registry HTTP API
 
-See https://github.com/cnpm/cnpmjs.org/blob/master/docs/registry-api.md#npm-registry-api
+See [registry-api.md](docs/registry-api.md)
 
 ## How to contribute
 
@@ -24,18 +23,10 @@ See [INTEGRATE.md](INTEGRATE.md)
 
 [MIT](LICENSE)
 
-<!-- GITCONTRIBUTOR_START -->
-
 ## Contributors
 
-|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/32174276?v=4" width="100px;"/><br/><sub><b>semantic-release-bot</b></sub>](https://github.com/semantic-release-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/5574625?v=4" width="100px;"/><br/><sub><b>elrrrrrrr</b></sub>](https://github.com/elrrrrrrr)<br/>|[<img src="https://avatars.githubusercontent.com/u/35598090?v=4" width="100px;"/><br/><sub><b>hezhengxu2018</b></sub>](https://github.com/hezhengxu2018)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|
-| :---: | :---: | :---: | :---: | :---: | :---: |
-|[<img src="https://avatars.githubusercontent.com/u/4635838?v=4" width="100px;"/><br/><sub><b>gemwuu</b></sub>](https://github.com/gemwuu)<br/>|[<img src="https://avatars.githubusercontent.com/u/17879221?v=4" width="100px;"/><br/><sub><b>laibao101</b></sub>](https://github.com/laibao101)<br/>|[<img src="https://avatars.githubusercontent.com/u/3478550?v=4" width="100px;"/><br/><sub><b>coolyuantao</b></sub>](https://github.com/coolyuantao)<br/>|[<img src="https://avatars.githubusercontent.com/u/13284978?v=4" width="100px;"/><br/><sub><b>Beace</b></sub>](https://github.com/Beace)<br/>|[<img src="https://avatars.githubusercontent.com/u/10163680?v=4" width="100px;"/><br/><sub><b>Wellaiyo</b></sub>](https://github.com/Wellaiyo)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/8198408?v=4" width="100px;"/><br/><sub><b>BlackHole1</b></sub>](https://github.com/BlackHole1)<br/>|[<img src="https://avatars.githubusercontent.com/u/1814071?v=4" width="100px;"/><br/><sub><b>xiekw2010</b></sub>](https://github.com/xiekw2010)<br/>|[<img src="https://avatars.githubusercontent.com/u/13471233?v=4" width="100px;"/><br/><sub><b>OpportunityLiu</b></sub>](https://github.com/OpportunityLiu)<br/>|[<img src="https://avatars.githubusercontent.com/u/958063?v=4" width="100px;"/><br/><sub><b>thonatos</b></sub>](https://github.com/thonatos)<br/>|[<img src="https://avatars.githubusercontent.com/u/11039003?v=4" width="100px;"/><br/><sub><b>chenpx976</b></sub>](https://github.com/chenpx976)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|
-[<img src="https://avatars.githubusercontent.com/u/1119126?v=4" width="100px;"/><br/><sub><b>looksgood</b></sub>](https://github.com/looksgood)<br/>|[<img src="https://avatars.githubusercontent.com/u/23701019?v=4" width="100px;"/><br/><sub><b>laoboxie</b></sub>](https://github.com/laoboxie)<br/>|[<img src="https://avatars.githubusercontent.com/u/5550931?v=4" width="100px;"/><br/><sub><b>shinima</b></sub>](https://github.com/shinima)<br/>
+[![Contributors](https://contrib.rocks/image?repo=cnpm/cnpmcore)](https://github.com/cnpm/cnpmcore/graphs/contributors)
 
-This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sat May 06 2023 12:40:20 GMT+0800`.
-
-<!-- GITCONTRIBUTOR_END -->
+Made with [contributors-img](https://contrib.rocks).
 
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore?ref=badge_large)

SECURITY.md

@@ -6,7 +6,7 @@ Currently being supported with security updates.
 
 | Version  | Supported          |
 | -------- | ------------------ |
-| >= 1.0.0 | :white_check_mark: |
+| >= 3.0.0 | :white_check_mark: |
 
 ## Reporting a Vulnerability
 
@@ -35,7 +35,7 @@ When the security team receives a security bug report, they will assign it
 to a primary handler. This person will coordinate the fix and release
 process, involving the following steps:
 
-  * Confirm the problem and determine the affected versions.
-  * Audit code to find any potential similar problems.
-  * Prepare fixes for all releases still under maintenance. These fixes
+* Confirm the problem and determine the affected versions.
+* Audit code to find any potential similar problems.
+* Prepare fixes for all releases still under maintenance. These fixes
     will be released as fast as possible to NPM.

app.ts

@@ -2,6 +2,7 @@ import path from 'path';
 import { readFile } from 'fs/promises';
 import { Application } from 'egg';
 import { ChangesStreamService } from './app/core/service/ChangesStreamService';
+
 declare module 'egg' {
   interface Application {
     binaryHTML: string;

app/common/CryptoUtil.ts

@@ -1,4 +1,5 @@
-import { generateKeyPairSync, publicEncrypt, privateDecrypt, constants } from 'crypto';
+import { generateKeyPairSync } from 'crypto';
+import NodeRSA from 'node-rsa';
 
 // generate rsa key pair
 export function genRSAKeys(): { publicKey: string, privateKey: string } {
@@ -17,17 +18,19 @@ export function genRSAKeys(): { publicKey: string, privateKey: string } {
 }
 
 // encrypt rsa private key
-export function encryptRSA(publicKey: string, data: string): string {
-  return publicEncrypt({
-    key: publicKey,
-    padding: constants.RSA_PKCS1_PADDING,
-  }, Buffer.from(data, 'utf8')).toString('base64');
+export function encryptRSA(publicKey: string, plainText: string): string {
+  const key = new NodeRSA(publicKey, 'pkcs1-public-pem', {
+    encryptionScheme: 'pkcs1',
+    environment: 'browser',
+  });
+  return key.encrypt(plainText, 'base64');
 }
 
 // decrypt rsa private key
-export function decryptRSA(privateKey: string, data: string) {
-  return privateDecrypt({
-    key: privateKey,
-    padding: constants.RSA_PKCS1_PADDING,
-  }, Buffer.from(data, 'base64')).toString('utf8');
+export function decryptRSA(privateKey: string, encryptedBase64: string): string {
+  const key = new NodeRSA(privateKey, 'pkcs1-private-pem', {
+    encryptionScheme: 'pkcs1',
+    environment: 'browser',
+  });
+  return key.decrypt(encryptedBase64, 'utf8');
 }

app/common/EnvUtil.ts

@@ -0,0 +1,40 @@
+export type ValueType = 'string' | 'boolean' | 'number';
+
+export function env(key: string, valueType: ValueType, defaultValue: string): string;
+export function env(key: string, valueType: ValueType, defaultValue: boolean): boolean;
+export function env(key: string, valueType: ValueType, defaultValue: number): number;
+export function env(key: string, valueType: ValueType, defaultValue: string | boolean | number): string | boolean | number {
+  let value = process.env[key];
+  if (typeof value === 'string') {
+    value = value.trim();
+  }
+  if (!value) {
+    return defaultValue;
+  }
+
+  if (valueType === 'string') {
+    return value;
+  }
+
+  if (valueType === 'boolean') {
+    let booleanValue = false;
+    if (value === 'true' || value === '1') {
+      booleanValue = true;
+    } else if (value === 'false' || value === '0') {
+      booleanValue = false;
+    } else {
+      throw new TypeError(`Invalid boolean value: ${value} on process.env.${key}`);
+    }
+    return booleanValue;
+  }
+
+  if (valueType === 'number') {
+    const numberValue = Number(value);
+    if (isNaN(numberValue)) {
+      throw new TypeError(`Invalid number value: ${value} on process.env.${key}`);
+    }
+    return numberValue;
+  }
+
+  throw new TypeError(`Invalid value type: ${valueType}`);
+}

app/common/ErrorUtil.ts

@@ -0,0 +1,27 @@
+const TimeoutErrorNames = [
+  'HttpClientRequestTimeoutError',
+  'HttpClientConnectTimeoutError',
+  'ConnectionError',
+  'ConnectTimeoutError',
+  'BodyTimeoutError',
+  'ResponseTimeoutError',
+];
+
+export function isTimeoutError(err: Error) {
+  if (TimeoutErrorNames.includes(err.name)) {
+    return true;
+  }
+  if (err instanceof AggregateError && err.errors) {
+    for (const subError of err.errors) {
+      if (TimeoutErrorNames.includes(subError.name)) {
+        return true;
+      }
+    }
+  }
+  if ('cause' in err && err.cause instanceof Error) {
+    if (TimeoutErrorNames.includes(err.cause.name)) {
+      return true;
+    }
+  }
+  return false;
+}

app/common/FileUtil.ts

@@ -64,10 +64,13 @@ async function _downloadToTempfile(httpclient: EggContextHttpClient,
   try {
     // max 10 mins to download
     // FIXME: should show download progress
-    const authorization = optionalConfig?.remoteAuthToken ? `Bearer ${optionalConfig?.remoteAuthToken}` : '';
+    const requestHeaders: Record<string, string> = {};
+    if (optionalConfig?.remoteAuthToken) {
+      requestHeaders.authorization = `Bearer ${optionalConfig.remoteAuthToken}`;
+    }
     const { status, headers, res } = await httpclient.request(url, {
       timeout: 60000 * 10,
-      headers: { authorization },
+      headers: requestHeaders,
       writeStream,
       timing: true,
       followRedirect: true,
@@ -105,13 +108,13 @@ const WHITE_FILENAME_CONTENT_TYPES = {
   '.eslintignore': PLAIN_TEXT,
   '.jshintrc': 'application/json',
   '.eslintrc': 'application/json',
-};
+} as const;
 
 export function mimeLookup(filepath: string) {
   const filename = path.basename(filepath).toLowerCase();
   if (filename.endsWith('.ts')) return PLAIN_TEXT;
   if (filename.endsWith('.lock')) return PLAIN_TEXT;
   return mime.lookup(filename) ||
-    WHITE_FILENAME_CONTENT_TYPES[filename] ||
+    WHITE_FILENAME_CONTENT_TYPES[filename as keyof typeof WHITE_FILENAME_CONTENT_TYPES] ||
     DEFAULT_CONTENT_TYPE;
 }

app/common/PackageUtil.ts

@@ -2,7 +2,9 @@ import { createReadStream } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import * as ssri from 'ssri';
-import tar from 'tar';
+import tar from '@fengmk2/tar';
+import { AuthorType, PackageJSONType } from '../repository/PackageRepository';
+
 
 // /@cnpm%2ffoo
 // /@cnpm%2Ffoo
@@ -27,6 +29,10 @@ export function cleanUserPrefix(username: string): string {
   return username.replace(/^.*:/, '');
 }
 
+export function getPrefixedName(prefix: string, username: string): string {
+  return prefix ? `${prefix}${username}` : username;
+}
+
 export async function calculateIntegrity(contentOrFile: Uint8Array | string) {
   let integrityObj;
   if (typeof contentOrFile === 'string') {
@@ -98,3 +104,37 @@ export async function hasShrinkWrapInTgz(contentOrFile: Uint8Array | string): Pr
     throw Object.assign(new Error('[hasShrinkWrapInTgz] Fail to parse input file'), { cause: e });
   }
 }
+
+/** 写入 ES 时，格式化 author */
+export function formatAuthor(author: string | AuthorType | undefined): AuthorType | undefined {
+  if (author === undefined) {
+    return author;
+  }
+
+  if (typeof author === 'string') {
+    return { name: author };
+  }
+
+  return author;
+}
+
+export async function extractPackageJSON(tarballBytes: Buffer): Promise<PackageJSONType> {
+  return new Promise((resolve, reject) => {
+    Readable.from(tarballBytes)
+      .pipe(tar.t({
+        filter: name => name === 'package/package.json',
+        onentry: async entry => {
+          const chunks: Buffer[] = [];
+          for await (const chunk of entry) {
+            chunks.push(chunk);
+          }
+          try {
+            const data = Buffer.concat(chunks);
+            return resolve(JSON.parse(data.toString()));
+          } catch (err) {
+            reject(new Error('Error parsing package.json'));
+          }
+        },
+      }));
+  });
+}

app/common/UserUtil.ts

@@ -34,7 +34,7 @@ export function integrity(plain: string): string {
 }
 
 export function checkIntegrity(plain: string, expectedIntegrity: string): boolean {
-  return ssri.checkData(plain, expectedIntegrity);
+  return !!ssri.checkData(plain, expectedIntegrity);
 }
 
 export function sha512(plain: string): string {

app/common/adapter/CacheAdapter.ts

@@ -64,12 +64,13 @@ export class CacheAdapter {
 
   async usingLock(key: string, seconds: number, func: () => Promise<void>) {
     const lockTimestamp = await this.lock(key, seconds);
-    if (!lockTimestamp) return;
+    if (!lockTimestamp) return false;
     try {
       await func();
     } finally {
       await this.unlock(key, lockTimestamp);
     }
+    return true;
   }
 
   private getLockName(key: string) {

app/common/adapter/NPMRegistry.ts

@@ -11,6 +11,8 @@ import {
   HttpClientRequestOptions,
   HttpClientResponse,
 } from 'egg';
+import { PackageManifestType } from '../../repository/PackageRepository';
+import { isTimeoutError } from '../ErrorUtil';
 
 type HttpMethod = HttpClientRequestOptions['method'];
 
@@ -40,7 +42,7 @@ export class NPMRegistry {
     this.registryHost = registryHost;
   }
 
-  public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise<RegistryResponse> {
+  public async getFullManifests(fullname: string, optionalConfig?: { retries?: number, remoteAuthToken?: string }): Promise<{ method: HttpMethod } & HttpClientResponse<PackageManifestType>> {
     let retries = optionalConfig?.retries || 3;
     // set query t=timestamp, make sure CDN cache disable
     // cache=0 is sync worker request flag
@@ -51,9 +53,14 @@ export class NPMRegistry {
         // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons
         // https://r.cnpmjs.org/intraactive-sdk-ui 44s
         const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken);
-        return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } });
+        return await this.request('GET', url, undefined, {
+          timeout: 120000,
+          headers: { authorization },
+        });
       } catch (err: any) {
-        if (err.name === 'ResponseTimeoutError') throw err;
+        if (isTimeoutError(err)) {
+          throw err;
+        }
         lastError = err;
       }
       retries--;
@@ -97,6 +104,7 @@ export class NPMRegistry {
       data: params,
       dataType: 'json',
       timing: true,
+      retry: 3,
       timeout: this.timeout,
       followRedirect: true,
       gzip: true,
@@ -109,7 +117,7 @@ export class NPMRegistry {
     };
   }
 
-  private genAuthorizationHeader(remoteAuthToken?:string) {
+  public genAuthorizationHeader(remoteAuthToken?:string) {
     return remoteAuthToken ? `Bearer ${remoteAuthToken}` : '';
   }
 }

app/common/adapter/binary/AbstractBinary.ts

@@ -17,6 +17,8 @@ export type FetchResult = {
   nextParams?: any;
 };
 
+const platforms = [ 'darwin', 'linux', 'win32' ] as const;
+
 export const BINARY_ADAPTER_ATTRIBUTE = Symbol('BINARY_ADAPTER_ATTRIBUTE');
 
 export abstract class AbstractBinary {
@@ -26,12 +28,17 @@ export abstract class AbstractBinary {
   @Inject()
   protected httpclient: EggHttpClient;
 
-  abstract init(binaryName: BinaryName): Promise<void>;
+  abstract initFetch(binaryName: BinaryName): Promise<void>;
   abstract fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined>;
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async finishFetch(_success: boolean, _binaryName: BinaryName): Promise<void> {
+    // do not thing by default
+  }
+
   protected async requestXml(url: string) {
     const { status, data, headers } = await this.httpclient.request(url, {
-      timeout: 20000,
+      timeout: 30000,
       followRedirect: true,
       gzip: true,
     });
@@ -43,12 +50,13 @@ export abstract class AbstractBinary {
     return xml;
   }
 
-  protected async requestJSON(url: string) {
+  protected async requestJSON(url: string, requestHeaders?: Record<string, string>) {
     const { status, data, headers } = await this.httpclient.request(url, {
-      timeout: 20000,
+      timeout: 30000,
       dataType: 'json',
       followRedirect: true,
       gzip: true,
+      headers: requestHeaders,
     });
     if (status !== 200) {
       this.logger.warn('[AbstractBinary.requestJSON:non-200-status] url: %s, status: %s, headers: %j', url, status, headers);
@@ -64,7 +72,7 @@ export abstract class AbstractBinary {
     for (const version of versions) {
       if (!version.modules) continue;
       const modulesVersion = parseInt(version.modules);
-      // node v6.0.0 moduels 48 min
+      // node v6.0.0 modules 48 min
       if (modulesVersion >= 48 && !nodeABIVersions.includes(modulesVersion)) {
         nodeABIVersions.push(modulesVersion);
       }
@@ -74,7 +82,7 @@ export abstract class AbstractBinary {
 
   protected listNodePlatforms() {
     // https://nodejs.org/api/os.html#osplatform
-    return [ 'darwin', 'linux', 'win32' ];
+    return platforms;
   }
 
   protected listNodeArchs(binaryConfig?: BinaryTaskConfig) {
@@ -87,11 +95,11 @@ export abstract class AbstractBinary {
     };
   }
 
-  protected listNodeLibcs() {
+  protected listNodeLibcs(): Record<typeof platforms[number], string[]> {
     // https://github.com/lovell/detect-libc/blob/master/lib/detect-libc.js#L42
     return {
-      linux: [ 'glibc', 'musl' ],
       darwin: [ 'unknown' ],
+      linux: [ 'glibc', 'musl' ],
       win32: [ 'unknown' ],
     };
   }

app/common/adapter/binary/ApiBinary.ts

@@ -9,7 +9,7 @@ export class ApiBinary extends AbstractBinary {
   @Inject()
   private readonly config: EggAppConfig;
 
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/BucketBinary.ts

@@ -7,7 +7,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Bucket)
 export class BucketBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/ChromeForTestingBinary.ts

@@ -1,3 +1,4 @@
+import { basename } from 'path';
 import { SingletonProto } from '@eggjs/tegg';
 import { BinaryType } from '../../enum/Binary';
 import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './AbstractBinary';
@@ -5,65 +6,151 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.ChromeForTesting)
 export class ChromeForTestingBinary extends AbstractBinary {
+  static lastTimestamp = '';
+  #timestamp = '';
+
   private dirItems?: {
     [key: string]: BinaryItem[];
   };
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 
-  async fetch(dir: string): Promise<FetchResult | undefined> {
-    if (!this.dirItems) {
-      this.dirItems = {};
-      this.dirItems['/'] = [];
-      let chromeVersion = '';
+  async finishFetch(success: boolean) {
+    if (success && this.#timestamp && ChromeForTestingBinary.lastTimestamp !== this.#timestamp) {
+      ChromeForTestingBinary.lastTimestamp = this.#timestamp;
+    }
+  }
 
-      // exports.PUPPETEER_REVISIONS = Object.freeze({
-      //   chrome: '113.0.5672.63',
-      //   firefox: 'latest',
-      // });
-      const unpkgURL = 'https://unpkg.com/puppeteer-core@latest/lib/cjs/puppeteer/revisions.js';
-      const text = await this.requestXml(unpkgURL);
-      const m = /chrome:\s+\'([\d\.]+)\'\,/.exec(text);
-      if (m) {
-        chromeVersion = m[1];
-      }
+  async #syncDirItems() {
+    this.dirItems = {};
+    this.dirItems['/'] = [];
+    const jsonApiEndpoint = 'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
+    const { data, status, headers } = await this.httpclient.request(jsonApiEndpoint, {
+      dataType: 'json',
+      timeout: 30000,
+      followRedirect: true,
+      gzip: true,
+    });
+    if (status !== 200) {
+      this.logger.warn('[ChromeForTestingBinary.request:non-200-status] url: %s, status: %s, headers: %j, data: %j',
+        jsonApiEndpoint, status, headers, data);
+      return;
+    }
+    this.#timestamp = data.timestamp;
+    const hasNewData = this.#timestamp !== ChromeForTestingBinary.lastTimestamp;
+    this.logger.info('[ChromeForTestingBinary] remote data timestamp: %j, last timestamp: %j, hasNewData: %s',
+      this.#timestamp, ChromeForTestingBinary.lastTimestamp, hasNewData);
+    if (!hasNewData) {
+      return;
+    }
 
-      const platforms = [ 'linux64', 'mac-arm64', 'mac-x64', 'win32', 'win64' ];
-      const date = new Date().toISOString();
+    this.dirItems['/'].push({
+      name: 'known-good-versions-with-downloads.json',
+      date: data.timestamp,
+      size: '-',
+      isDir: false,
+      url: jsonApiEndpoint,
+    });
+    this.dirItems['/'].push({
+      name: 'latest-patch-versions-per-build.json',
+      date: data.timestamp,
+      size: '-',
+      isDir: false,
+      url: 'https://googlechromelabs.github.io/chrome-for-testing/latest-patch-versions-per-build.json',
+    });
+    this.dirItems['/'].push({
+      name: 'last-known-good-versions.json',
+      date: data.timestamp,
+      size: '-',
+      isDir: false,
+      url: 'https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json',
+    });
+
+    // "timestamp": "2023-09-16T00:21:21.964Z",
+    // "versions": [
+    //   {
+    //     "version": "113.0.5672.0",
+    //     "revision": "1121455",
+    //     "downloads": {
+    //       "chrome": [
+    //         {
+    //           "platform": "linux64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-arm64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-arm64/chrome-mac-arm64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-x64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-x64/chrome-mac-x64.zip"
+    //         },
+    //         {
+    //           "platform": "win32",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win32/chrome-win32.zip"
+    //         },
+    //         {
+    //           "platform": "win64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win64/chrome-win64.zip"
+    //         }
+    //       ]
+    //     }
+    //   },
+    const versions = data.versions as {
+      version: string;
+      revision: string;
+      downloads: {
+        [key: string]: {
+          platform: string;
+          url: string;
+        }[];
+      };
+    }[];
+    for (const item of versions) {
       this.dirItems['/'].push({
-        name: `${chromeVersion}/`,
-        date,
+        name: `${item.version}/`,
+        date: item.revision,
         size: '-',
         isDir: true,
         url: '',
       });
-      this.dirItems[`/${chromeVersion}/`] = [];
-
-      for (const platform of platforms) {
-        this.dirItems[`/${chromeVersion}/`].push({
-          name: `${platform}/`,
-          date,
-          size: '-',
-          isDir: true,
-          url: '',
-        });
-
-        // https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.63/mac-arm64/chrome-mac-arm64.zip
-        const name = `chrome-${platform}.zip`;
-        this.dirItems[`/${chromeVersion}/${platform}/`] = [
-          {
-            name,
-            date,
+      const versionDir = `/${item.version}/`;
+      if (!this.dirItems[versionDir]) {
+        this.dirItems[versionDir] = [];
+      }
+      for (const category in item.downloads) {
+        const downloads = item.downloads[category];
+        for (const download of downloads) {
+          const platformDir = `${versionDir}${download.platform}/`;
+          if (!this.dirItems[platformDir]) {
+            this.dirItems[platformDir] = [];
+            this.dirItems[versionDir].push({
+              name: `${download.platform}/`,
+              date: item.revision,
+              size: '-',
+              isDir: true,
+              url: '',
+            });
+          }
+          this.dirItems[platformDir].push({
+            name: basename(download.url),
+            date: data.timestamp,
             size: '-',
             isDir: false,
-            url: `https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${chromeVersion}/${platform}/${name}`,
-          },
-        ];
+            url: download.url,
+          });
+        }
       }
     }
+  }
 
-    return { items: this.dirItems[dir], nextParams: null };
+  async fetch(dir: string): Promise<FetchResult | undefined> {
+    // use https://github.com/GoogleChromeLabs/chrome-for-testing#json-api-endpoints
+    if (!this.dirItems) {
+      await this.#syncDirItems();
+    }
+    return { items: this.dirItems![dir], nextParams: null };
   }
 }

app/common/adapter/binary/CypressBinary.ts

@@ -9,7 +9,7 @@ export class CypressBinary extends AbstractBinary {
     [key: string]: BinaryItem[];
   } | null;
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 
@@ -42,8 +42,21 @@ export class CypressBinary extends AbstractBinary {
         // "https://cdn.cypress.io/desktop/9.2.0/darwin-arm64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/darwin-x64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/linux-x64/cypress.zip"
+        // "https://cdn.cypress.io/desktop/9.2.0/linux-arm64/cypress.zip"
         // "https://cdn.cypress.io/desktop/9.2.0/win32-x64/cypress.zip"
-        const platforms = [ 'darwin-x64', 'darwin-arm64', 'linux-x64', 'win32-x64' ];
+        // https://github.com/cypress-io/cypress/blob/develop/scripts/binary/index.js#L146
+        // const systems = [
+        //   { platform: 'linux', arch: 'x64' },
+        //   { platform: 'linux', arch: 'arm64' },
+        //   { platform: 'darwin', arch: 'x64' },
+        //   { platform: 'darwin', arch: 'arm64' },
+        //   { platform: 'win32', arch: 'x64' },
+        // ]
+        const platforms = [
+          'darwin-x64', 'darwin-arm64',
+          'linux-x64', 'linux-arm64',
+          'win32-x64',
+        ];
         for (const platform of platforms) {
           this.dirItems[subDir].push({
             name: `${platform}/`,

app/common/adapter/binary/EdgedriverBinary.ts

@@ -0,0 +1,213 @@
+import path from 'node:path';
+import { SingletonProto } from '@eggjs/tegg';
+import {
+  AbstractBinary, FetchResult, BinaryItem, BinaryAdapter,
+} from './AbstractBinary';
+import { BinaryType } from '../../enum/Binary';
+
+@SingletonProto()
+@BinaryAdapter(BinaryType.Edgedriver)
+export class EdgedriverBinary extends AbstractBinary {
+  private dirItems?: {
+    [key: string]: BinaryItem[];
+  };
+
+  async initFetch() {
+    this.dirItems = undefined;
+  }
+
+  async #syncDirItems() {
+    this.dirItems = {};
+    this.dirItems['/'] = [];
+    const jsonApiEndpoint = 'https://edgeupdates.microsoft.com/api/products';
+    const { data, status, headers } = await this.httpclient.request(jsonApiEndpoint, {
+      dataType: 'json',
+      timeout: 30000,
+      followRedirect: true,
+      gzip: true,
+    });
+    if (status !== 200) {
+      this.logger.warn('[EdgedriverBinary.request:non-200-status] url: %s, status: %s, headers: %j, data: %j',
+        jsonApiEndpoint, status, headers, data);
+      return;
+    }
+    this.logger.info('[EdgedriverBinary] remote data length: %s', data.length);
+    // [
+    //   {
+    //     "Product": "Stable",
+    //     "Releases": [
+    //       {
+    //         "ReleaseId": 73376,
+    //         "Platform": "iOS",
+    //         "Architecture": "arm64",
+    //         "CVEs": [],
+    //         "ProductVersion": "124.0.2478.89",
+    //         "Artifacts": [],
+    //         "PublishedTime": "2024-05-07T02:57:00",
+    //         "ExpectedExpiryDate": "2025-05-07T02:57:00"
+    //       },
+    //       {
+    //         "ReleaseId": 73629,
+    //         "Platform": "Windows",
+    //         "Architecture": "x86",
+    //         "CVEs": [
+    //           "CVE-2024-4559",
+    //           "CVE-2024-4671"
+    //         ],
+    //         "ProductVersion": "124.0.2478.97",
+    //         "Artifacts": [
+    //           {
+    //             "ArtifactName": "msi",
+    //             "Location": "https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/aa1c9fe3-bb9c-4a80-9ff7-5c109701fbfe/MicrosoftEdgeEnterpriseX86.msi",
+    //             "Hash": "4CEF7B907D3E2371E953C41190E32C3560CEE7D3F16D7550CA156DC976EBCB80",
+    //             "HashAlgorithm": "SHA256",
+    //             "SizeInBytes": 162029568
+    //           }
+    //         ],
+    //         "PublishedTime": "2024-05-11T06:47:00",
+    //         "ExpectedExpiryDate": "2025-05-10T16:59:00"
+    //       },
+    //       {
+    //         "ReleaseId": 73630,
+    //         "Platform": "Linux",
+    //         "Architecture": "x64",
+    //         "CVEs": [
+    //           "CVE-2024-4559"
+    //         ],
+    //         "ProductVersion": "124.0.2478.97",
+    //         "Artifacts": [
+    //           {
+    //             "ArtifactName": "rpm",
+    //             "Location": "https://packages.microsoft.com/yumrepos/edge/microsoft-edge-stable-124.0.2478.97-1.x86_64.rpm",
+    //             "Hash": "32D9C333544DDD9C56FED54844E89EF00F3E5620942C07B9B68D214016687895",
+    //             "HashAlgorithm": "SHA256",
+    //             "SizeInBytes": 169877932
+    //           },
+    //           {
+    //             "ArtifactName": "deb",
+    //             "Location": "https://packages.microsoft.com/repos/edge/pool/main/m/microsoft-edge-stable/microsoft-edge-stable_124.0.2478.97-1_amd64.deb",
+    //             "Hash": "85D0AD1D63847B3DD54F0F214D18A2B54462BB43291536E773AD1B8B29BBF799",
+    //             "HashAlgorithm": "SHA256",
+    //             "SizeInBytes": 167546042
+    //           }
+    //         ],
+    //         "PublishedTime": "2024-05-10T17:01:00",
+    //         "ExpectedExpiryDate": "2025-05-10T17:01:00"
+    //       },
+    // {
+    //   "Product": "EdgeUpdate",
+    //   "Releases": [
+    //     {
+    //       "ReleaseId": 73493,
+    //       "Platform": "Windows",
+    //       "Architecture": "x86",
+    //       "CVEs": [],
+    //       "ProductVersion": "1.3.187.37",
+    //       "Artifacts": [
+    //         {
+    //           "ArtifactName": "exe",
+    //           "Location": "https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/a2fa84fe-796b-4f80-b1cd-f4d1f5731aa8/MicrosoftEdgeUpdateSetup_X86_1.3.187.37.exe",
+    //           "Hash": "503088D22461FEE5D7B6B011609D73FFD5869D3ACE1DBB0F00F8F3B9D122C514",
+    //           "HashAlgorithm": "SHA256",
+    //           "SizeInBytes": 1622072
+    //         }
+    //       ],
+    //       "PublishedTime": "2024-05-08T05:44:00",
+    //       "ExpectedExpiryDate": "2025-05-08T05:44:00"
+    //     }
+    //   ]
+    // }
+    const products = data as {
+      Product: string;
+      Releases: {
+        ReleaseId: number;
+        Platform: string;
+        Architecture: string;
+        CVEs: string[];
+        ProductVersion: string;
+        Artifacts: {
+          ArtifactName: string;
+          Location: string;
+          Hash: string;
+          HashAlgorithm: string;
+          SizeInBytes: string;
+        }[];
+        PublishedTime: string;
+        ExpectedExpiryDate: string;
+      }[];
+    }[];
+    const existsVersions = new Set<string>();
+    for (const product of products) {
+      if (product.Product === 'EdgeUpdate') continue;
+      for (const release of product.Releases) {
+        if (!release.Artifacts || release.Artifacts.length === 0) continue;
+        if (existsVersions.has(release.ProductVersion)) continue;
+        this.dirItems['/'].push({
+          name: `${release.ProductVersion}/`,
+          date: release.PublishedTime,
+          size: '-',
+          isDir: true,
+          url: '',
+        });
+        existsVersions.add(release.ProductVersion);
+      }
+    }
+  }
+
+  async fetch(dir: string): Promise<FetchResult | undefined> {
+    if (!this.dirItems) {
+      await this.#syncDirItems();
+    }
+    // fetch root dir
+    if (dir === '/') {
+      return { items: this.dirItems![dir], nextParams: null };
+    }
+
+    // fetch sub dir
+    // /foo/ => foo/
+    const subDir = dir.substring(1);
+    // https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver?prefix=124.0.2478.97/&delimiter=/&maxresults=100&restype=container&comp=list
+    const url = `https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver?prefix=${encodeURIComponent(subDir)}&delimiter=/&maxresults=100&restype=container&comp=list`;
+    const xml = await this.requestXml(url);
+    return { items: this.#parseItems(xml), nextParams: null };
+  }
+
+  #parseItems(xml: string): BinaryItem[] {
+    const items: BinaryItem[] = [];
+    // <Blob><Name>124.0.2478.97/edgedriver_arm64.zip</Name><Url>https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/124.0.2478.97/edgedriver_arm64.zip</Url><Properties><Last-Modified>Fri, 10 May 2024 18:35:44 GMT</Last-Modified><Etag>0x8DC712000713C13</Etag><Content-Length>9191362</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding /><Content-Language /><Content-MD5>1tjPTf5JU6KKB06Qf1JOGw==</Content-MD5><Cache-Control /><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus></Properties></Blob>
+    const fileRe = /<Blob><Name>([^<]+?)<\/Name><Url>([^<]+?)<\/Url><Properties><Last\-Modified>([^<]+?)<\/Last\-Modified><Etag>(?:[^<]+?)<\/Etag><Content\-Length>(\d+)<\/Content\-Length>/g;
+    const matchItems = xml.matchAll(fileRe);
+    for (const m of matchItems) {
+      const fullname = m[1].trim();
+      // <Blob>
+      //   <Name>124.0.2478.97/edgedriver_arm64.zip</Name>
+      //   <Url>https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/124.0.2478.97/edgedriver_arm64.zip</Url>
+      //   <Properties>
+      //     <Last-Modified>Fri, 10 May 2024 18:35:44 GMT</Last-Modified>
+      //     <Etag>0x8DC712000713C13</Etag>
+      //     <Content-Length>9191362</Content-Length>
+      //     <Content-Type>application/octet-stream</Content-Type>
+      //     <Content-Encoding/>
+      //     <Content-Language/>
+      //     <Content-MD5>1tjPTf5JU6KKB06Qf1JOGw==</Content-MD5>
+      //     <Cache-Control/>
+      //     <BlobType>BlockBlob</BlobType>
+      //     <LeaseStatus>unlocked</LeaseStatus>
+      //   </Properties>
+      // </Blob>
+      // ignore size = 0 dir
+      const name = path.basename(fullname);
+      const url = m[2].trim();
+      const date = m[3].trim();
+      const size = parseInt(m[4].trim());
+      items.push({
+        name,
+        isDir: false,
+        url,
+        size,
+        date,
+      });
+    }
+    return items;
+  }
+}

app/common/adapter/binary/GithubBinary.ts

@@ -8,7 +8,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 export class GithubBinary extends AbstractBinary {
   private releases: Record<string, any[]> = {};
 
-  async init(binaryName: BinaryName) {
+  async initFetch(binaryName: BinaryName) {
     delete this.releases[binaryName];
   }
 
@@ -21,7 +21,11 @@ export class GithubBinary extends AbstractBinary {
       const maxPage = binaryConfig.options?.maxPage || 1;
       for (let i = 0; i < maxPage; i++) {
         const url = `https://api.github.com/repos/${binaryConfig.repo}/releases?per_page=100&page=${i + 1}`;
-        const data = await this.requestJSON(url);
+        const requestHeaders: Record<string, string> = {};
+        if (process.env.GITHUB_TOKEN) {
+          requestHeaders.Authorization = `token ${process.env.GITHUB_TOKEN}`;
+        }
+        const data = await this.requestJSON(url, requestHeaders);
         if (!Array.isArray(data)) {
           // {"message":"API rate limit exceeded for 47.57.239.54. (But here's the good news: Authenticated requests get a higher rate limit. Check out the documentation for more details.)","documentation_url":"https://docs.github.com/rest/overview/resources-in-the-rest-api#rate-limiting"}
           if (typeof data?.message === 'string' && data.message.includes('rate limit')) {
@@ -40,10 +44,13 @@ export class GithubBinary extends AbstractBinary {
 
   protected formatItems(releaseItem: any, binaryConfig: BinaryTaskConfig) {
     const items: BinaryItem[] = [];
-    // 200MB
-    const maxFileSize = 1024 * 1024 * 200;
+    // 250MB
+    const maxFileSize = 1024 * 1024 * 250;
     for (const asset of releaseItem.assets) {
-      if (asset.size > maxFileSize) continue;
+      if (asset.size > maxFileSize) {
+        this.logger.info('[GithubBinary.formatItems] asset reach max file size(> 250MB), ignore download it, asset: %j', asset);
+        continue;
+      }
       items.push({
         name: asset.name,
         isDir: false,

app/common/adapter/binary/ImageminBinary.ts

@@ -6,7 +6,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Imagemin)
 export class ImageminBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/NodeBinary.ts

@@ -1,3 +1,4 @@
+import { basename } from 'node:path';
 import { SingletonProto } from '@eggjs/tegg';
 import { BinaryType } from '../../enum/Binary';
 import binaries, { BinaryName } from '../../../../config/binaries';
@@ -6,7 +7,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Node)
 export class NodeBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }
@@ -21,26 +22,39 @@ export class NodeBinary extends AbstractBinary {
     // <a href="index.tab">index.tab</a>                                          17-Dec-2021 23:16              136319
     // <a href="node-0.0.1.tar.gz">node-0.0.1.tar.gz</a>                                  26-Aug-2011 16:22             2846972
     // <a href="node-v14.0.0-nightly20200119b318926634-linux-armv7l.tar.xz">node-v14.0.0-nightly20200119b318926634-linux-ar..&gt;</a> 19-Jan-2020 06:07            18565976
-    const re = /<a href="([^\"]+?)"[^>]*?>[^<]+?<\/a>\s+?([\w\-]+? \w{2}\:\d{2})\s+?(\d+|\-)/ig;
+
+    // new html format
+    //     <a href="docs/">docs/</a>                                                             -                   -
+    // <a href="win-x64/">win-x64/</a>                                                          -                   -
+    // <a href="win-x86/">win-x86/</a>                                                          -                   -
+    // <a href="/dist/v18.15.0/SHASUMS256.txt.asc">SHASUMS256.txt.asc</a>                                 04-Nov-2024 17:29               3.7 KB
+    // <a href="/dist/v18.15.0/SHASUMS256.txt.sig">SHASUMS256.txt.sig</a>                                 04-Nov-2024 17:29                310 B
+    // <a href="/dist/v18.15.0/SHASUMS256.txt">SHASUMS256.txt</a>                                     04-Nov-2024 17:29               3.2 KB
+    const re = /<a href="([^\"]+?)"[^>]*?>[^<]+?<\/a>\s+?((?:[\w\-]+? \w{2}\:\d{2})|\-)\s+?([\d\.\-\s\w]+)/ig;
     const matchs = html.matchAll(re);
     const items: BinaryItem[] = [];
     for (const m of matchs) {
-      const name = m[1];
+      let name = m[1];
       const isDir = name.endsWith('/');
+      if (!isDir) {
+        // /dist/v18.15.0/SHASUMS256.txt => SHASUMS256.txt
+        name = basename(name);
+      }
       const fileUrl = isDir ? '' : `${url}${name}`;
       const date = m[2];
-      const size = m[3];
+      const size = m[3].trim();
       if (size === '0') continue;
       if (binaryConfig.ignoreFiles?.includes(`${dir}${name}`)) continue;
 
-      items.push({
+      const item = {
         name,
         isDir,
         url: fileUrl,
         size,
         date,
         ignoreDownloadStatuses: binaryConfig.options?.ignoreDownloadStatuses,
-      });
+      };
+      items.push(item);
     }
     return { items, nextParams: null };
   }

app/common/adapter/binary/NodePreGypBinary.ts

@@ -7,7 +7,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.NodePreGyp)
 export class NodePreGypBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }
@@ -15,7 +15,8 @@ export class NodePreGypBinary extends AbstractBinary {
   // https://github.com/mapbox/node-pre-gyp
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     const binaryConfig = binaries[binaryName];
-    const pkgUrl = `https://registry.npmjs.com/${binaryName}`;
+    const npmPackageName = binaryConfig.options?.npmPackageName ?? binaryName;
+    const pkgUrl = `https://registry.npmjs.com/${npmPackageName}`;
     const data = await this.requestJSON(pkgUrl);
     const dirItems: {
       [key: string]: BinaryItem[];

app/common/adapter/binary/PlaywrightBinary.ts

@@ -13,14 +13,18 @@ const DOWNLOAD_HOST = 'https://playwright.azureedge.net/';
 const DOWNLOAD_PATHS = {
   'chromium': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/chromium/%s/chromium-linux.zip',
-    'generic-linux-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
-    'ubuntu18.04': 'builds/chromium/%s/chromium-linux.zip',
-    'ubuntu20.04': 'builds/chromium/%s/chromium-linux.zip',
-    'ubuntu22.04': 'builds/chromium/%s/chromium-linux.zip',
-    'ubuntu18.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/chromium/%s/chromium-linux.zip',
+    'ubuntu22.04-x64': 'builds/chromium/%s/chromium-linux.zip',
+    'ubuntu24.04-x64': 'builds/chromium/%s/chromium-linux.zip',
+    'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'debian11-x64': 'builds/chromium/%s/chromium-linux.zip',
+    'debian11-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'debian12-x64': 'builds/chromium/%s/chromium-linux.zip',
+    'debian12-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
     'mac10.13': 'builds/chromium/%s/chromium-mac.zip',
     'mac10.14': 'builds/chromium/%s/chromium-mac.zip',
     'mac10.15': 'builds/chromium/%s/chromium-mac.zip',
@@ -28,18 +32,57 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
     'mac12': 'builds/chromium/%s/chromium-mac.zip',
     'mac12-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
+    'mac13': 'builds/chromium/%s/chromium-mac.zip',
+    'mac13-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
+    'mac14': 'builds/chromium/%s/chromium-mac.zip',
+    'mac14-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
+    'mac15': 'builds/chromium/%s/chromium-mac.zip',
+    'mac15-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
     'win64': 'builds/chromium/%s/chromium-win64.zip',
   },
+  'chromium-headless-shell': {
+    '<unknown>': undefined,
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/chromium/%s/chromium-headless-shell-linux.zip',
+    'ubuntu22.04-x64': 'builds/chromium/%s/chromium-headless-shell-linux.zip',
+    'ubuntu24.04-x64': 'builds/chromium/%s/chromium-headless-shell-linux.zip',
+    'ubuntu18.04-arm64': undefined,
+    'ubuntu20.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip',
+    'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip',
+    'debian11-x64': 'builds/chromium/%s/chromium-headless-shell-linux.zip',
+    'debian11-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip',
+    'debian12-x64': 'builds/chromium/%s/chromium-headless-shell-linux.zip',
+    'debian12-arm64': 'builds/chromium/%s/chromium-headless-shell-linux-arm64.zip',
+    'mac10.13': undefined,
+    'mac10.14': undefined,
+    'mac10.15': undefined,
+    'mac11': 'builds/chromium/%s/chromium-headless-shell-mac.zip',
+    'mac11-arm64': 'builds/chromium/%s/chromium-headless-shell-mac-arm64.zip',
+    'mac12': 'builds/chromium/%s/chromium-headless-shell-mac.zip',
+    'mac12-arm64': 'builds/chromium/%s/chromium-headless-shell-mac-arm64.zip',
+    'mac13': 'builds/chromium/%s/chromium-headless-shell-mac.zip',
+    'mac13-arm64': 'builds/chromium/%s/chromium-headless-shell-mac-arm64.zip',
+    'mac14': 'builds/chromium/%s/chromium-headless-shell-mac.zip',
+    'mac14-arm64': 'builds/chromium/%s/chromium-headless-shell-mac-arm64.zip',
+    'mac15': 'builds/chromium/%s/chromium-headless-shell-mac.zip',
+    'mac15-arm64': 'builds/chromium/%s/chromium-headless-shell-mac-arm64.zip',
+    'win64': 'builds/chromium/%s/chromium-headless-shell-win64.zip',
+  },
   'chromium-tip-of-tree': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
-    'generic-linux-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
-    'ubuntu18.04': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
-    'ubuntu20.04': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
-    'ubuntu22.04': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
-    'ubuntu18.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'ubuntu22.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'ubuntu24.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'debian11-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'debian11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'debian12-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'debian12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
     'mac10.13': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac10.14': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac10.15': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
@@ -47,94 +90,144 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
     'mac12': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
+    'mac13': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
+    'mac13-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
+    'mac14': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
+    'mac14-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
+    'mac15': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
+    'mac15-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
     'win64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-win64.zip',
   },
-  'chromium-with-symbols': {
+  'chromium-tip-of-tree-headless-shell': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
-    'generic-linux-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
-    'ubuntu18.04': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
-    'ubuntu20.04': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
-    'ubuntu22.04': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
-    'ubuntu18.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
-    'ubuntu20.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
-    'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
-    'mac10.13': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
-    'mac10.14': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
-    'mac10.15': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
-    'mac11': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
-    'mac11-arm64': 'builds/chromium/%s/chromium-with-symbols-mac-arm64.zip',
-    'mac12': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
-    'mac12-arm64': 'builds/chromium/%s/chromium-with-symbols-mac-arm64.zip',
-    'win64': 'builds/chromium/%s/chromium-with-symbols-win64.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux.zip',
+    'ubuntu22.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux.zip',
+    'ubuntu24.04-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux.zip',
+    'ubuntu18.04-arm64': undefined,
+    'ubuntu20.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip',
+    'ubuntu22.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip',
+    'debian11-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux.zip',
+    'debian11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip',
+    'debian12-x64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux.zip',
+    'debian12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-linux-arm64.zip',
+    'mac10.13': undefined,
+    'mac10.14': undefined,
+    'mac10.15': undefined,
+    'mac11': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac.zip',
+    'mac11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac-arm64.zip',
+    'mac12': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac.zip',
+    'mac12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac-arm64.zip',
+    'mac13': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac.zip',
+    'mac13-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac-arm64.zip',
+    'mac14': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac.zip',
+    'mac14-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac-arm64.zip',
+    'mac15': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac.zip',
+    'mac15-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-mac-arm64.zip',
+    'win64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-headless-shell-win64.zip',
   },
   'firefox': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/firefox/%s/firefox-ubuntu-20.04.zip',
-    'generic-linux-arm64': 'builds/firefox/%s/firefox-ubuntu-20.04-arm64.zip',
-    'ubuntu18.04': 'builds/firefox/%s/firefox-ubuntu-18.04.zip',
-    'ubuntu20.04': 'builds/firefox/%s/firefox-ubuntu-20.04.zip',
-    'ubuntu22.04': 'builds/firefox/%s/firefox-ubuntu-22.04.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/firefox/%s/firefox-ubuntu-20.04.zip',
+    'ubuntu22.04-x64': 'builds/firefox/%s/firefox-ubuntu-22.04.zip',
+    'ubuntu24.04-x64': 'builds/firefox/%s/firefox-ubuntu-24.04.zip',
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/firefox/%s/firefox-ubuntu-20.04-arm64.zip',
     'ubuntu22.04-arm64': 'builds/firefox/%s/firefox-ubuntu-22.04-arm64.zip',
-    'mac10.13': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac10.14': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac10.15': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac11': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac11-arm64': 'builds/firefox/%s/firefox-mac-11-arm64.zip',
-    'mac12': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac12-arm64': 'builds/firefox/%s/firefox-mac-11-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/firefox/%s/firefox-ubuntu-24.04-arm64.zip',
+    'debian11-x64': 'builds/firefox/%s/firefox-debian-11.zip',
+    'debian11-arm64': 'builds/firefox/%s/firefox-debian-11-arm64.zip',
+    'debian12-x64': 'builds/firefox/%s/firefox-debian-12.zip',
+    'debian12-arm64': 'builds/firefox/%s/firefox-debian-12-arm64.zip',
+    'mac10.13': 'builds/firefox/%s/firefox-mac.zip',
+    'mac10.14': 'builds/firefox/%s/firefox-mac.zip',
+    'mac10.15': 'builds/firefox/%s/firefox-mac.zip',
+    'mac11': 'builds/firefox/%s/firefox-mac.zip',
+    'mac11-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip',
+    'mac12': 'builds/firefox/%s/firefox-mac.zip',
+    'mac12-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip',
+    'mac13': 'builds/firefox/%s/firefox-mac.zip',
+    'mac13-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip',
+    'mac14': 'builds/firefox/%s/firefox-mac.zip',
+    'mac14-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip',
+    'mac15': 'builds/firefox/%s/firefox-mac.zip',
+    'mac15-arm64': 'builds/firefox/%s/firefox-mac-arm64.zip',
     'win64': 'builds/firefox/%s/firefox-win64.zip',
   },
   'firefox-beta': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/firefox-beta/%s/firefox-beta-ubuntu-20.04.zip',
-    'generic-linux-arm64': undefined,
-    'ubuntu18.04': 'builds/firefox-beta/%s/firefox-beta-ubuntu-18.04.zip',
-    'ubuntu20.04': 'builds/firefox-beta/%s/firefox-beta-ubuntu-20.04.zip',
-    'ubuntu22.04': 'builds/firefox-beta/%s/firefox-beta-ubuntu-22.04.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-20.04.zip',
+    'ubuntu22.04-x64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-22.04.zip',
+    'ubuntu24.04-x64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-24.04.zip',
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': undefined,
     'ubuntu22.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-22.04-arm64.zip',
-    'mac10.13': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac10.14': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac10.15': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac11': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac11-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-11-arm64.zip',
-    'mac12': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac12-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-11-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-24.04-arm64.zip',
+    'debian11-x64': 'builds/firefox-beta/%s/firefox-beta-debian-11.zip',
+    'debian11-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-11-arm64.zip',
+    'debian12-x64': 'builds/firefox-beta/%s/firefox-beta-debian-12.zip',
+    'debian12-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-12-arm64.zip',
+    'mac10.13': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac10.14': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac10.15': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac11': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac11-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip',
+    'mac12': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac12-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip',
+    'mac13': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac13-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip',
+    'mac14': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac14-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip',
+    'mac15': 'builds/firefox-beta/%s/firefox-beta-mac.zip',
+    'mac15-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-arm64.zip',
     'win64': 'builds/firefox-beta/%s/firefox-beta-win64.zip',
   },
   'webkit': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/webkit/%s/webkit-ubuntu-20.04.zip',
-    'generic-linux-arm64': 'builds/webkit/%s/webkit-ubuntu-20.04-arm64.zip',
-    'ubuntu18.04': 'builds/webkit/%s/webkit-ubuntu-18.04.zip',
-    'ubuntu20.04': 'builds/webkit/%s/webkit-ubuntu-20.04.zip',
-    'ubuntu22.04': 'builds/webkit/%s/webkit-ubuntu-22.04.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/webkit/%s/webkit-ubuntu-20.04.zip',
+    'ubuntu22.04-x64': 'builds/webkit/%s/webkit-ubuntu-22.04.zip',
+    'ubuntu24.04-x64': 'builds/webkit/%s/webkit-ubuntu-24.04.zip',
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/webkit/%s/webkit-ubuntu-20.04-arm64.zip',
     'ubuntu22.04-arm64': 'builds/webkit/%s/webkit-ubuntu-22.04-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/webkit/%s/webkit-ubuntu-24.04-arm64.zip',
+    'debian11-x64': 'builds/webkit/%s/webkit-debian-11.zip',
+    'debian11-arm64': 'builds/webkit/%s/webkit-debian-11-arm64.zip',
+    'debian12-x64': 'builds/webkit/%s/webkit-debian-12.zip',
+    'debian12-arm64': 'builds/webkit/%s/webkit-debian-12-arm64.zip',
     'mac10.13': undefined,
     'mac10.14': 'builds/deprecated-webkit-mac-10.14/%s/deprecated-webkit-mac-10.14.zip',
-    'mac10.15': 'builds/webkit/%s/webkit-mac-10.15.zip',
+    'mac10.15': 'builds/deprecated-webkit-mac-10.15/%s/deprecated-webkit-mac-10.15.zip',
     'mac11': 'builds/webkit/%s/webkit-mac-11.zip',
     'mac11-arm64': 'builds/webkit/%s/webkit-mac-11-arm64.zip',
     'mac12': 'builds/webkit/%s/webkit-mac-12.zip',
     'mac12-arm64': 'builds/webkit/%s/webkit-mac-12-arm64.zip',
+    'mac13': 'builds/webkit/%s/webkit-mac-13.zip',
+    'mac13-arm64': 'builds/webkit/%s/webkit-mac-13-arm64.zip',
+    'mac14': 'builds/webkit/%s/webkit-mac-14.zip',
+    'mac14-arm64': 'builds/webkit/%s/webkit-mac-14-arm64.zip',
+    'mac15': 'builds/webkit/%s/webkit-mac-15.zip',
+    'mac15-arm64': 'builds/webkit/%s/webkit-mac-15-arm64.zip',
     'win64': 'builds/webkit/%s/webkit-win64.zip',
   },
   'ffmpeg': {
     '<unknown>': undefined,
-    'generic-linux': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
-    'generic-linux-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
-    'ubuntu18.04': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
-    'ubuntu20.04': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
-    'ubuntu22.04': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
-    'ubuntu18.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'ubuntu22.04-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'ubuntu24.04-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'ubuntu24.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'debian11-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'debian11-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'debian12-x64': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'debian12-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
     'mac10.13': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac10.14': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac10.15': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
@@ -142,15 +235,79 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
     'mac12': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac12-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
+    'mac13': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
+    'mac13-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
+    'mac14': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
+    'mac14-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
+    'mac15': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
+    'mac15-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
     'win64': 'builds/ffmpeg/%s/ffmpeg-win64.zip',
   },
-};
+  'winldd': {
+    '<unknown>': undefined,
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': undefined,
+    'ubuntu22.04-x64': undefined,
+    'ubuntu24.04-x64': undefined,
+    'ubuntu18.04-arm64': undefined,
+    'ubuntu20.04-arm64': undefined,
+    'ubuntu22.04-arm64': undefined,
+    'ubuntu24.04-arm64': undefined,
+    'debian11-x64': undefined,
+    'debian11-arm64': undefined,
+    'debian12-x64': undefined,
+    'debian12-arm64': undefined,
+    'mac10.13': undefined,
+    'mac10.14': undefined,
+    'mac10.15': undefined,
+    'mac11': undefined,
+    'mac11-arm64': undefined,
+    'mac12': undefined,
+    'mac12-arm64': undefined,
+    'mac13': undefined,
+    'mac13-arm64': undefined,
+    'mac14': undefined,
+    'mac14-arm64': undefined,
+    'mac15': undefined,
+    'mac15-arm64': undefined,
+    'win64': 'builds/winldd/%s/winldd-win64.zip',
+  },
+  'android': {
+    '<unknown>': 'builds/android/%s/android.zip',
+    'ubuntu18.04-x64': undefined,
+    'ubuntu20.04-x64': 'builds/android/%s/android.zip',
+    'ubuntu22.04-x64': 'builds/android/%s/android.zip',
+    'ubuntu24.04-x64': 'builds/android/%s/android.zip',
+    'ubuntu18.04-arm64': undefined,
+    'ubuntu20.04-arm64': 'builds/android/%s/android.zip',
+    'ubuntu22.04-arm64': 'builds/android/%s/android.zip',
+    'ubuntu24.04-arm64': 'builds/android/%s/android.zip',
+    'debian11-x64': 'builds/android/%s/android.zip',
+    'debian11-arm64': 'builds/android/%s/android.zip',
+    'debian12-x64': 'builds/android/%s/android.zip',
+    'debian12-arm64': 'builds/android/%s/android.zip',
+    'mac10.13': 'builds/android/%s/android.zip',
+    'mac10.14': 'builds/android/%s/android.zip',
+    'mac10.15': 'builds/android/%s/android.zip',
+    'mac11': 'builds/android/%s/android.zip',
+    'mac11-arm64': 'builds/android/%s/android.zip',
+    'mac12': 'builds/android/%s/android.zip',
+    'mac12-arm64': 'builds/android/%s/android.zip',
+    'mac13': 'builds/android/%s/android.zip',
+    'mac13-arm64': 'builds/android/%s/android.zip',
+    'mac14': 'builds/android/%s/android.zip',
+    'mac14-arm64': 'builds/android/%s/android.zip',
+    'mac15': 'builds/android/%s/android.zip',
+    'mac15-arm64': 'builds/android/%s/android.zip',
+    'win64': 'builds/android/%s/android.zip',
+  },
+} as const;
 
 @SingletonProto()
 @BinaryAdapter(BinaryType.Playwright)
 export class PlaywrightBinary extends AbstractBinary {
   private dirItems?: Record<string, BinaryItem[]>;
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 
@@ -158,19 +315,30 @@ export class PlaywrightBinary extends AbstractBinary {
     if (!this.dirItems) {
       const packageData = await this.requestJSON(PACKAGE_URL);
       const nowDateISO = new Date().toISOString();
+      const buildDirs: BinaryItem[] = [];
+      for (const browserName of Object.keys(DOWNLOAD_PATHS)) {
+        if (browserName === 'chromium-headless-shell' || browserName === 'chromium-tip-of-tree-headless-shell') {
+          continue;
+        }
+        buildDirs.push({ name: `${browserName}/`, isDir: true, url: '', size: '-', date: nowDateISO });
+      }
       this.dirItems = {
         '/': [{ name: 'builds/', isDir: true, url: '', size: '-', date: nowDateISO }],
-        '/builds/': Object.keys(DOWNLOAD_PATHS).map(
-          dist => ({ name: `${dist}/`, isDir: true, url: '', size: '-', date: nowDateISO })),
-        ...Object.fromEntries(Object.keys(DOWNLOAD_PATHS).map(dist => [ `/builds/${dist}/`, []])),
+        '/builds/': buildDirs,
       };
+      for (const browserName of Object.keys(DOWNLOAD_PATHS)) {
+        if (browserName === 'chromium-headless-shell' || browserName === 'chromium-tip-of-tree-headless-shell') {
+          continue;
+        }
+        this.dirItems[`/builds/${browserName}/`] = [];
+      }
 
       // Only download beta and release versions of packages to reduce amount of request
       const packageVersions = Object.keys(packageData.versions)
         .filter(version => version.match(/^(?:\d+\.\d+\.\d+)(?:-beta-\d+)?$/))
         // select recently update 20 items
         .slice(-20);
-      const browsers: { name: string; revision: string; browserVersion: string; revisionOverrides?: Record<string, string> }[] = [];
+      const browsers: { name: keyof typeof DOWNLOAD_PATHS; revision: string; browserVersion: string; revisionOverrides?: Record<string, string> }[] = [];
       await Promise.all(
         packageVersions.map(version =>
           this.requestJSON(
@@ -195,19 +363,55 @@ export class PlaywrightBinary extends AbstractBinary {
             }),
         ),
       );
+      // if chromium-headless-shell not exists on browsers, copy chromium to chromium-headless-shell
+      if (!browsers.find(browser => browser.name === 'chromium-headless-shell')) {
+        const chromium = browsers.find(browser => browser.name === 'chromium');
+        // {
+        //   "name": "chromium",
+        //   "revision": "1155",
+        //   "installByDefault": true,
+        //   "browserVersion": "133.0.6943.16"
+        // }
+        if (chromium) {
+          browsers.push({
+            ...chromium,
+            name: 'chromium-headless-shell',
+          });
+        }
+      }
+      // if chromium-tip-of-tree-headless-shell not exists on browsers, copy chromium-tip-of-tree to chromium-tip-of-tree-headless-shell
+      if (!browsers.find(browser => browser.name === 'chromium-tip-of-tree-headless-shell')) {
+        const chromiumTipOfTree = browsers.find(browser => browser.name === 'chromium-tip-of-tree');
+        if (chromiumTipOfTree) {
+          browsers.push({
+            ...chromiumTipOfTree,
+            name: 'chromium-tip-of-tree-headless-shell',
+          });
+        }
+      }
 
       for (const browser of browsers) {
         const downloadPaths = DOWNLOAD_PATHS[browser.name];
         if (!downloadPaths) continue;
+        let browserDirname = browser.name;
+        if (browser.name === 'chromium-headless-shell') {
+          // chromium-headless-shell should be under chromium
+          // https://playwright.azureedge.net/builds/chromium/1155/chromium-headless-shell-mac-arm64.zip
+          browserDirname = 'chromium';
+        } else if (browser.name === 'chromium-tip-of-tree-headless-shell') {
+          // chromium-tip-of-tree-headless-shell should be under chromium-tip-of-tree
+          // https://playwright.azureedge.net/builds/chromium-tip-of-tree/1293/chromium-tip-of-tree-headless-shell-mac-arm64.zip
+          browserDirname = 'chromium-tip-of-tree';
+        }
         for (const [ platform, remotePath ] of Object.entries(downloadPaths)) {
           if (typeof remotePath !== 'string') continue;
           const revision = browser.revisionOverrides?.[platform] ?? browser.revision;
           const itemDate = browser.browserVersion || revision;
           const url = DOWNLOAD_HOST + util.format(remotePath, revision);
           const name = path.basename(remotePath);
-          const dir = `/builds/${browser.name}/${revision}/`;
+          const dir = `/builds/${browserDirname}/${revision}/`;
           if (!this.dirItems[dir]) {
-            this.dirItems[`/builds/${browser.name}/`].push({
+            this.dirItems[`/builds/${browserDirname}/`].push({
               name: `${revision}/`,
               isDir: true,
               url: '',

app/common/adapter/binary/PrismaBinary.ts

@@ -0,0 +1,132 @@
+import path from 'node:path';
+import { SingletonProto } from '@eggjs/tegg';
+import { BinaryType } from '../../enum/Binary';
+import binaries, { BinaryName } from '../../../../config/binaries';
+import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './AbstractBinary';
+
+@SingletonProto()
+@BinaryAdapter(BinaryType.Prisma)
+export class PrismaBinary extends AbstractBinary {
+  private dirItems: {
+    [key: string]: BinaryItem[];
+  } = {};
+
+  async initFetch() {
+    // https://github.com/cnpm/cnpmcore/issues/473#issuecomment-1562115738
+    const pkgUrl = 'https://registry.npmjs.com/@prisma/engines';
+    const data = await this.requestJSON(pkgUrl);
+    const modified = data.time.modified;
+    this.dirItems = {};
+    this.dirItems['/'] = [
+      {
+        name: 'all_commits/',
+        date: modified,
+        size: '-',
+        isDir: true,
+        url: '',
+      },
+    ];
+    this.dirItems['/all_commits/'] = [];
+    const commitIdMap: Record<string, boolean> = {};
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/
+    for (const version in data.versions) {
+      const major = parseInt(version.split('.', 1)[0]);
+      // need >= 3.0.0
+      if (major < 3) continue;
+      const date = data.time[version];
+      const pkg = data.versions[version];
+      // https://registry.npmjs.com/@prisma/engines/4.14.1
+      // https://registry.npmjs.com/@prisma/engines/5.7.0 should read from dependencies
+      const enginesVersion = pkg.devDependencies?.['@prisma/engines-version']
+        || pkg.dependencies?.['@prisma/engines-version'] || '';
+      // "@prisma/engines-version": "4.14.0-67.d9a4c5988f480fa576d43970d5a23641aa77bc9c"
+      // "@prisma/engines-version": "5.7.0-41.79fb5193cf0a8fdbef536e4b4a159cad677ab1b9"
+      const matched = /\.(\w{30,})$/.exec(enginesVersion);
+      if (!matched) continue;
+      const commitId = matched[1];
+      if (commitIdMap[commitId]) continue;
+      commitIdMap[commitId] = true;
+      this.dirItems['/all_commits/'].push({
+        name: `${commitId}/`,
+        date,
+        size: '-',
+        isDir: true,
+        url: '',
+      });
+    }
+  }
+
+  async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
+    const existsItems = this.dirItems[dir];
+    if (existsItems) {
+      return { items: existsItems, nextParams: null };
+    }
+    // /foo/ => foo/
+    const binaryConfig = binaries[binaryName];
+    const subDir = dir.substring(1);
+    const url = `${binaryConfig.distUrl}?delimiter=/&prefix=${encodeURIComponent(subDir)}`;
+    const result = await this.requestJSON(url);
+    return { items: this.#parseItems(result), nextParams: null };
+  }
+
+  #parseItems(result: any): BinaryItem[] {
+    const items: BinaryItem[] = [];
+    // objects": [
+    //   {
+    //   "uploaded": "2023-05-23T15:43:05.772Z",
+    //   "checksums": {
+    //   "md5": "d41d8cd98f00b204e9800998ecf8427e"
+    //   },
+    //   "httpEtag": "\"d41d8cd98f00b204e9800998ecf8427e\"",
+    //   "etag": "d41d8cd98f00b204e9800998ecf8427e",
+    //   "size": 0,
+    //   "version": "7e77b6b8c1d214f2c6be3c959749b5a6",
+    //   "key": "all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/.finished"
+    //   },
+    // {
+    //   "uploaded": "2023-05-23T15:41:33.861Z",
+    //   "checksums": {
+    //   "md5": "4822215a13ae372ae82afd12689fce37"
+    //   },
+    //   "httpEtag": "\"4822215a13ae372ae82afd12689fce37\"",
+    //   "etag": "4822215a13ae372ae82afd12689fce37",
+    //   "size": 96,
+    //   "version": "7e77b6ba29d4e776023e4fa62825c13a",
+    //   "key": "all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/libquery_engine.dylib.node.gz.sha256"
+    //   },
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/
+    const objects: {
+      uploaded: string;
+      size: number;
+      key: string;
+    }[] = result.objects || [];
+    for (const o of objects) {
+      const fullname = o.key;
+      // ignore size = 0
+      if (o.size === 0) continue;
+      const name = path.basename(fullname);
+      items.push({
+        name,
+        isDir: false,
+        // https://binaries.prisma.sh/all_commits/2452cc6313d52b8b9a96999ac0e974d0aedf88db/darwin-arm64/prisma-fmt.gz
+        url: `https://binaries.prisma.sh/${fullname}`,
+        size: o.size,
+        date: o.uploaded,
+      });
+    }
+    // delimitedPrefixes: [ 'all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/' ]
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/
+    const delimitedPrefixes: string[] = result.delimitedPrefixes || [];
+    for (const fullname of delimitedPrefixes) {
+      const name = `${path.basename(fullname)}/`;
+      items.push({
+        name,
+        isDir: true,
+        url: '',
+        size: '-',
+        date: new Date().toISOString(),
+      });
+    }
+    return items;
+  }
+}

app/common/adapter/binary/PuppeteerBinary.ts

@@ -9,7 +9,7 @@ export class PuppeteerBinary extends AbstractBinary {
     [key: string]: BinaryItem[];
   };
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 

app/common/adapter/binary/SqlcipherBinary.ts

@@ -5,7 +5,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Sqlcipher)
 export class SqlcipherBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/changesStream/CnpmjsorgChangesStream.ts

@@ -6,6 +6,16 @@ import { AbstractChangeStream, RegistryChangesStream } from './AbstractChangesSt
 
 const MAX_LIMIT = 10000;
 
+type FetchResults = {
+  results: {
+    seq: number;
+    type: string;
+    id: string;
+    changes: Record<string, string>[];
+    gmt_modified: Date,
+  }[];
+};
+
 @SingletonProto()
 @RegistryChangesStream(RegistryType.Cnpmjsorg)
 export class CnpmjsorgChangesStream extends AbstractChangeStream {
@@ -18,13 +28,13 @@ export class CnpmjsorgChangesStream extends AbstractChangeStream {
     return since;
   }
 
-  private async tryFetch(registry: Registry, since: string, limit = 1000) {
+  private async tryFetch(registry: Registry, since: string, limit = 1000): Promise<{ data: FetchResults }> {
     if (limit > MAX_LIMIT) {
       throw new E500(`limit too large, current since: ${since}, limit: ${limit}`);
     }
     const db = this.getChangesStreamUrl(registry, since, limit);
     // json mode
-    const res = await this.httpclient.request(db, {
+    const res = await this.httpclient.request<FetchResults>(db, {
       followRedirect: true,
       timeout: 30000,
       dataType: 'json',

app/common/adapter/changesStream/NpmChangesStream.ts

@@ -28,7 +28,7 @@ export class NpmChangesStream extends AbstractChangeStream {
     const db = this.getChangesStreamUrl(registry, since);
     const { res } = await this.httpclient.request(db, {
       streaming: true,
-      timeout: 10000,
+      timeout: 60000,
     });
 
     let buf = '';

app/common/constants.ts

@@ -1,12 +1,21 @@
 export const BUG_VERSIONS = 'bug-versions';
 export const LATEST_TAG = 'latest';
 export const GLOBAL_WORKER = 'GLOBAL_WORKER';
+export const PROXY_CACHE_DIR_NAME = 'proxy-cache-packages';
+export const ABBREVIATED_META_TYPE = 'application/vnd.npm.install-v1+json';
+export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ];
+
 export enum SyncMode {
   none = 'none',
   admin = 'admin',
+  proxy = 'proxy',
   exist = 'exist',
   all = 'all',
 }
+export enum ChangesStreamMode {
+  json = 'json',
+  streaming = 'streaming',
+}
 export enum SyncDeleteMode {
   ignore = 'ignore',
   block = 'block',

app/common/enum/Binary.ts

@@ -10,6 +10,8 @@ export enum BinaryType {
   Nwjs = 'nwjs',
   Playwright = 'playwright',
   Puppeteer = 'puppeteer',
+  Prisma = 'prisma',
   Sqlcipher = 'sqlcipher',
   ChromeForTesting = 'chromeForTesting',
+  Edgedriver = 'edgedriver',
 }

app/common/enum/Task.ts

@@ -2,6 +2,7 @@ export enum TaskType {
   SyncPackage = 'sync_package',
   ChangesStream = 'changes_stream',
   SyncBinary = 'sync_binary',
+  UpdateProxyCache = 'update_proxy_cache',
   CreateHook = 'create_hook',
   TriggerHook = 'trigger_hook',
 }

app/common/typing.ts

@@ -1,6 +1,8 @@
+import { CnpmcoreConfig } from '../port/config';
 import { Readable } from 'stream';
 import { IncomingHttpHeaders } from 'http';
 import { EggContext } from '@eggjs/tegg';
+import { estypes } from '@elastic/elasticsearch';
 
 export interface UploadResult {
   key: string;
@@ -49,6 +51,12 @@ export interface QueueAdapter {
   length(key: string): Promise<number>;
 }
 
+export interface SearchAdapter {
+  search<T>(query: any): Promise<estypes.SearchHitsMetadata<T>>;
+  upsert<T>(id: string, document: T): Promise<string>;
+  delete(id: string): Promise<string>;
+}
+
 export interface AuthUrlResult {
   loginUrl: string;
   doneUrl: string;
@@ -62,3 +70,12 @@ export interface AuthClient {
   getAuthUrl(ctx: EggContext): Promise<AuthUrlResult>;
   ensureCurrentUser(): Promise<userResult | null>;
 }
+
+declare module 'egg' {
+  // eslint-disable-next-line
+  // @ts-ignore
+  // avoid TS2310 Type 'EggAppConfig' recursively references itself as a base type.
+  interface EggAppConfig {
+    cnpmcore: CnpmcoreConfig;
+  }
+}

app/core/entity/BugVersion.ts

@@ -8,7 +8,7 @@ export type BugVersionPackages = Record<string, BugVersionPackage>;
 export class BugVersion {
   private readonly data: BugVersionPackages;
 
-  constructor(data) {
+  constructor(data: BugVersionPackages) {
     this.data = data;
   }
 

app/core/entity/Hook.ts

@@ -48,7 +48,7 @@ export class Hook extends Entity {
   }
 
   // payload 可能会特别大，如果做多次 stringify 浪费太多 cpu
-  signPayload(payload: object): { digest, payloadStr } {
+  signPayload(payload: object) {
     const payloadStr = JSON.stringify(payload);
     const digest = crypto.createHmac('sha256', this.secret)
       .update(JSON.stringify(payload))

app/core/entity/PackageVersion.ts

@@ -1,6 +1,7 @@
 import { Dist } from './Dist';
 import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
+import { PaddingSemVer } from './PaddingSemVer';
 
 interface PackageVersionData extends EntityData {
   packageId: string;
@@ -11,6 +12,8 @@ interface PackageVersionData extends EntityData {
   tarDist: Dist;
   readmeDist: Dist;
   publishTime: Date;
+  paddingVersion?: string | null;
+  isPreRelease?: boolean | null;
 }
 
 export class PackageVersion extends Entity {
@@ -22,6 +25,8 @@ export class PackageVersion extends Entity {
   tarDist: Dist;
   readmeDist: Dist;
   publishTime: Date;
+  paddingVersion: string;
+  isPreRelease: boolean;
 
   constructor(data: PackageVersionData) {
     super(data);
@@ -33,6 +38,14 @@ export class PackageVersion extends Entity {
     this.tarDist = data.tarDist;
     this.readmeDist = data.readmeDist;
     this.publishTime = data.publishTime;
+    if (data.paddingVersion && typeof data.isPreRelease === 'boolean') {
+      this.paddingVersion = data.paddingVersion;
+      this.isPreRelease = data.isPreRelease;
+    } else {
+      const paddingSemVer = new PaddingSemVer(this.version);
+      this.paddingVersion = paddingSemVer.paddingVersion;
+      this.isPreRelease = paddingSemVer.isPreRelease;
+    }
   }
 
   static create(data: EasyData<PackageVersionData, 'packageVersionId'>): PackageVersion {

app/core/entity/PaddingSemVer.ts

@@ -0,0 +1,50 @@
+import { SemVer, valid } from 'semver';
+
+export class PaddingSemVer {
+  private readonly semver: SemVer;
+  // 跳过 semver 中的 buildInfo, buildInfo 不参与版本比较
+  private _paddingVersion: string;
+  readonly isPreRelease: boolean;
+
+  constructor(semver: string | SemVer) {
+    // ignore invalid version, e.g.: '1000000000000000000.0.0' on https://registry.npmjs.com/latentflip-test
+    if (!valid(semver)) {
+      this.isPreRelease = true;
+      this._paddingVersion = PaddingSemVer.anyVersion();
+      return;
+    }
+    this.semver = new SemVer(semver);
+    if ((this.semver as any).includePrerelease) {
+      this.isPreRelease = true;
+    } else if (this.semver.prerelease && this.semver.prerelease.length) {
+      this.isPreRelease = true;
+    } else {
+      this.isPreRelease = false;
+    }
+  }
+
+  get paddingVersion(): string {
+    if (!this._paddingVersion) {
+      this._paddingVersion = PaddingSemVer.paddingVersion(this.semver.major)
+        + PaddingSemVer.paddingVersion(this.semver.minor)
+        + PaddingSemVer.paddingVersion(this.semver.patch);
+    }
+    return this._paddingVersion;
+  }
+
+  // 版本信息中为纯数字, JS 中支持的最大整型为 16 位
+  // 因此填充成 16 位对齐，如果版本号超过 16 位，则抛出异常
+  static paddingVersion(v: number) {
+    const t = String(v);
+    if (t.length <= 16) {
+      const padding = new Array(16 - t.length).fill(0)
+        .join('');
+      return padding + t;
+    }
+    throw new Error(`v ${v} too long`);
+  }
+
+  static anyVersion() {
+    return '000000000000000000000000000000000000000000000000';
+  }
+}

app/core/entity/ProxyCache.ts

@@ -0,0 +1,42 @@
+import { Entity, EntityData } from './Entity';
+import { EasyData } from '../util/EntityUtil';
+import { DIST_NAMES } from './Package';
+import { isPkgManifest } from '../service/ProxyCacheService';
+import { PROXY_CACHE_DIR_NAME } from '../../common/constants';
+interface ProxyCacheData extends EntityData {
+  fullname: string;
+  fileType: DIST_NAMES;
+  version?: string;
+}
+
+export type CreateProxyCacheData = Omit<EasyData<ProxyCacheData, 'id'>, 'id'| 'filePath'>;
+
+export class ProxyCache extends Entity {
+  readonly fullname: string;
+  readonly fileType: DIST_NAMES;
+  readonly filePath: string;
+  readonly version?: string;
+
+  constructor(data: ProxyCacheData) {
+    super(data);
+    this.fullname = data.fullname;
+    this.fileType = data.fileType;
+    this.version = data.version;
+    if (isPkgManifest(data.fileType)) {
+      this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.fileType}`;
+    } else {
+      this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`;
+    }
+  }
+
+  public static create(data: CreateProxyCacheData): ProxyCache {
+    const newData = { ...data, createdAt: new Date(), updatedAt: new Date() };
+    return new ProxyCache(newData);
+  }
+
+  public static update(data: ProxyCache): ProxyCache {
+    data.updatedAt = new Date();
+    return data;
+  }
+
+}

app/core/entity/Registry.ts

@@ -9,6 +9,7 @@ interface RegistryData extends EntityData {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 }
 
 export type CreateRegistryData = Omit<EasyData<RegistryData, 'registryId'>, 'id'>;
@@ -20,6 +21,7 @@ export class Registry extends Entity {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 
   constructor(data: RegistryData) {
     super(data);
@@ -29,10 +31,11 @@ export class Registry extends Entity {
     this.changeStream = data.changeStream;
     this.userPrefix = data.userPrefix;
     this.type = data.type;
+    this.authToken = data.authToken;
   }
 
   public static create(data: CreateRegistryData): Registry {
-    const newData = EntityUtil.defaultData(data, 'registryId');
+    const newData = EntityUtil.defaultData<RegistryData, 'registryId'>(data, 'registryId');
     return new Registry(newData);
   }
 }

app/core/entity/SqlRange.ts

@@ -0,0 +1,81 @@
+import { Range, Comparator } from 'semver';
+import { PaddingSemVer } from './PaddingSemVer';
+
+const OPERATOR_MAP = {
+  '<': '$lt',
+  '<=': '$lte',
+  '>': '$gt',
+  '>=': '$gte',
+  '': '$eq',
+};
+
+export class SqlRange {
+  private readonly range: Range;
+  private _containPreRelease: boolean;
+  readonly condition: object;
+
+  constructor(range: string | Range) {
+    this.range = new Range(range);
+    this._containPreRelease = false;
+    this.condition = this.generateWhere();
+  }
+
+  private comparatorToSql(comparator: Comparator) {
+    if (comparator.semver === (Comparator as any).ANY) {
+      return {
+        $and: [
+          {
+            isPreRelease: {
+              $lte: 0,
+            },
+          },
+          {
+            paddingVersion: {
+              $gte: PaddingSemVer.anyVersion(),
+            },
+          },
+        ],
+      };
+    }
+    const paddingSemver = new PaddingSemVer(comparator.semver);
+    const operator = OPERATOR_MAP[comparator.operator as keyof typeof OPERATOR_MAP];
+    if (!operator) {
+      throw new Error(`unknown operator ${comparator.operator}`);
+    }
+    this._containPreRelease = this._containPreRelease || paddingSemver.isPreRelease;
+    return {
+      $and: [
+        {
+          isPreRelease: {
+            $lte: paddingSemver.isPreRelease ? 1 : 0,
+          },
+        },
+        {
+          paddingVersion: {
+            [operator]: paddingSemver.paddingVersion,
+          },
+        },
+      ],
+    };
+  }
+
+  private comparatorSetToSql(comparatorSet: Array<Comparator>) {
+    const condition: Array<object> = [];
+    for (const comparator of comparatorSet) {
+      condition.push(this.comparatorToSql(comparator));
+    }
+    return { $and: condition };
+  }
+
+  private generateWhere() {
+    const conditions: Array<object> = [];
+    for (const rangeSet of this.range.set) {
+      conditions.push(this.comparatorSetToSql(rangeSet as Comparator[]));
+    }
+    return { $or: conditions };
+  }
+
+  get containPreRelease(): boolean {
+    return this._containPreRelease;
+  }
+}

app/core/entity/Task.ts

@@ -3,8 +3,12 @@ import path from 'path';
 import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
 import { TaskType, TaskState } from '../../common/enum/Task';
+import { PROXY_CACHE_DIR_NAME } from '../../common/constants';
 import dayjs from '../../common/dayjs';
 import { HookEvent } from './HookEvent';
+import { DIST_NAMES } from './Package';
+import { isPkgManifest } from '../service/ProxyCacheService';
+import { InternalServerError } from 'egg-errors';
 
 export const HOST_NAME = os.hostname();
 export const PID = process.pid;
@@ -31,13 +35,19 @@ export interface TaskData<T = TaskBaseData> extends EntityData {
 export type SyncPackageTaskOptions = {
   authorId?: string;
   authorIp?: string;
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
   // force sync history version
   forceSyncHistory?: boolean;
   registryId?: string;
+  specificVersions?: Array<string>;
+};
+
+export type UpdateProxyCacheTaskOptions = {
+  fullname: string,
+  version?: string,
+  fileType: DIST_NAMES,
 };
 
 export interface CreateHookTaskData extends TaskBaseData {
@@ -51,11 +61,18 @@ export interface TriggerHookTaskData extends TaskBaseData {
 }
 
 export interface CreateSyncPackageTaskData extends TaskBaseData {
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
   forceSyncHistory?: boolean;
+  specificVersions?: Array<string>;
+}
+
+export interface CreateUpdateProxyCacheTaskData extends TaskBaseData {
+  fullname: string,
+  version?: string,
+  fileType: DIST_NAMES,
+  filePath: string
 }
 
 export interface ChangesStreamTaskData extends TaskBaseData {
@@ -75,6 +92,7 @@ export type CreateHookTask = Task<CreateHookTaskData>;
 export type TriggerHookTask = Task<TriggerHookTaskData>;
 export type CreateSyncPackageTask = Task<CreateSyncPackageTaskData>;
 export type ChangesStreamTask = Task<ChangesStreamTaskData>;
+export type CreateUpdateProxyCacheTask = Task<CreateUpdateProxyCacheTaskData>;
 
 export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
   taskId: string;
@@ -131,12 +149,12 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
       data: {
         // task execute worker
         taskWorker: '',
-        remoteAuthToken: options?.remoteAuthToken,
         tips: options?.tips,
         registryId: options?.registryId ?? '',
         skipDependencies: options?.skipDependencies,
         syncDownloadData: options?.syncDownloadData,
         forceSyncHistory: options?.forceSyncHistory,
+        specificVersions: options?.specificVersions,
       },
     };
     const task = this.create(data);
@@ -231,6 +249,34 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
     return task;
   }
 
+  public static needMergeWhenWaiting(type: TaskType) {
+    return [ TaskType.SyncBinary, TaskType.SyncPackage ].includes(type);
+  }
+
+  public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask {
+    if (!isPkgManifest(options.fileType)) {
+      throw new InternalServerError('should not update package version manifest.');
+    }
+    const filePath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/${options.fileType}`;
+    const data = {
+      type: TaskType.UpdateProxyCache,
+      state: TaskState.Waiting,
+      targetName,
+      authorId: `pid_${PID}`,
+      authorIp: HOST_NAME,
+      data: {
+        taskWorker: '',
+        fullname: options.fullname,
+        version: options?.version,
+        fileType: options.fileType,
+        filePath,
+      },
+    };
+    const task = this.create(data);
+    task.logPath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`;
+    return task;
+  }
+
   start(): TaskUpdateCondition {
     const condition = {
       taskId: this.taskId,

app/core/entity/Token.ts

@@ -13,7 +13,8 @@ interface BaseTokenData extends EntityData {
   cidrWhitelist?: string[];
   userId: string;
   isReadonly?: boolean;
-  type?: TokenType;
+  type?: TokenType | string;
+  lastUsedAt?: Date;
 }
 
 interface ClassicTokenData extends BaseTokenData{
@@ -30,7 +31,7 @@ interface GranularTokenData extends BaseTokenData {
 
 type TokenData = ClassicTokenData | GranularTokenData;
 
-export function isGranularToken(data: TokenData): data is GranularTokenData {
+export function isGranularToken(data: TokenData | Token): data is GranularTokenData {
   return data.type === TokenType.granular;
 }
 
@@ -48,6 +49,7 @@ export class Token extends Entity {
   readonly allowedScopes?: string[];
   readonly expiredAt?: Date;
   readonly expires?: number;
+  lastUsedAt: Date | null;
   allowedPackages?: string[];
   token?: string;
 
@@ -59,7 +61,8 @@ export class Token extends Entity {
     this.tokenKey = data.tokenKey;
     this.cidrWhitelist = data.cidrWhitelist || [];
     this.isReadonly = data.isReadonly || false;
-    this.type = data.type || TokenType.classic;
+    this.type = (data.type as TokenType) || TokenType.classic;
+    this.lastUsedAt = data.lastUsedAt || null;
 
     if (isGranularToken(data)) {
       this.name = data.name;
@@ -67,6 +70,7 @@ export class Token extends Entity {
       this.allowedScopes = data.allowedScopes;
       this.expiredAt = data.expiredAt;
       this.allowedPackages = data.allowedPackages;
+      this.isAutomation = false;
     } else {
       this.isAutomation = data.isAutomation || false;
     }

app/core/event/BugVersionFixHandler.ts

@@ -2,15 +2,12 @@ import { Event, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import { PACKAGE_VERSION_ADDED } from './index';
 import { BUG_VERSIONS } from '../../common/constants';
-import { PackageManagerService } from '../service/PackageManagerService';
 import { BugVersionService } from '../service/BugVersionService';
 
 @Event(PACKAGE_VERSION_ADDED)
 export class BugVersionFixHandler {
   @Inject()
   private readonly bugVersionService: BugVersionService;
-  @Inject()
-  private readonly packageManagerService: PackageManagerService;
 
   @Inject()
   private readonly logger: EggLogger;
@@ -18,7 +15,7 @@ export class BugVersionFixHandler {
   async handle(fullname: string) {
     if (fullname !== BUG_VERSIONS) return;
     try {
-      const bugVersion = await this.packageManagerService.getBugVersion();
+      const bugVersion = await this.bugVersionService.getBugVersion();
       if (!bugVersion) return;
       await this.bugVersionService.cleanBugVersionPackageCaches(bugVersion);
     } catch (e) {

app/core/event/CacheCleaner.ts

@@ -24,77 +24,77 @@ class CacheCleanerEvent {
 }
 
 @Event(PACKAGE_UNPUBLISHED)
-export class PackageUnpublished extends CacheCleanerEvent {
+export class PackageUnpublishedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_BLOCKED)
-export class PackageBlocked extends CacheCleanerEvent {
+export class PackageBlockedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_UNBLOCKED)
-export class PackageUnblocked extends CacheCleanerEvent {
+export class PackageUnblockedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
-export class PackageVersionAdded extends CacheCleanerEvent {
+export class PackageVersionAddedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_VERSION_REMOVED)
-export class PackageVersionRemoved extends CacheCleanerEvent {
+export class PackageVersionRemovedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_ADDED)
-export class PackageTagAdded extends CacheCleanerEvent {
+export class PackageTagAddedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_CHANGED)
-export class PackageTagChanged extends CacheCleanerEvent {
+export class PackageTagChangedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_TAG_REMOVED)
-export class PackageTagRemoved extends CacheCleanerEvent {
+export class PackageTagRemovedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_MAINTAINER_CHANGED)
-export class PackageMaintainerChanged extends CacheCleanerEvent {
+export class PackageMaintainerChangedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_MAINTAINER_REMOVED)
-export class PackageMaintainerRemoved extends CacheCleanerEvent {
+export class PackageMaintainerRemovedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }
 }
 
 @Event(PACKAGE_META_CHANGED)
-export class PackageMetaChanged extends CacheCleanerEvent {
+export class PackageMetaChangedCacheCleanEvent extends CacheCleanerEvent {
   async handle(fullname: string) {
     await this.removeCache(fullname);
   }

app/core/event/ChangesStream.ts

@@ -29,7 +29,7 @@ class ChangesStreamEvent {
   protected readonly config: EggAppConfig;
 
   protected get hookEnable() {
-    return this.config.hookEnable;
+    return this.config.cnpmcore.hookEnable;
   }
 
   protected async addChange(type: string, fullname: string, data: object): Promise<Change> {
@@ -44,7 +44,7 @@ class ChangesStreamEvent {
 }
 
 @Event(PACKAGE_UNPUBLISHED)
-export class PackageUnpublished extends ChangesStreamEvent {
+export class PackageUnpublishedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string) {
     const change = await this.addChange(PACKAGE_UNPUBLISHED, fullname, {});
     if (this.hookEnable) {
@@ -55,7 +55,7 @@ export class PackageUnpublished extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_VERSION_ADDED)
-export class PackageVersionAdded extends ChangesStreamEvent {
+export class PackageVersionAddedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, version: string, tag?: string) {
     const change = await this.addChange(PACKAGE_VERSION_ADDED, fullname, { version });
     if (this.hookEnable) {
@@ -66,7 +66,7 @@ export class PackageVersionAdded extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_VERSION_REMOVED)
-export class PackageVersionRemoved extends ChangesStreamEvent {
+export class PackageVersionRemovedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, version: string, tag?: string) {
     const change = await this.addChange(PACKAGE_VERSION_REMOVED, fullname, { version });
     if (this.hookEnable) {
@@ -77,7 +77,7 @@ export class PackageVersionRemoved extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_TAG_ADDED)
-export class PackageTagAdded extends ChangesStreamEvent {
+export class PackageTagAddedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
     const change = await this.addChange(PACKAGE_TAG_ADDED, fullname, { tag });
     if (this.hookEnable) {
@@ -88,7 +88,7 @@ export class PackageTagAdded extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_TAG_CHANGED)
-export class PackageTagChanged extends ChangesStreamEvent {
+export class PackageTagChangedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
     const change = await this.addChange(PACKAGE_TAG_CHANGED, fullname, { tag });
     if (this.hookEnable) {
@@ -99,7 +99,7 @@ export class PackageTagChanged extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_TAG_REMOVED)
-export class PackageTagRemoved extends ChangesStreamEvent {
+export class PackageTagRemovedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, tag: string) {
     const change = await this.addChange(PACKAGE_TAG_REMOVED, fullname, { tag });
     if (this.hookEnable) {
@@ -110,7 +110,7 @@ export class PackageTagRemoved extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_MAINTAINER_CHANGED)
-export class PackageMaintainerChanged extends ChangesStreamEvent {
+export class PackageMaintainerChangedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, maintainers: User[]) {
     const change = await this.addChange(PACKAGE_MAINTAINER_CHANGED, fullname, {});
     // TODO 应该比较差值，而不是全量推送
@@ -124,7 +124,7 @@ export class PackageMaintainerChanged extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_MAINTAINER_REMOVED)
-export class PackageMaintainerRemoved extends ChangesStreamEvent {
+export class PackageMaintainerRemovedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, maintainer: string) {
     const change = await this.addChange(PACKAGE_MAINTAINER_REMOVED, fullname, { maintainer });
     if (this.hookEnable) {
@@ -135,7 +135,7 @@ export class PackageMaintainerRemoved extends ChangesStreamEvent {
 }
 
 @Event(PACKAGE_META_CHANGED)
-export class PackageMetaChanged extends ChangesStreamEvent {
+export class PackageMetaChangedChangesStreamEvent extends ChangesStreamEvent {
   async handle(fullname: string, meta: PackageMetaChange) {
     const change = await this.addChange(PACKAGE_META_CHANGED, fullname, { ...meta });
     const { deprecateds } = meta;

app/core/event/StoreManifest.ts

@@ -36,7 +36,7 @@ class StoreManifestEvent {
 }
 
 @Event(PACKAGE_VERSION_ADDED)
-export class PackageVersionAdded extends StoreManifestEvent {
+export class PackageVersionAddedStoreManifestEvent extends StoreManifestEvent {
   async handle(fullname: string, version: string) {
     await this.savePackageVersionManifest(fullname, version);
   }

app/core/event/SyncESPackage.ts

@@ -0,0 +1,56 @@
+// TODO sync event
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { EggAppConfig } from 'egg';
+import { Event, Inject } from '@eggjs/tegg';
+import {
+  PACKAGE_UNPUBLISHED,
+  PACKAGE_VERSION_ADDED,
+  PACKAGE_VERSION_REMOVED,
+  PACKAGE_TAG_ADDED,
+  PACKAGE_TAG_CHANGED,
+  PACKAGE_TAG_REMOVED,
+  PACKAGE_MAINTAINER_CHANGED,
+  PACKAGE_MAINTAINER_REMOVED,
+  PACKAGE_META_CHANGED,
+  PACKAGE_BLOCKED,
+  PACKAGE_UNBLOCKED,
+} from './index';
+
+import { PackageSearchService } from '../service/PackageSearchService';
+
+class SyncESPackage {
+  @Inject()
+  protected readonly packageSearchService: PackageSearchService;
+
+  @Inject()
+  protected readonly config: EggAppConfig;
+
+  protected async syncPackage(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.syncPackage(fullname, true);
+  }
+}
+
+@Event(PACKAGE_UNPUBLISHED)
+@Event(PACKAGE_BLOCKED)
+export class PackageUnpublishedSyncESEvent extends SyncESPackage {
+  async handle(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.removePackage(fullname);
+  }
+}
+
+@Event(PACKAGE_VERSION_ADDED)
+@Event(PACKAGE_META_CHANGED)
+@Event(PACKAGE_VERSION_REMOVED)
+@Event(PACKAGE_TAG_ADDED)
+@Event(PACKAGE_TAG_CHANGED)
+@Event(PACKAGE_TAG_REMOVED)
+@Event(PACKAGE_MAINTAINER_CHANGED)
+@Event(PACKAGE_MAINTAINER_REMOVED)
+@Event(PACKAGE_UNBLOCKED)
+export class PackageVersionAddedSyncESEvent extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}

app/core/event/SyncPackageVersionFile.ts

@@ -1,8 +1,9 @@
 import { Event, Inject } from '@eggjs/tegg';
 import {
-  EggAppConfig,
+  EggAppConfig, EggLogger,
 } from 'egg';
-import { PACKAGE_VERSION_ADDED } from './index';
+import { ForbiddenError } from 'egg-errors';
+import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { PackageManagerService } from '../service/PackageManagerService';
 import { PackageVersionFileService } from '../service/PackageVersionFileService';
@@ -11,25 +12,63 @@ class SyncPackageVersionFileEvent {
   @Inject()
   protected readonly config: EggAppConfig;
   @Inject()
+  protected readonly logger: EggLogger;
+  @Inject()
   private readonly packageManagerService: PackageManagerService;
   @Inject()
   private readonly packageVersionFileService: PackageVersionFileService;
 
   protected async syncPackageVersionFile(fullname: string, version: string) {
+    // must set enableUnpkg and enableSyncUnpkgFiles = true both
     if (!this.config.cnpmcore.enableUnpkg) return;
+    if (!this.config.cnpmcore.enableSyncUnpkgFiles) return;
     // ignore sync on unittest
     if (this.config.env === 'unittest' && fullname !== '@cnpm/unittest-unpkg-demo') return;
     const [ scope, name ] = getScopeAndName(fullname);
     const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
       scope, name, version);
     if (!packageVersion) return;
-    await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
+    try {
+      await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
+    } catch (err) {
+      if (err instanceof ForbiddenError) {
+        this.logger.info('[SyncPackageVersionFileEvent.syncPackageVersionFile] ignore sync files, cause: %s',
+          err.message,
+        );
+        return;
+      }
+      throw err;
+    }
+  }
+
+  protected async syncPackageReadmeToLatestVersion(fullname: string) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const { pkg, packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
+      scope, name, 'latest');
+    if (!pkg || !packageVersion) return;
+    await this.packageVersionFileService.syncPackageReadme(pkg, packageVersion);
   }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
-export class PackageVersionAdded extends SyncPackageVersionFileEvent {
+export class PackageVersionAddedSyncPackageVersionFileEvent extends SyncPackageVersionFileEvent {
   async handle(fullname: string, version: string) {
     await this.syncPackageVersionFile(fullname, version);
   }
 }
+
+@Event(PACKAGE_TAG_ADDED)
+export class PackageTagAddedSyncPackageVersionFileEvent extends SyncPackageVersionFileEvent {
+  async handle(fullname: string, tag: string) {
+    if (tag !== 'latest') return;
+    await this.syncPackageReadmeToLatestVersion(fullname);
+  }
+}
+
+@Event(PACKAGE_TAG_CHANGED)
+export class PackageTagChangedSyncPackageVersionFileEvent extends SyncPackageVersionFileEvent {
+  async handle(fullname: string, tag: string) {
+    if (tag !== 'latest') return;
+    await this.syncPackageReadmeToLatestVersion(fullname);
+  }
+}

app/core/service/BinarySyncerService.ts

@@ -9,18 +9,19 @@ import {
   EggHttpClient,
 } from 'egg';
 import fs from 'fs/promises';
+import { sortBy } from 'lodash';
 import binaries, { BinaryName, CategoryName } from '../../../config/binaries';
-import { NFSAdapter } from '../../common/adapter/NFSAdapter';
-import { TaskType, TaskState } from '../../common/enum/Task';
-import { downloadToTempfile } from '../../common/FileUtil';
 import { BinaryRepository } from '../../repository/BinaryRepository';
 import { Task } from '../entity/Task';
 import { Binary } from '../entity/Binary';
 import { TaskService } from './TaskService';
+import { NFSAdapter } from '../../common/adapter/NFSAdapter';
+import { downloadToTempfile } from '../../common/FileUtil';
+import { isTimeoutError } from '../../common/ErrorUtil';
 import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/AbstractBinary';
 import { AbstractService } from '../../common/AbstractService';
-import { TaskRepository } from '../../repository/TaskRepository';
 import { BinaryType } from '../../common/enum/Binary';
+import { TaskType, TaskState } from '../../common/enum/Task';
 
 function isoNow() {
   return new Date().toISOString();
@@ -35,8 +36,6 @@ export class BinarySyncerService extends AbstractService {
   @Inject()
   private readonly taskService: TaskService;
   @Inject()
-  private readonly taskRepository: TaskRepository;
-  @Inject()
   private readonly httpclient: EggHttpClient;
   @Inject()
   private readonly nfsAdapter: NFSAdapter;
@@ -88,13 +87,7 @@ export class BinarySyncerService extends AbstractService {
     return await this.nfsAdapter.getDownloadUrlOrStream(binary.storePath);
   }
 
-  // SyncBinary 由定时任务每台单机定时触发，手动去重
-  // 添加 bizId 在 db 防止重复，记录 id 错误
   public async createTask(binaryName: BinaryName, lastData?: any) {
-    const existsTask = await this.taskRepository.findTaskByTargetName(binaryName, TaskType.SyncBinary);
-    if (existsTask) {
-      return existsTask;
-    }
     try {
       return await this.taskService.createTask(Task.createSyncBinary(binaryName, lastData), false);
     } catch (e) {
@@ -135,24 +128,33 @@ export class BinarySyncerService extends AbstractService {
     this.logger.info('[BinarySyncerService.executeTask:start] taskId: %s, targetName: %s, log: %s',
       task.taskId, task.targetName, logUrl);
     try {
-      await this.syncDir(binaryAdapter, task, '/');
+      const [ hasDownloadError ] = await this.syncDir(binaryAdapter, task, '/');
       logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
       logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 "${binaryName}" 🟢🟢🟢🟢🟢`);
       await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
-      this.logger.info('[BinarySyncerService.executeTask:success] taskId: %s, targetName: %s, log: %s',
-        task.taskId, task.targetName, logUrl);
+      // 确保没有下载异常才算 success
+      await binaryAdapter.finishFetch(!hasDownloadError, binaryName);
+      this.logger.info('[BinarySyncerService.executeTask:success] taskId: %s, targetName: %s, log: %s, hasDownloadError: %s',
+        task.taskId, task.targetName, logUrl, hasDownloadError);
     } catch (err: any) {
-      task.error = err.message;
+      task.error = `${err.name}: ${err.message}`;
       logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`);
       logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`);
-      this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
-        task.taskId, task.targetName, task.error);
-      this.logger.error(err);
+      if (isTimeoutError(err)) {
+        this.logger.warn('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
+          task.taskId, task.targetName, task.error);
+        this.logger.warn(err);
+      } else {
+        this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
+          task.taskId, task.targetName, task.error);
+        this.logger.error(err);
+      }
+      await binaryAdapter.finishFetch(false, binaryName);
       await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
     }
   }
 
-  private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '') {
+  private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') {
     const binaryName = task.targetName as BinaryName;
     const result = await binaryAdapter.fetch(dir, binaryName);
     let hasDownloadError = false;
@@ -160,14 +162,15 @@ export class BinarySyncerService extends AbstractService {
     if (result && result.items.length > 0) {
       hasItems = true;
       let logs: string[] = [];
-      const newItems = await this.diff(binaryName, dir, result.items);
+      const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent);
       logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`);
+      // re-check latest version
       for (const [ index, { item, reason }] of newItems.entries()) {
         if (item.isDir) {
           logs.push(`[${isoNow()}][${dir}] 🚧 [${parentIndex}${index}] Start sync dir ${JSON.stringify(item)}, reason: ${reason}`);
           await this.taskService.appendTaskLog(task, logs.join('\n'));
           logs = [];
-          const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`);
+          const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`, latestVersionDir);
           if (hasError) {
             hasDownloadError = true;
           } else {
@@ -195,7 +198,9 @@ export class BinarySyncerService extends AbstractService {
             const { tmpfile, headers, timing } =
               await downloadToTempfile(
                 this.httpclient, this.config.dataDir, item.sourceUrl!, { ignoreDownloadStatuses: item.ignoreDownloadStatuses });
-            logs.push(`[${isoNow()}][${dir}] 🟢 [${parentIndex}${index}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)}, ${item.sourceUrl} => ${tmpfile}`);
+            const log = `[${isoNow()}][${dir}] 🟢 [${parentIndex}${index}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)}, ${item.sourceUrl} => ${tmpfile}`;
+            logs.push(log);
+            this.logger.info('[BinarySyncerService.syncDir:downloadToTempfile] %s', log);
             localFile = tmpfile;
             const binary = await this.saveBinaryItem(item, tmpfile);
             logs.push(`[${isoNow()}][${dir}] 🟢 [${parentIndex}${index}] Synced file success, binaryId: ${binary.binaryId}`);
@@ -206,7 +211,11 @@ export class BinarySyncerService extends AbstractService {
               this.logger.info('Not found %s, skip it', item.sourceUrl);
               logs.push(`[${isoNow()}][${dir}] 🧪️ [${parentIndex}${index}] Download ${item.sourceUrl} not found, skip it`);
             } else {
-              this.logger.error('Download binary %s %s', item.sourceUrl, err);
+              if (err.name === 'DownloadStatusInvalidError') {
+                this.logger.warn('Download binary %s %s', item.sourceUrl, err);
+              } else {
+                this.logger.error('Download binary %s %s', item.sourceUrl, err);
+              }
               hasDownloadError = true;
               logs.push(`[${isoNow()}][${dir}] ❌ [${parentIndex}${index}] Download ${item.sourceUrl} error: ${err}`);
             }
@@ -222,20 +231,26 @@ export class BinarySyncerService extends AbstractService {
       if (hasDownloadError) {
         logs.push(`[${isoNow()}][${dir}] ❌ Synced dir fail`);
       } else {
-        logs.push(`[${isoNow()}][${dir}] 🟢 Synced dir success`);
+        logs.push(`[${isoNow()}][${dir}] 🟢 Synced dir success, hasItems: ${hasItems}`);
       }
       await this.taskService.appendTaskLog(task, logs.join('\n'));
     }
     return [ hasDownloadError, hasItems ];
   }
 
-  private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[]) {
+
+  // see https://github.com/cnpm/cnpmcore/issues/556
+  // 上游可能正在发布新版本、同步流程中断，导致同步的时候，文件列表不一致
+  // 如果的当前目录命中 latestVersionParent 父目录，那么就再校验一下当前目录
+  // 如果 existsItems 为空或者经过修改，那么就不需要 revalidate 了
+  private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') {
     const existsItems = await this.binaryRepository.listBinaries(binaryName, dir);
     const existsMap = new Map<string, Binary>();
     for (const item of existsItems) {
       existsMap.set(item.name, item);
     }
     const diffItems: { item: Binary; reason: string }[] = [];
+    let latestItem: BinaryItem | undefined;
     for (const item of fetchItems) {
       const existsItem = existsMap.get(item.name);
       if (!existsItem) {
@@ -260,9 +275,26 @@ export class BinarySyncerService extends AbstractService {
         existsItem.sourceUrl = item.url;
         existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses;
         existsItem.date = item.date;
+      } else if (dir.endsWith(latestVersionParent)) {
+        if (!latestItem) {
+          latestItem = sortBy(fetchItems, [ 'date' ]).pop();
+        }
+        const isLatestItem = latestItem?.name === item.name;
+        if (isLatestItem && existsItem.isDir) {
+          diffItems.push({
+            item: existsItem,
+            reason: `revalidate latest version, latest parent dir is ${latestVersionParent}, current dir is ${dir}, current name is ${existsItem.name}`,
+          });
+          latestVersionParent = `${latestVersionParent}${existsItem.name}`;
+        }
       }
     }
-    return diffItems;
+
+
+    return {
+      newItems: diffItems,
+      latestVersionDir: latestVersionParent,
+    };
   }
 
   private async saveBinaryItem(binary: Binary, tmpfile?: string) {
@@ -287,7 +319,7 @@ export class BinarySyncerService extends AbstractService {
     } else {
       binaryAdapter = await this.eggObjectFactory.getEggObject(AbstractBinary, binaryConfig.type);
     }
-    await binaryAdapter.init(binaryName);
+    await binaryAdapter.initFetch(binaryName);
     return binaryAdapter;
   }
 }

app/core/service/BugVersionService.ts

@@ -2,10 +2,12 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import pMap from 'p-map';
 import { BugVersion } from '../entity/BugVersion';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageJSONType, PackageRepository } from '../../repository/PackageRepository';
 import { DistRepository } from '../../repository/DistRepository';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { CacheService } from './CacheService';
+import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
+import { BugVersionStore } from '../../common/adapter/BugVersionStore';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -23,6 +25,27 @@ export class BugVersionService {
   @Inject()
   private readonly cacheService: CacheService;
 
+  @Inject()
+  private readonly bugVersionStore: BugVersionStore;
+
+  async getBugVersion(): Promise<BugVersion | undefined> {
+    // TODO performance problem, cache bugVersion and update with schedule
+    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
+    if (!pkg) return;
+    /* c8 ignore next 10 */
+    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
+    if (!tag) return;
+    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
+    if (!bugVersion) {
+      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
+      if (!packageVersionJson) return;
+      const data = packageVersionJson.config?.['bug-versions'];
+      bugVersion = new BugVersion(data || {});
+      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
+    }
+    return bugVersion;
+  }
+
   async cleanBugVersionPackageCaches(bugVersion: BugVersion) {
     const fullnames = bugVersion.listAllPackagesHasBugs();
     await pMap(fullnames, async fullname => {

app/core/service/ChangesStreamService.ts

@@ -6,20 +6,21 @@ import {
   EggObjectFactory,
   Inject,
 } from '@eggjs/tegg';
-import { TaskState, TaskType } from '../../common/enum/Task';
-import { AbstractService } from '../../common/AbstractService';
-import { TaskRepository } from '../../repository/TaskRepository';
-import { HOST_NAME, ChangesStreamTask, Task } from '../entity/Task';
+import { E500 } from 'egg-errors';
 import { PackageSyncerService, RegistryNotMatchError } from './PackageSyncerService';
 import { TaskService } from './TaskService';
 import { RegistryManagerService } from './RegistryManagerService';
-import { E500 } from 'egg-errors';
+import { ScopeManagerService } from './ScopeManagerService';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { TaskRepository } from '../../repository/TaskRepository';
+import { HOST_NAME, ChangesStreamTask, Task } from '../entity/Task';
 import { Registry } from '../entity/Registry';
 import { AbstractChangeStream } from '../../common/adapter/changesStream/AbstractChangesStream';
 import { getScopeAndName } from '../../common/PackageUtil';
+import { isTimeoutError } from '../../common/ErrorUtil';
 import { GLOBAL_WORKER } from '../../common/constants';
-import { ScopeManagerService } from './ScopeManagerService';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { TaskState, TaskType } from '../../common/enum/Task';
+import { AbstractService } from '../../common/AbstractService';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -101,8 +102,12 @@ export class ChangesStreamService extends AbstractService {
         await setTimeout(this.config.cnpmcore.checkChangesStreamInterval);
       }
     } catch (err) {
-      this.logger.error('[ChangesStreamService.executeTask:error] %s, exit now', err);
-      this.logger.error(err);
+      this.logger.warn('[ChangesStreamService.executeTask:error] %s, exit now', err.message);
+      if (isTimeoutError(err)) {
+        this.logger.warn(err);
+      } else {
+        this.logger.error(err);
+      }
       task.error = `${err}`;
       await this.taskRepository.saveTask(task);
       await this.suspendSync();

app/core/service/FixNoPaddingVersionService.ts

@@ -0,0 +1,33 @@
+import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
+import { EggLogger } from 'egg';
+import pMap from 'p-map';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
+import { PaddingSemVer } from '../entity/PaddingSemVer';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class FixNoPaddingVersionService {
+  @Inject()
+  private readonly packageVersionRepository: PackageVersionRepository;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async fixPaddingVersion(id?: number): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const packageVersions = await this.packageVersionRepository.findHaveNotPaddingVersion(id);
+      if (packageVersions.length === 0) {
+        break;
+      }
+      id = packageVersions[packageVersions.length - 1].id as unknown as number + 1;
+      this.logger.info('[FixNoPaddingVersionService] fix padding version ids %j', packageVersions.map(t => t.id));
+
+      await pMap(packageVersions, async packageVersion => {
+        const paddingSemver = new PaddingSemVer(packageVersion.version);
+        await this.packageVersionRepository.fixPaddingVersion(packageVersion.packageVersionId, paddingSemver);
+      }, { concurrency: 30 });
+    }
+  }
+}

app/core/service/HomeService.ts

@@ -0,0 +1,19 @@
+import {
+  AccessLevel,
+  SingletonProto,
+} from '@eggjs/tegg';
+import { AbstractService } from '../../common/AbstractService';
+import { NOT_IMPLEMENTED_PATH } from '../../common/constants';
+import { NotFoundError, NotImplementedError } from 'egg-errors';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HomeService extends AbstractService {
+  async misc(path: string) {
+    if (NOT_IMPLEMENTED_PATH.includes(path)) {
+      throw new NotImplementedError(`${path} not implemented yet`);
+    }
+    throw new NotFoundError(`${path} not found`);
+  }
+}

app/core/service/PackageManagerService.ts

@@ -1,13 +1,16 @@
-import { stat } from 'fs/promises';
+import { stat, readFile } from 'node:fs/promises';
+import { strict as assert } from 'node:assert';
 import {
   AccessLevel,
   SingletonProto,
   EventBus,
   Inject,
 } from '@eggjs/tegg';
-import { ForbiddenError } from 'egg-errors';
+import { BadRequestError, ForbiddenError, NotFoundError } from 'egg-errors';
 import { RequireAtLeastOne } from 'type-fest';
+import npa from 'npm-package-arg';
 import semver from 'semver';
+import pMap from 'p-map';
 import {
   calculateIntegrity,
   detectInstallScript,
@@ -17,12 +20,11 @@ import {
   hasShrinkWrapInTgz,
 } from '../../common/PackageUtil';
 import { AbstractService } from '../../common/AbstractService';
-import { BugVersionStore } from '../../common/adapter/BugVersionStore';
-import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
 import { AbbreviatedPackageJSONType, AbbreviatedPackageManifestType, PackageJSONType, PackageManifestType, PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { DistRepository } from '../../repository/DistRepository';
+import { isDuplicateKeyError } from '../../repository/util/ErrorUtil';
 import { Package } from '../entity/Package';
 import { PackageVersion } from '../entity/PackageVersion';
 import { PackageVersionBlock } from '../entity/PackageVersionBlock';
@@ -46,6 +48,7 @@ import { BugVersionService } from './BugVersionService';
 import { BugVersion } from '../entity/BugVersion';
 import { RegistryManagerService } from './RegistryManagerService';
 import { Registry } from '../entity/Registry';
+import { PackageVersionService } from './PackageVersionService';
 
 export interface PublishPackageCmd {
   // maintainer: Maintainer;
@@ -53,7 +56,7 @@ export interface PublishPackageCmd {
   // name don't include scope
   name: string;
   version: string;
-  description: string;
+  description?: string;
   packageJson: PackageJSONType;
   registryId?: string;
   readme: string;
@@ -64,7 +67,7 @@ export interface PublishPackageCmd {
     // sync worker will use localFile field
     localFile?: string;
   }, 'content' | 'localFile'>;
-  tag?: string;
+  tags?: string[];
   isPrivate: boolean;
   // only use on sync package
   publishTime?: Date;
@@ -91,30 +94,33 @@ export class PackageManagerService extends AbstractService {
   @Inject()
   private readonly bugVersionService: BugVersionService;
   @Inject()
-  private readonly bugVersionStore: BugVersionStore;
-  @Inject()
   private readonly distRepository: DistRepository;
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly packageVersionService: PackageVersionService;
 
   private static downloadCounters = {};
 
   // support user publish private package and sync worker publish public package
   async publish(cmd: PublishPackageCmd, publisher: User) {
+    if (this.config.cnpmcore.strictValidatePackageDeps) {
+      await this._checkPackageDepsVersion(cmd.packageJson);
+    }
     let pkg = await this.packageRepository.findPackage(cmd.scope, cmd.name);
     if (!pkg) {
       pkg = Package.create({
         scope: cmd.scope,
         name: cmd.name,
         isPrivate: cmd.isPrivate,
-        description: cmd.description,
+        description: cmd.description || '',
         registryId: cmd.registryId,
       });
     } else {
       // update description
       // will read database twice to update description by model to entity and entity to model
       if (pkg.description !== cmd.description) {
-        pkg.description = cmd.description;
+        pkg.description = cmd.description || '';
       }
 
       /* c8 ignore next 3 */
@@ -155,17 +161,17 @@ export class PackageManagerService extends AbstractService {
       cmd.packageJson._hasShrinkwrap = await hasShrinkWrapInTgz(cmd.dist.content || cmd.dist.localFile!);
     }
 
+    // set _npmUser field to cmd.packageJson
+    cmd.packageJson._npmUser = {
+      // clean user scope prefix
+      name: publisher.displayName,
+      email: publisher.email,
+    };
+
     // add _registry_name field to cmd.packageJson
-    if (!cmd.packageJson._source_registry_name) {
-      let registry: Registry | null;
-      if (cmd.registryId) {
-        registry = await this.registryManagerService.findByRegistryId(cmd.registryId);
-      } else {
-        registry = await this.registryManagerService.ensureDefaultRegistry();
-      }
-      if (registry) {
-        cmd.packageJson._source_registry_name = registry.name;
-      }
+    const registry = await this.getSourceRegistry(pkg);
+    if (registry) {
+      cmd.packageJson._source_registry_name = registry.name;
     }
 
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
@@ -201,26 +207,30 @@ export class PackageManagerService extends AbstractService {
       integrity: tarDistIntegrity.integrity,
     };
 
+    // https://github.com/npm/registry/blob/main/docs/responses/package-metadata.md#abbreviated-version-object
+    // Abbreviated version object
     const abbreviated = JSON.stringify({
       name: cmd.packageJson.name,
       version: cmd.packageJson.version,
       deprecated: cmd.packageJson.deprecated,
       dependencies: cmd.packageJson.dependencies,
+      acceptDependencies: cmd.packageJson.acceptDependencies,
       optionalDependencies: cmd.packageJson.optionalDependencies,
       devDependencies: cmd.packageJson.devDependencies,
       bundleDependencies: cmd.packageJson.bundleDependencies,
       peerDependencies: cmd.packageJson.peerDependencies,
       peerDependenciesMeta: cmd.packageJson.peerDependenciesMeta,
       bin: cmd.packageJson.bin,
+      directories: cmd.packageJson.directories,
       os: cmd.packageJson.os,
       cpu: cmd.packageJson.cpu,
       libc: cmd.packageJson.libc,
       workspaces: cmd.packageJson.workspaces,
-      directories: cmd.packageJson.directories,
       dist: cmd.packageJson.dist,
       engines: cmd.packageJson.engines,
       _hasShrinkwrap: cmd.packageJson._hasShrinkwrap,
       hasInstallScript,
+      funding: cmd.packageJson.funding,
       // https://github.com/cnpm/npminstall/blob/13efc7eec21a61e509226e3772bfb75cd5605612/lib/install_package.js#L176
       // npminstall require publish time to show the recently update versions
       publish_time: cmd.packageJson.publish_time,
@@ -262,7 +272,7 @@ export class PackageManagerService extends AbstractService {
     try {
       await this.packageRepository.createPackageVersion(pkgVersion);
     } catch (e) {
-      if (e.code === 'ER_DUP_ENTRY') {
+      if (isDuplicateKeyError(e)) {
         throw new ForbiddenError(`Can't modify pre-existing version: ${pkg.fullname}@${cmd.version}`);
       }
       throw e;
@@ -270,13 +280,27 @@ export class PackageManagerService extends AbstractService {
     if (cmd.skipRefreshPackageManifests !== true) {
       await this.refreshPackageChangeVersionsToDists(pkg, [ pkgVersion.version ]);
     }
-    if (cmd.tag) {
-      await this.savePackageTag(pkg, cmd.tag, cmd.version, true);
+    if (cmd.tags) {
+      for (const tag of cmd.tags) {
+        await this.savePackageTag(pkg, tag, cmd.version, true);
+        this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, tag);
+      }
+    } else {
+      this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, undefined);
     }
-    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, cmd.tag);
+
     return pkgVersion;
   }
 
+  async blockPackageByFullname(name: string, reason: string) {
+    const [ scope, pkgName ] = getScopeAndName(name);
+    const pkg = await this.packageRepository.findPackage(scope, pkgName);
+    if (!pkg) {
+      throw new NotFoundError(`Package name(${name}) not found`);
+    }
+    return await this.blockPackage(pkg, reason);
+  }
+
   async blockPackage(pkg: Package, reason: string) {
     let block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
     if (block) {
@@ -306,6 +330,15 @@ export class PackageManagerService extends AbstractService {
     return block;
   }
 
+  async unblockPackageByFullname(name: string) {
+    const [ scope, pkgName ] = getScopeAndName(name);
+    const pkg = await this.packageRepository.findPackage(scope, pkgName);
+    if (!pkg) {
+      throw new NotFoundError(`Package name(${name}) not found`);
+    }
+    return await this.unblockPackage(pkg);
+  }
+
   async unblockPackage(pkg: Package) {
     const block = await this.packageVersionBlockRepository.findPackageVersionBlock(pkg.packageId, '*');
     if (block) {
@@ -327,9 +360,9 @@ export class PackageManagerService extends AbstractService {
     }
   }
 
-  async replacePackageMaintainers(pkg: Package, maintainers: User[]) {
+  async replacePackageMaintainersAndDist(pkg: Package, maintainers: User[]) {
     await this.packageRepository.replacePackageMaintainers(pkg.packageId, maintainers.map(m => m.userId));
-    await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
+    await this.refreshPackageMaintainersToDists(pkg);
     this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
   }
 
@@ -342,14 +375,12 @@ export class PackageManagerService extends AbstractService {
       }
     }
     if (hasNewRecord) {
-      await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
       this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
     }
   }
 
   async removePackageMaintainer(pkg: Package, maintainer: User) {
     await this.packageRepository.removePackageMaintainer(pkg.packageId, maintainer.userId);
-    await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
     this.eventBus.emit(PACKAGE_MAINTAINER_REMOVED, pkg.fullname, maintainer.name);
   }
 
@@ -369,7 +400,7 @@ export class PackageManagerService extends AbstractService {
     return await this._listPackageFullOrAbbreviatedManifests(scope, name, false, isSync);
   }
 
-  async showPackageVersionByVersionOrTag(scope: string, name: string, versionOrTag: string): Promise<{
+  async showPackageVersionByVersionOrTag(scope: string, name: string, spec: string): Promise<{
     blockReason?: string,
     pkg?: Package,
     packageVersion?: PackageVersion | null,
@@ -380,40 +411,27 @@ export class PackageManagerService extends AbstractService {
     if (block) {
       return { blockReason: block.reason, pkg };
     }
-    let version = versionOrTag;
-    if (!semver.valid(versionOrTag)) {
-      // invalid version, versionOrTag is a tag
-      const packageTag = await this.packageRepository.findPackageTag(pkg.packageId, versionOrTag);
-      if (packageTag) {
-        version = packageTag.version;
-      }
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const version = await this.packageVersionService.getVersion(result);
+    if (!version) {
+      return {};
     }
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
     return { packageVersion, pkg };
   }
 
-  async showPackageVersionManifest(scope: string, name: string, versionOrTag: string, isSync = false) {
-    let manifest;
-    const { blockReason, packageVersion, pkg } = await this.showPackageVersionByVersionOrTag(scope, name, versionOrTag);
-    if (blockReason) {
-      return {
-        blockReason,
-        manifest,
-        pkg,
-      };
+  async showPackageVersionManifest(scope: string, name: string, spec: string, isSync = false, isFullManifests = false) {
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) return {};
+    const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
+    if (block) {
+      return { blockReason: block.reason, pkg };
     }
-    if (!packageVersion) return { manifest: null, blockReason, pkg };
-    manifest = await this.distRepository.findPackageVersionManifest(packageVersion.packageId, packageVersion.version);
-    let bugVersion: BugVersion | undefined;
-    // sync mode response no bug version fixed
-    if (!isSync) {
-      bugVersion = await this.getBugVersion();
-    }
-    if (bugVersion) {
-      const fullname = getFullname(scope, name);
-      manifest = await this.bugVersionService.fixPackageBugVersion(bugVersion, fullname, manifest);
-    }
-    return { manifest, blockReason, pkg };
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const manifest = await this.packageVersionService.readManifest(pkg.packageId, result, isFullManifests, !isSync);
+    return { manifest, blockReason: null, pkg };
   }
 
   async downloadPackageVersionTar(packageVersion: PackageVersion) {
@@ -422,7 +440,7 @@ export class PackageManagerService extends AbstractService {
 
   public plusPackageVersionCounter(fullname: string, version: string) {
     // set counter + 1, schedule will store them into database
-    const counters = PackageManagerService.downloadCounters;
+    const counters: Record<string, Record<string, number>> = PackageManagerService.downloadCounters;
     if (!counters[fullname]) counters[fullname] = {};
     counters[fullname][version] = (counters[fullname][version] || 0) + 1;
     // Total
@@ -441,7 +459,7 @@ export class PackageManagerService extends AbstractService {
   // will be call by schedule/SavePackageVersionDownloadCounter.ts
   async savePackageVersionCounters() {
     // { [fullname]: { [version]: number } }
-    const counters = PackageManagerService.downloadCounters;
+    const counters: Record<string, Record<string, number>> = PackageManagerService.downloadCounters;
     const fullnames = Object.keys(counters);
     if (fullnames.length === 0) return;
 
@@ -492,6 +510,25 @@ export class PackageManagerService extends AbstractService {
     await this._mergeManifestDist(pkgVersion.abbreviatedDist, mergeAbbreviated);
   }
 
+  /**
+   * save package version readme
+   */
+  public async savePackageVersionReadme(pkgVersion: PackageVersion, readmeFile: string) {
+    await this.distRepository.saveDist(pkgVersion.readmeDist, readmeFile);
+    this.logger.info('[PackageManagerService.savePackageVersionReadme] save packageVersionId:%s readme:%s to dist:%s',
+      pkgVersion.packageVersionId, readmeFile, pkgVersion.readmeDist.distId);
+  }
+
+  public async savePackageReadme(pkg: Package, readmeFile: string) {
+    if (!pkg.manifestsDist) return;
+    const fullManifests = await this.distRepository.readDistBytesToJSON<PackageManifestType>(pkg.manifestsDist);
+    if (!fullManifests) return;
+    fullManifests.readme = await readFile(readmeFile, 'utf-8');
+    await this._updatePackageManifestsToDists(pkg, fullManifests, null);
+    this.logger.info('[PackageManagerService.savePackageReadme] save packageId:%s readme, size: %s',
+      pkg.packageId, fullManifests.readme.length);
+  }
+
   private async _removePackageVersionAndDist(pkgVersion: PackageVersion) {
     // remove nfs dists
     await Promise.all([
@@ -504,10 +541,10 @@ export class PackageManagerService extends AbstractService {
     await this.packageRepository.removePackageVersion(pkgVersion);
   }
 
-  public async unpublishPackage(pkg: Package, forceRefresh = false) {
+  public async unpublishPackage(pkg: Package) {
     const pkgVersions = await this.packageRepository.listPackageVersions(pkg.packageId);
     // already unpublished
-    if (pkgVersions.length === 0 && !forceRefresh) {
+    if (pkgVersions.length === 0) {
       this.logger.info(`[packageManagerService.unpublishPackage:skip] ${pkg.packageId} already unpublished`);
       return;
     }
@@ -533,8 +570,14 @@ export class PackageManagerService extends AbstractService {
   }
 
   public async removePackageVersion(pkg: Package, pkgVersion: PackageVersion, skipRefreshPackageManifests = false) {
+    const currentVersions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
+    // only one version, unpublish the package
+    if (currentVersions.length === 1 && currentVersions[0] === pkgVersion.version) {
+      await this.unpublishPackage(pkg);
+      return;
+    }
+    // remove version & update tags
     await this._removePackageVersionAndDist(pkgVersion);
-    // all versions removed
     const versions = await this.packageRepository.listPackageVersionNames(pkg.packageId);
     if (versions.length > 0) {
       let updateTag: string | undefined;
@@ -555,8 +598,6 @@ export class PackageManagerService extends AbstractService {
       }
       return;
     }
-    // unpublish
-    await this.unpublishPackage(pkg, true);
   }
 
   public async savePackageTag(pkg: Package, tag: string, version: string, skipEvent = false) {
@@ -640,22 +681,14 @@ export class PackageManagerService extends AbstractService {
     await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
   }
 
-  async getBugVersion(): Promise<BugVersion | undefined> {
-    // TODO performance problem, cache bugVersion and update with schedule
-    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
-    if (!pkg) return;
-    /* c8 ignore next 10 */
-    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
-    if (!tag) return;
-    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
-    if (!bugVersion) {
-      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
-      if (!packageVersionJson) return;
-      const data = packageVersionJson.config?.['bug-versions'];
-      bugVersion = new BugVersion(data);
-      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
+  async getSourceRegistry(pkg: Package): Promise<Registry | null> {
+    let registry: Registry | null;
+    if (pkg.registryId) {
+      registry = await this.registryManagerService.findByRegistryId(pkg.registryId);
+    } else {
+      registry = await this.registryManagerService.ensureDefaultRegistry();
     }
-    return bugVersion;
+    return registry;
   }
 
   private async _listPackageDistTags(pkg: Package) {
@@ -706,13 +739,16 @@ export class PackageManagerService extends AbstractService {
     const fieldsFromLatestManifest = [
       'author', 'bugs', 'contributors', 'description', 'homepage', 'keywords', 'license',
       'readmeFilename', 'repository',
-    ];
+    ] as const;
     // the latest version metas
     for (const field of fieldsFromLatestManifest) {
-      fullManifests[field] = latestManifest[field];
+      if (latestManifest[field]) {
+        (fullManifests as Record<string, unknown>)[field] = latestManifest[field];
+      }
     }
   }
 
+
   private async _setPackageDistTagsAndLatestInfos(pkg: Package, fullManifests: PackageManifestType, abbreviatedManifests: AbbreviatedPackageManifestType) {
     const distTags = await this._listPackageDistTags(pkg);
     if (distTags.latest) {
@@ -789,6 +825,7 @@ export class PackageManagerService extends AbstractService {
     let blockReason = '';
     const pkg = await this.packageRepository.findPackage(scope, name);
     if (!pkg) return { etag, data: null, blockReason };
+    const registry = await this.getSourceRegistry(pkg);
 
     const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
     if (block) {
@@ -798,7 +835,7 @@ export class PackageManagerService extends AbstractService {
     let bugVersion: BugVersion | undefined;
     // sync mode response no bug version fixed
     if (!isSync) {
-      bugVersion = await this.getBugVersion();
+      bugVersion = await this.bugVersionService.getBugVersion();
     }
     const fullname = getFullname(scope, name);
 
@@ -810,6 +847,11 @@ export class PackageManagerService extends AbstractService {
       if (bugVersion) {
         await this.bugVersionService.fixPackageBugVersions(bugVersion, fullname, data.versions);
       }
+      // set _source_registry_name in full manifestDist
+      if (registry) {
+        data._source_registry_name = registry?.name;
+      }
+
       const distBytes = Buffer.from(JSON.stringify(data));
       const distIntegrity = await calculateIntegrity(distBytes);
       etag = `"${distIntegrity.shasum}"`;
@@ -850,8 +892,9 @@ export class PackageManagerService extends AbstractService {
 
     const distTags = await this._listPackageDistTags(pkg);
     const maintainers = await this._listPackageMaintainers(pkg);
+    const registry = await this.getSourceRegistry(pkg);
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
-    const data:PackageManifestType = {
+    const data: PackageManifestType = {
       _id: `${pkg.fullname}`,
       _rev: `${pkg.id}-${pkg.packageId}`,
       'dist-tags': distTags,
@@ -884,6 +927,7 @@ export class PackageManagerService extends AbstractService {
       // as given in package.json, for the latest version
       repository: undefined,
       // users: an object whose keys are the npm user names of people who have starred this package
+      _source_registry_name: registry?.name,
     };
 
     let latestTagVersion = '';
@@ -940,4 +984,25 @@ export class PackageManagerService extends AbstractService {
     }
     return data;
   }
+
+  private async _checkPackageDepsVersion(pkgJSON: PackageJSONType) {
+    // 只校验 dependencies
+    // devDependencies、optionalDependencies、peerDependencies 不会影响依赖安装 不在这里进行校验
+    const { dependencies } = pkgJSON;
+    await pMap(Object.entries(dependencies || {}), async ([ fullname, spec ]) => {
+      try {
+        const specResult = npa(`${fullname}@${spec}`);
+        // 对于 git、alias、file 等类型的依赖，不进行版本校验
+        if (![ 'range', 'tag', 'version' ].includes(specResult.type)) {
+          return;
+        }
+        const pkgVersion = await this.packageVersionService.getVersion(npa(`${fullname}@${spec}`));
+        assert(pkgVersion);
+      } catch (e) {
+        throw new BadRequestError(`deps ${fullname}@${spec} not found`);
+      }
+    }, {
+      concurrency: 12,
+    });
+  }
 }

app/core/service/PackageSearchService.ts

@@ -0,0 +1,259 @@
+import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
+import { estypes, errors } from '@elastic/elasticsearch';
+import dayjs from 'dayjs';
+
+import { AbstractService } from '../../common/AbstractService';
+import { formatAuthor, getScopeAndName } from '../../common/PackageUtil';
+import { PackageManagerService } from './PackageManagerService';
+import { SearchManifestType, SearchMappingType, SearchRepository } from '../../repository/SearchRepository';
+import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
+
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class PackageSearchService extends AbstractService {
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
+  @Inject()
+  private readonly searchRepository: SearchRepository;
+  @Inject()
+  private packageVersionDownloadRepository: PackageVersionDownloadRepository;
+  @Inject()
+  protected packageRepository: PackageRepository;
+  @Inject()
+  protected packageVersionBlockRepository: PackageVersionBlockRepository;
+
+  async syncPackage(fullname: string, isSync = true) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const fullManifests = await this.packageManagerService.listPackageFullManifests(scope, name, isSync);
+
+    if (!fullManifests.data) {
+      this.logger.warn('[PackageSearchService.syncPackage] save package:%s not found', fullname);
+      return;
+    }
+
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      this.logger.warn('[PackageSearchService.syncPackage] findPackage:%s not found', fullname);
+      return;
+    }
+
+    const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
+    if (block) {
+      this.logger.warn('[PackageSearchService.syncPackage] package:%s is blocked, try to remove es', fullname);
+      await this.removePackage(fullname);
+      return;
+    }
+
+    // get last year download data
+    const startDate = dayjs().subtract(1, 'year');
+    const endDate = dayjs();
+
+    const entities = await this.packageVersionDownloadRepository.query(pkg.packageId, startDate.toDate(), endDate.toDate());
+    let downloadsAll = 0;
+    for (const entity of entities) {
+      for (let i = 1; i <= 31; i++) {
+        const day = String(i).padStart(2, '0');
+        const field = `d${day}`;
+        const counter = entity[field];
+        if (!counter) continue;
+        downloadsAll += counter;
+      }
+    }
+
+    const { data: manifest } = fullManifests;
+
+    const latestVersion = manifest['dist-tags'].latest;
+    const latestManifest = manifest.versions[latestVersion];
+
+    const packageDoc: SearchMappingType = {
+      name: manifest.name,
+      version: latestVersion,
+      _rev: manifest._rev,
+      scope: scope ? scope.replace('@', '') : 'unscoped',
+      keywords: manifest.keywords || [],
+      versions: Object.keys(manifest.versions),
+      description: manifest.description,
+      license: typeof manifest.license === 'object' ? manifest.license?.type : manifest.license,
+      maintainers: manifest.maintainers,
+      author: formatAuthor(manifest.author),
+      'dist-tags': manifest['dist-tags'],
+      date: manifest.time[latestVersion],
+      created: manifest.time.created,
+      modified: manifest.time.modified,
+      // 归属 registry，keywords 枚举值
+      _source_registry_name: manifest._source_registry_name,
+      // 最新版本发布人 _npmUser:
+      _npmUser: latestManifest?._npmUser,
+      // 最新版本发布信息
+      publish_time: latestManifest?.publish_time,
+    };
+
+    // http://npmmirror.com/package/npm/files/lib/utils/format-search-stream.js#L147-L148
+    // npm cli 使用 username 字段
+    if (packageDoc.maintainers) {
+      packageDoc.maintainers = packageDoc.maintainers.map(maintainer => {
+        return {
+          username: maintainer.name,
+          ...maintainer,
+        };
+      });
+    }
+
+    const document: SearchManifestType = {
+      package: packageDoc,
+      downloads: {
+        all: downloadsAll,
+      },
+    };
+
+    return await this.searchRepository.upsertPackage(document);
+  }
+
+  async searchPackage(text: string, from: number, size: number): Promise<{ objects: (SearchManifestType | undefined)[], total: number }> {
+    const matchQueries = this._buildMatchQueries(text);
+    const scriptScore = this._buildScriptScore({
+      text,
+      scoreEffect: 0.25,
+    });
+
+    const res = await this.searchRepository.searchPackage({
+      body: {
+        size,
+        from,
+        query: {
+          function_score: {
+            boost_mode: 'replace',
+            query: {
+              bool: {
+                should: matchQueries,
+                minimum_should_match: matchQueries.length ? 1 : 0,
+              },
+            },
+            script_score: scriptScore,
+          },
+        },
+      },
+    });
+    const { hits, total } = res;
+    return {
+      objects: hits?.map(item => {
+        // 从 https://github.com/npm/cli/pull/7407 (npm cli v10.6.0) 开始，npm cli 使用 publisher 字段(以前使用 maintainers 字段)
+        // 从现有数据来看，_npmUser 字段和 publisher 字段是等价的
+        // 为了兼容老版本，不删除 _npmUser 字段
+        if (!item._source?.package.publisher && item._source?.package._npmUser) {
+          item._source.package.publisher = {
+            username: item._source.package._npmUser.name,
+            email: item._source.package._npmUser.email,
+          };
+        }
+
+        return item._source;
+      }),
+      total: (total as estypes.SearchTotalHits).value,
+    };
+  }
+
+  async removePackage(fullname: string) {
+    try {
+      return await this.searchRepository.removePackage(fullname);
+    } catch (error) {
+      // if the package does not exist, returns success
+      if (error instanceof errors.ResponseError && error?.statusCode === 404) {
+        this.logger.warn('[PackageSearchService.removePackage] remove package:%s not found', fullname);
+        return fullname;
+      }
+      throw error;
+    }
+  }
+
+  // https://github.com/npms-io/queries/blob/master/lib/search.js#L8C1-L78C2
+  private _buildMatchQueries(text: string) {
+    if (!text) {
+      return [];
+    }
+    return [
+      // Standard match using cross_fields
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.standard^4',
+            'package.description.standard',
+            'package.keywords.standard^2',
+          ],
+          type: 'cross_fields',
+          boost: 6,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Partial match using edge-ngram
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.edge_ngram^4',
+            'package.description.edge_ngram',
+            'package.keywords.edge_ngram^2',
+          ],
+          type: 'phrase',
+          slop: 3,
+          boost: 3,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Normal term match with an english stemmer
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.english_docs^4',
+            'package.description.english_docs',
+            'package.keywords.english_docs^2',
+          ],
+          type: 'cross_fields',
+          boost: 3,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Normal term match with a more aggressive english stemmer (not so important)
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.english_aggressive_docs^4',
+            'package.description.english_aggressive_docs',
+            'package.keywords.english_aggressive_docs^2',
+          ],
+          type: 'cross_fields',
+          tie_breaker: 0.5,
+        },
+      },
+    ];
+  }
+
+  private _buildScriptScore(params: { text: string | undefined, scoreEffect: number }) {
+    // keep search simple, only download(popularity)
+    const downloads = 'doc["downloads.all"].value';
+    const source = `doc["package.name.raw"].value.equals(params.text) ? 100000 + ${downloads} : _score * Math.pow(${downloads}, params.scoreEffect)`;
+    return {
+      script: {
+        source,
+        params: {
+          text: params.text || '',
+          scoreEffect: params.scoreEffect,
+        },
+      },
+    };
+  }
+}

app/core/service/PackageSyncerService.ts

@@ -1,15 +1,14 @@
-import os from 'os';
+import os from 'node:os';
+import { setTimeout } from 'node:timers/promises';
+import { rm } from 'node:fs/promises';
 import {
   AccessLevel,
   SingletonProto,
   Inject,
 } from '@eggjs/tegg';
 import { Pointcut } from '@eggjs/tegg/aop';
-import {
-  EggContextHttpClient,
-} from 'egg';
-import { setTimeout } from 'timers/promises';
-import { rm } from 'fs/promises';
+import { EggHttpClient } from 'egg';
+import { isEqual, isEmpty } from 'lodash';
 import semver from 'semver';
 import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
 import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
@@ -17,7 +16,7 @@ import { downloadToTempfile } from '../../common/FileUtil';
 import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageJSONType, PackageManifestType, PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { UserRepository } from '../../repository/UserRepository';
 import { Task, SyncPackageTaskOptions, CreateSyncPackageTask } from '../entity/Task';
@@ -32,7 +31,7 @@ import { Registry } from '../entity/Registry';
 import { BadRequestError } from 'egg-errors';
 import { ScopeManagerService } from './ScopeManagerService';
 import { EventCorkAdvice } from './EventCorkerAdvice';
-import { SyncDeleteMode } from '../../common/constants';
+import { PresetRegistryName, SyncDeleteMode } from '../../common/constants';
 
 type syncDeletePkgOptions = {
   task: Task,
@@ -73,7 +72,7 @@ export class PackageSyncerService extends AbstractService {
   @Inject()
   private readonly cacheService: CacheService;
   @Inject()
-  private readonly httpclient: EggContextHttpClient;
+  private readonly httpclient: EggHttpClient;
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
   @Inject()
@@ -115,17 +114,18 @@ export class PackageSyncerService extends AbstractService {
     if (!this.allowSyncDownloadData) {
       return;
     }
+
     const fullname = pkg.fullname;
     const start = '2011-01-01';
     const end = this.config.cnpmcore.syncDownloadDataMaxDate;
     const registry = this.config.cnpmcore.syncDownloadDataSourceRegistry;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     const logs: string[] = [];
     let downloads: { day: string; downloads: number }[];
 
     logs.push(`[${isoNow()}][DownloadData] 🚧🚧🚧🚧🚧 Syncing "${fullname}" download data "${start}:${end}" on ${registry} 🚧🚧🚧🚧🚧`);
     const failEnd = '❌❌❌❌❌ 🚮 give up 🚮 ❌❌❌❌❌';
     try {
-      const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
       const { data, status, res } = await this.npmRegistry.getDownloadRanges(registry, fullname, start, end, { remoteAuthToken });
       downloads = data.downloads || [];
       logs.push(`[${isoNow()}][DownloadData] 🚧 HTTP [${status}] timing: ${JSON.stringify(res.timing)}, downloads: ${downloads.length}`);
@@ -162,7 +162,7 @@ export class PackageSyncerService extends AbstractService {
   private async syncUpstream(task: Task) {
     const registry = this.npmRegistry.registry;
     const fullname = task.targetName;
-    const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     let logs: string[] = [];
     let logId = '';
     logs.push(`[${isoNow()}][UP] 🚧🚧🚧🚧🚧 Waiting sync "${fullname}" task on ${registry} 🚧🚧🚧🚧🚧`);
@@ -173,7 +173,9 @@ export class PackageSyncerService extends AbstractService {
       logId = data.logId;
     } catch (err: any) {
       const status = err.status || 'unknow';
-      logs.push(`[${isoNow()}][UP] ❌ Sync ${fullname} fail, create sync task error: ${err}, status: ${status}`);
+      // 可能会抛出 AggregateError 异常
+      // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AggregateError
+      logs.push(`[${isoNow()}][UP] ❌ Sync ${fullname} fail, create sync task error: ${err}, status: ${status} ${err instanceof AggregateError ? err.errors : ''}`);
       logs.push(`[${isoNow()}][UP] ${failEnd}`);
       await this.taskService.appendTaskLog(task, logs.join('\n'));
       return;
@@ -202,8 +204,8 @@ export class PackageSyncerService extends AbstractService {
         const log = data && data.log || '';
         offset += log.length;
         if (data && data.syncDone) {
-          logs.push(`[${isoNow()}][UP] 🟢 Sync ${fullname} success [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
-          logs.push(`[${isoNow()}][UP] 🟢🟢🟢🟢🟢 ${registry}/${fullname} 🟢🟢🟢🟢🟢`);
+          logs.push(`[${isoNow()}][UP] 🎉 Sync ${fullname} success [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
+          logs.push(`[${isoNow()}][UP] 🔗 ${registry}/${fullname}`);
           await this.taskService.appendTaskLog(task, logs.join('\n'));
           return;
         }
@@ -226,7 +228,7 @@ export class PackageSyncerService extends AbstractService {
     const { status, data } = remoteFetchResult;
 
     // deleted or blocked
-    if (status === 404 || status === 451) {
+    if (status === 451) {
       return true;
     }
 
@@ -243,6 +245,23 @@ export class PackageSyncerService extends AbstractService {
 
     // security holder
     // test/fixtures/registry.npmjs.org/security-holding-package.json
+    // {
+    //   "_id": "xxx",
+    //   "_rev": "9-a740a77bcd978abeec47d2d027bf688c",
+    //   "name": "xxx",
+    //   "time": {
+    //     "modified": "2017-11-28T00:45:24.162Z",
+    //     "created": "2013-09-20T23:25:18.122Z",
+    //     "0.0.0": "2013-09-20T23:25:20.242Z",
+    //     "1.0.0": "2016-06-22T00:07:41.958Z",
+    //     "0.0.1-security": "2016-12-15T01:03:58.663Z",
+    //     "unpublished": {
+    //       "time": "2017-11-28T00:45:24.163Z",
+    //       "versions": []
+    //     }
+    //   },
+    //   "_attachments": {}
+    // }
     let isSecurityHolder = true;
     for (const versionInfo of Object.entries<{ _npmUser?: { name: string } }>(data.versions || {})) {
       const [ v, info ] = versionInfo;
@@ -297,8 +316,8 @@ export class PackageSyncerService extends AbstractService {
     }
 
     // update log
-    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    logs.push(`[${isoNow()}] 📝 Log URL: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🔗 ${url}`);
     await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
     this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
       task.taskId, task.targetName);
@@ -310,7 +329,7 @@ export class PackageSyncerService extends AbstractService {
   // 1. 其次从 task.data.registryId (创建单包同步任务时传入)
   // 2. 接着根据 scope 进行计算 (作为子包依赖同步时候，无 registryId)
   // 3. 最后返回 default registryId (可能 default registry 也不存在)
-  public async initSpecRegistry(task: Task, pkg: Package | null = null, scope?: string): Promise<Registry | null> {
+  public async initSpecRegistry(task: Task, pkg: Package | null = null, scope?: string): Promise<Registry> {
     const registryId = pkg?.registryId || (task.data as SyncPackageTaskOptions).registryId;
     let targetHost: string = this.config.cnpmcore.sourceRegistry;
     let registry: Registry | null = null;
@@ -349,25 +368,38 @@ export class PackageSyncerService extends AbstractService {
   public async executeTask(task: Task) {
     const fullname = task.targetName;
     const [ scope, name ] = getScopeAndName(fullname);
-    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, remoteAuthToken } = task.data as SyncPackageTaskOptions;
+    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, specificVersions } = task.data as SyncPackageTaskOptions;
     let pkg = await this.packageRepository.findPackage(scope, name);
     const registry = await this.initSpecRegistry(task, pkg, scope);
     const registryHost = this.npmRegistry.registry;
+    const remoteAuthToken = registry.authToken;
     let logs: string[] = [];
     if (tips) {
       logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`);
     }
+
     const taskQueueLength = await this.taskService.getTaskQueueLength(task.type);
     const taskQueueHighWaterSize = this.config.cnpmcore.taskQueueHighWaterSize;
     const taskQueueInHighWaterState = taskQueueLength >= taskQueueHighWaterSize;
     const skipDependencies = taskQueueInHighWaterState ? true : !!originSkipDependencies;
-    const syncUpstream = !!(!taskQueueInHighWaterState && this.config.cnpmcore.sourceRegistryIsCNpm && this.config.cnpmcore.syncUpstreamFirst);
+    const syncUpstream = !!(!taskQueueInHighWaterState && this.config.cnpmcore.sourceRegistryIsCNpm && this.config.cnpmcore.syncUpstreamFirst && registry.name === PresetRegistryName.default);
     const logUrl = `${this.config.cnpmcore.registry}/-/package/${fullname}/syncs/${task.taskId}/log`;
     this.logger.info('[PackageSyncerService.executeTask:start] taskId: %s, targetName: %s, attempts: %s, taskQueue: %s/%s, syncUpstream: %s, log: %s',
       task.taskId, task.targetName, task.attempts, taskQueueLength, taskQueueHighWaterSize, syncUpstream, logUrl);
     logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registryHost}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, forceSyncHistory: ${!!forceSyncHistory} attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
+    if (specificVersions) {
+      logs.push(`[${isoNow()}] 👉 syncing specific versions: ${specificVersions.join(' | ')} 👈`);
+    }
     logs.push(`[${isoNow()}] 🚧 log: ${logUrl}`);
 
+    if (registry?.name === PresetRegistryName.self) {
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} has been published to the self registry, skip sync ❌❌❌❌❌`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, invalid registryId',
+        task.taskId, task.targetName);
+      return;
+    }
+
     if (pkg && pkg?.registryId !== registry?.registryId) {
       if (pkg.registryId) {
         logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} registry is ${pkg.registryId} not belong to ${registry?.registryId}, skip sync ❌❌❌❌❌`);
@@ -440,6 +472,19 @@ export class PackageSyncerService extends AbstractService {
       return;
     }
 
+    if (status === 404) {
+      // ignore 404 status
+      // https://github.com/node-modules/detect-port/issues/57
+      task.error = `Package not found, status 404, data: ${JSON.stringify(data)}`;
+      logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ❌ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
+      this.logger.info('[PackageSyncerService.executeTask:fail-request-error] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      return;
+    }
+
     let readme = data.readme || '';
     if (typeof readme !== 'string') {
       readme = JSON.stringify(readme);
@@ -472,7 +517,7 @@ export class PackageSyncerService extends AbstractService {
     //   { name: 'jasonlaster11', email: 'jason.laster.11@gmail.com' }
     // ],
     let maintainers = data.maintainers;
-    const maintainersMap = {};
+    const maintainersMap: Record<string, PackageManifestType['maintainers']> = {};
     const users: User[] = [];
     let changedUserCount = 0;
     if (!Array.isArray(maintainers) || maintainers.length === 0) {
@@ -530,9 +575,9 @@ export class PackageSyncerService extends AbstractService {
       task.error = `invalid maintainers: ${JSON.stringify(maintainers)}`;
       logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`);
       logs.push(`[${isoNow()}] ${failEnd}`);
-      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
       this.logger.info('[PackageSyncerService.executeTask:fail-invalid-maintainers] taskId: %s, targetName: %s, %s',
         task.taskId, task.targetName, task.error);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
       return;
     }
 
@@ -544,10 +589,40 @@ export class PackageSyncerService extends AbstractService {
     const existsVersionCount = Object.keys(existsVersionMap).length;
     const abbreviatedVersionMap = abbreviatedManifests?.versions ?? {};
     // 2. save versions
-    const versions = Object.values<any>(versionMap);
-    logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
+    if (specificVersions && !this.config.cnpmcore.strictSyncSpecivicVersion && !specificVersions.includes(distTags.latest)) {
+      logs.push(`[${isoNow()}] 📦 Add latest tag version "${fullname}: ${distTags.latest}"`);
+      specificVersions.push(distTags.latest);
+    }
+    const versions = specificVersions ?
+      Object.values<PackageJSONType>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) :
+      Object.values<PackageJSONType>(versionMap);
+    // 全量同步时跳过排序
+    const sortedAvailableVersions = specificVersions ?
+      versions.map(item => item.version).sort(semver.rcompare) : [];
+    // 在strictSyncSpecivicVersion模式下（不同步latest）且所有传入的version均不可用
+    if (specificVersions && sortedAvailableVersions.length === 0) {
+      logs.push(`[${isoNow()}] ❌ `);
+      task.error = 'There is no available specific versions, stop task.';
+      logs.push(`[${isoNow()}]  ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
+      this.logger.info('[PackageSyncerService.executeTask:fail-empty-list] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      return;
+    }
+    if (specificVersions) {
+      // specific versions may not in manifest.
+      const notAvailableVersionList = specificVersions.filter(i => !sortedAvailableVersions.includes(i));
+      logs.push(`[${isoNow()}] 🚧 Syncing specific versions: ${sortedAvailableVersions.join(' | ')}`);
+      if (notAvailableVersionList.length > 0) {
+        logs.push(`🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
+      }
+    } else {
+      logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
+    }
+
     const updateVersions: string[] = [];
-    const differentMetas: any[] = [];
+    const differentMetas: [PackageJSONType, Partial<PackageJSONType>][] = [];
     let syncIndex = 0;
     for (const item of versions) {
       const version: string = item.version;
@@ -581,10 +656,15 @@ export class PackageSyncerService extends AbstractService {
         // check metaDataKeys, if different value, override exists one
         // https://github.com/cnpm/cnpmjs.org/issues/1667
         // need libc field https://github.com/cnpm/cnpmcore/issues/187
+        // fix _npmUser field since https://github.com/cnpm/cnpmcore/issues/553
         const metaDataKeys = [
-          'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', 'deprecated',
+          'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript',
+          'deprecated', '_npmUser', 'funding',
+          // https://github.com/cnpm/cnpmcore/issues/689
+          'acceptDependencies',
         ];
-        let diffMeta: any;
+        const ignoreInAbbreviated = [ '_npmUser' ];
+        const diffMeta: Partial<PackageJSONType> = {};
         for (const key of metaDataKeys) {
           let remoteItemValue = item[key];
           // make sure hasInstallScript exists
@@ -593,34 +673,30 @@ export class PackageSyncerService extends AbstractService {
               remoteItemValue = true;
             }
           }
-          const remoteItemDiffValue = JSON.stringify(remoteItemValue);
-          if (remoteItemDiffValue !== JSON.stringify(existsItem[key])) {
-            if (!diffMeta) diffMeta = {};
+          if (!isEqual(remoteItemValue, existsItem[key])) {
             diffMeta[key] = remoteItemValue;
-          } else if (existsAbbreviatedItem && remoteItemDiffValue !== JSON.stringify(existsAbbreviatedItem[key])) {
+          } else if (!ignoreInAbbreviated.includes(key) && existsAbbreviatedItem && !isEqual(remoteItemValue, (existsAbbreviatedItem as Record<string, unknown>)[key])) {
             // should diff exists abbreviated item too
-            if (!diffMeta) diffMeta = {};
             diffMeta[key] = remoteItemValue;
           }
         }
         // should delete readme
         if (shouldDeleteReadme) {
-          if (!diffMeta) diffMeta = {};
           diffMeta.readme = undefined;
         }
-        if (diffMeta) {
+        if (!isEmpty(diffMeta)) {
           differentMetas.push([ existsItem, diffMeta ]);
         }
         continue;
       }
       syncIndex++;
-      const description: string = item.description;
+      const description = item.description;
       // "dist": {
       //   "shasum": "943e0ec03df00ebeb6273a5b94b916ba54b47581",
       //   "tarball": "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz"
       // },
       const dist = item.dist;
-      const tarball: string = dist && dist.tarball;
+      const tarball = dist && dist.tarball;
       if (!tarball) {
         lastErrorMessage = `missing tarball, dist: ${JSON.stringify(dist)}`;
         logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
@@ -639,7 +715,11 @@ export class PackageSyncerService extends AbstractService {
         localFile = tmpfile;
         logs.push(`[${isoNow()}] 🚧 [${syncIndex}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)} => ${localFile}`);
       } catch (err: any) {
-        this.logger.error('Download tarball %s error: %s', tarball, err);
+        if (err.name === 'DownloadNotFoundError' || err.name === 'DownloadStatusInvalidError') {
+          this.logger.warn('Download tarball %s error: %s', tarball, err);
+        } else {
+          this.logger.error('Download tarball %s error: %s', tarball, err);
+        }
         lastErrorMessage = `download tarball error: ${err}`;
         logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
         await this.taskService.appendTaskLog(task, logs.join('\n'));
@@ -667,9 +747,10 @@ export class PackageSyncerService extends AbstractService {
       };
       try {
         // 当 version 记录已经存在时，还需要校验一下 pkg.manifests 是否存在
-        const pkgVersion = await this.packageManagerService.publish(publishCmd, users[0]);
+        const publisher = users.find(user => user.displayName === item._npmUser?.name) || users[0];
+        const pkgVersion = await this.packageManagerService.publish(publishCmd, publisher);
         updateVersions.push(pkgVersion.version);
-        logs.push(`[${isoNow()}] 🟢 [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
+        logs.push(`[${isoNow()}] 🎉 [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
       } catch (err: any) {
         if (err.name === 'ForbiddenError') {
           logs.push(`[${isoNow()}] 🐛 [${syncIndex}] Synced version ${version} already exists, skip publish, try to set in local manifest`);
@@ -680,6 +761,14 @@ export class PackageSyncerService extends AbstractService {
           this.logger.error(err);
           lastErrorMessage = `publish error: ${err}`;
           logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} error, ${lastErrorMessage}`);
+          if (err.name === 'BadRequestError') {
+            // 由于当前版本的依赖不满足，尝试重试
+            // 默认会在当前队列最后重试
+            this.logger.info('[PackageSyncerService.executeTask:fail-validate-deps] taskId: %s, targetName: %s, %s',
+              task.taskId, task.targetName, task.error);
+            await this.taskService.retryTask(task, logs.join('\n'));
+            return;
+          }
         }
       }
       await this.taskService.appendTaskLog(task, logs.join('\n'));
@@ -705,9 +794,9 @@ export class PackageSyncerService extends AbstractService {
       logs.push(`[${isoNow()}] ❌ All versions sync fail, package not exists, log: ${logUrl}`);
       logs.push(`[${isoNow()}] ${failEnd}`);
       task.error = lastErrorMessage;
-      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
       this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, package not exists',
         task.taskId, task.targetName);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
       return;
     }
 
@@ -787,6 +876,22 @@ export class PackageSyncerService extends AbstractService {
         }
       }
     }
+    // 3.2 shoud add latest tag
+    // 在同步 sepcific version 时如果没有同步 latestTag 的版本会出现 latestTag 丢失或指向版本不正确的情况
+    if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
+      // 不允许自动同步 latest 版本，从已同步版本中选出 latest
+      let latestStableVersion = semver.maxSatisfying(sortedAvailableVersions, '*');
+      // 所有版本都不是稳定版本则指向非稳定版本保证 latest 存在
+      if (!latestStableVersion) {
+        latestStableVersion = sortedAvailableVersions[0];
+      }
+      if (!existsDistTags.latest || semver.rcompare(existsDistTags.latest, latestStableVersion) === 1) {
+        logs.push(`[${isoNow()}] 🚧 patch latest tag from specific versions 🚧`);
+        changedTags.push({ action: 'change', tag: 'latest', version: latestStableVersion });
+        await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);
+      }
+    }
+
     if (changedTags.length > 0) {
       logs.push(`[${isoNow()}] 🟢 Synced ${changedTags.length} tags: ${JSON.stringify(changedTags)}`);
     }
@@ -814,6 +919,15 @@ export class PackageSyncerService extends AbstractService {
       logs.push(`[${isoNow()}] 🟢 Removed ${removedMaintainers.length} maintainers: ${JSON.stringify(removedMaintainers)}`);
     }
 
+    // 4.2 update package maintainers in dist
+    // The event is initialized in the repository and distributed after uncork.
+    // maintainers' information is updated in bulk to ensure consistency.
+    if (!isEqual(maintainers, existsMaintainers)) {
+      logs.push(`[${isoNow()}] 🚧 Syncing maintainers to package manifest, from: ${JSON.stringify(maintainers)} to: ${JSON.stringify(existsMaintainers)}`);
+      await this.packageManagerService.refreshPackageMaintainersToDists(pkg);
+      logs.push(`[${isoNow()}] 🟢 Syncing maintainers to package manifest done`);
+    }
+
     // 5. add deps sync task
     for (const dependencyName of dependenciesSet) {
       const existsTask = await this.taskRepository.findTaskByTargetName(dependencyName, TaskType.SyncPackage, TaskState.Waiting);
@@ -836,12 +950,12 @@ export class PackageSyncerService extends AbstractService {
 
     // clean cache
     await this.cacheService.removeCache(fullname);
-    logs.push(`[${isoNow()}] 🟢 Clean cache`);
-    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    logs.push(`[${isoNow()}] 🗑️ Clean cache`);
+    logs.push(`[${isoNow()}] 📝 Log URL: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🔗 ${url}`);
     task.error = lastErrorMessage;
-    await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
     this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s',
       task.taskId, task.targetName);
+    await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
   }
 }

app/core/service/PackageVersionFileService.ts

@@ -1,36 +1,59 @@
 import fs from 'node:fs/promises';
 import { join, dirname, basename } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import tar from 'tar';
+import tar from '@fengmk2/tar';
 import {
   AccessLevel,
   SingletonProto,
   Inject,
 } from '@eggjs/tegg';
+import { ConflictError, ForbiddenError } from 'egg-errors';
+import semver from 'semver';
 import { AbstractService } from '../../common/AbstractService';
 import {
   calculateIntegrity,
+  getFullname,
 } from '../../common/PackageUtil';
 import { createTempDir, mimeLookup } from '../../common/FileUtil';
 import {
   PackageRepository,
 } from '../../repository/PackageRepository';
 import { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
 import { DistRepository } from '../../repository/DistRepository';
+import { isDuplicateKeyError } from '../../repository/util/ErrorUtil';
 import { PackageVersionFile } from '../entity/PackageVersionFile';
 import { PackageVersion } from '../entity/PackageVersion';
 import { Package } from '../entity/Package';
+import { PackageManagerService } from './PackageManagerService';
+import { CacheAdapter } from '../../common/adapter/CacheAdapter';
+
+const unpkgWhiteListUrl = 'https://github.com/cnpm/unpkg-white-list';
+const CHECK_TIMEOUT = process.env.NODE_ENV === 'test' ? 1 : 60000;
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
 })
 export class PackageVersionFileService extends AbstractService {
+  @Inject()
+  private readonly packageVersionRepository: PackageVersionRepository;
   @Inject()
   private readonly packageRepository: PackageRepository;
   @Inject()
   private readonly packageVersionFileRepository: PackageVersionFileRepository;
   @Inject()
   private readonly distRepository: DistRepository;
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
+  @Inject()
+  private readonly cacheAdapter: CacheAdapter;
+
+  #unpkgWhiteListCheckTime: number = 0;
+  #unpkgWhiteListCurrentVersion: string = '';
+  #unpkgWhiteListAllowPackages: Record<string, {
+    version: string;
+  }> = {};
+  #unpkgWhiteListAllowScopes: string[] = [];
 
   async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) {
     await this.#ensurePackageVersionFilesSync(pkgVersion);
@@ -47,18 +70,131 @@ export class PackageVersionFileService extends AbstractService {
   async #ensurePackageVersionFilesSync(pkgVersion: PackageVersion) {
     const hasFiles = await this.packageVersionFileRepository.hasPackageVersionFiles(pkgVersion.packageVersionId);
     if (!hasFiles) {
-      await this.syncPackageVersionFiles(pkgVersion);
+      const lockName = `${pkgVersion.packageVersionId}:syncFiles`;
+      const lockRes = await this.cacheAdapter.usingLock(lockName, 60, async () => {
+        await this.syncPackageVersionFiles(pkgVersion);
+      });
+      // lock fail
+      if (!lockRes) {
+        this.logger.warn('[package:version:syncPackageVersionFiles] check lock:%s fail', lockName);
+        throw new ConflictError('Package version file sync is currently in progress. Please try again later.');
+      }
+    }
+  }
+
+  async #updateUnpkgWhiteList() {
+    if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
+    if (Date.now() - this.#unpkgWhiteListCheckTime <= CHECK_TIMEOUT) {
+      // check update every 60s
+      return;
+    }
+    this.#unpkgWhiteListCheckTime = Date.now();
+    const whiteListScope = '';
+    const whiteListPackageName = 'unpkg-white-list';
+    const whiteListPackageVersion = await this.packageVersionRepository.findVersionByTag(
+      whiteListScope, whiteListPackageName, 'latest');
+    if (!whiteListPackageVersion) return;
+    // same version, skip update for performance
+    if (this.#unpkgWhiteListCurrentVersion === whiteListPackageVersion) return;
+
+    // update the new version white list
+    const { manifest } = await this.packageManagerService.showPackageVersionManifest(
+      whiteListScope, whiteListPackageName, whiteListPackageVersion, false, true);
+    if (!manifest) return;
+    this.#unpkgWhiteListCurrentVersion = manifest.version;
+    this.#unpkgWhiteListAllowPackages = manifest.allowPackages ?? {} as any;
+    this.#unpkgWhiteListAllowScopes = manifest.allowScopes ?? [] as any;
+    this.logger.info('[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes',
+      whiteListPackageVersion,
+      Object.keys(this.#unpkgWhiteListAllowPackages).length,
+      this.#unpkgWhiteListAllowScopes.length,
+    );
+  }
+
+  async checkPackageVersionInUnpkgWhiteList(pkgScope: string, pkgName: string, pkgVersion: string) {
+    if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return;
+    await this.#updateUnpkgWhiteList();
+
+    // check allow scopes
+    if (this.#unpkgWhiteListAllowScopes.includes(pkgScope)) return;
+
+    // check allow packages
+    const fullname = getFullname(pkgScope, pkgName);
+    const pkgConfig = this.#unpkgWhiteListAllowPackages[fullname];
+    if (!pkgConfig?.version) {
+      throw new ForbiddenError(`"${fullname}" is not allow to unpkg files, see ${unpkgWhiteListUrl}`);
+    }
+
+    // satisfies 默认不会包含 prerelease 版本
+    // https://docs.npmjs.com/about-semantic-versioning#using-semantic-versioning-to-specify-update-types-your-package-can-accept
+    // [x, *] 代表任意版本，这里统一通过 semver 来判断
+    if (!semver.satisfies(pkgVersion, pkgConfig.version, { includePrerelease: true })) {
+      throw new ForbiddenError(`"${fullname}@${pkgVersion}" not satisfies "${pkgConfig.version}" to unpkg files, see ${unpkgWhiteListUrl}`);
+    }
+  }
+
+  // 基于 latest version 同步 package readme
+  async syncPackageReadme(pkg: Package, latestPkgVersion: PackageVersion) {
+    const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${latestPkgVersion.version}_latest_readme_${randomUUID()}`;
+    const tmpdir = await createTempDir(this.config.dataDir, dirname);
+    const tarFile = `${tmpdir}.tgz`;
+    const readmeFilenames: string[] = [];
+    try {
+      this.logger.info('[PackageVersionFileService.syncPackageReadme:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
+        latestPkgVersion.tarDist.distId, latestPkgVersion.tarDist.path, latestPkgVersion.tarDist.size, tarFile);
+      await this.distRepository.downloadDistToFile(latestPkgVersion.tarDist, tarFile);
+      this.logger.info('[PackageVersionFileService.syncPackageReadme:extract-start] tmpdir:%s', tmpdir);
+      await tar.extract({
+        file: tarFile,
+        cwd: tmpdir,
+        strip: 1,
+        onentry: entry => {
+          const filename = this.#formatTarEntryFilename(entry);
+          if (!filename) return;
+          if (this.#matchReadmeFilename(filename)) {
+            readmeFilenames.push(filename);
+          }
+        },
+      });
+      if (readmeFilenames.length > 0) {
+        const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
+        const readmeFile = join(tmpdir, readmeFilename);
+        await this.packageManagerService.savePackageReadme(pkg, readmeFile);
+      }
+    } catch (err) {
+      this.logger.warn('[PackageVersionFileService.syncPackageReadme:error] packageVersionId: %s, readmeFilenames: %j, tmpdir: %s, error: %s',
+        latestPkgVersion.packageVersionId, readmeFilenames, tmpdir, err);
+      // ignore TAR_BAD_ARCHIVE error
+      if (err.code === 'TAR_BAD_ARCHIVE') return;
+      throw err;
+    } finally {
+      try {
+        await fs.rm(tarFile, { force: true });
+        await fs.rm(tmpdir, { recursive: true, force: true });
+      } catch (err) {
+        this.logger.warn('[PackageVersionFileService.syncPackageReadme:warn] remove tmpdir: %s, error: %s',
+          tmpdir, err);
+      }
     }
   }
 
   async syncPackageVersionFiles(pkgVersion: PackageVersion) {
     const files: PackageVersionFile[] = [];
+    // must set enableUnpkg and enableSyncUnpkgFiles = true both
+    if (!this.config.cnpmcore.enableUnpkg) return files;
+    if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files;
+
     const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
     if (!pkg) return files;
+
+    // check unpkg white list
+    await this.checkPackageVersionInUnpkgWhiteList(pkg.scope, pkg.name, pkgVersion.version);
+
     const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`;
     const tmpdir = await createTempDir(this.config.dataDir, dirname);
     const tarFile = `${tmpdir}.tgz`;
     const paths: string[] = [];
+    const readmeFilenames: string[] = [];
     try {
       this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
         pkgVersion.tarDist.distId, pkgVersion.tarDist.path, pkgVersion.tarDist.size, tarFile);
@@ -69,11 +205,12 @@ export class PackageVersionFileService extends AbstractService {
         cwd: tmpdir,
         strip: 1,
         onentry: entry => {
-          if (entry.type !== 'File') return;
-          if (!entry.path.startsWith('package/')) return;
-          // ignore hidden dir
-          if (entry.path.includes('/./')) return;
-          paths.push(entry.path.replace(/^package\//i, '/'));
+          const filename = this.#formatTarEntryFilename(entry);
+          if (!filename) return;
+          paths.push('/' + filename);
+          if (this.#matchReadmeFilename(filename)) {
+            readmeFilenames.push(filename);
+          }
         },
       });
       for (const path of paths) {
@@ -83,6 +220,11 @@ export class PackageVersionFileService extends AbstractService {
       }
       this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:success] packageVersionId: %s, %d paths, %d files, tmpdir: %s',
         pkgVersion.packageVersionId, paths.length, files.length, tmpdir);
+      if (readmeFilenames.length > 0) {
+        const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
+        const readmeFile = join(tmpdir, readmeFilename);
+        await this.packageManagerService.savePackageVersionReadme(pkgVersion, readmeFile);
+      }
       return files;
     } catch (err) {
       this.logger.warn('[PackageVersionFileService.syncPackageVersionFiles:error] packageVersionId: %s, %d paths, tmpdir: %s, error: %s',
@@ -123,7 +265,7 @@ export class PackageVersionFileService extends AbstractService {
       name,
       dist,
       contentType: mimeLookup(path),
-      mtime: stat.mtime,
+      mtime: pkgVersion.publishTime,
     });
     try {
       await this.packageVersionFileRepository.createPackageVersionFile(file);
@@ -131,7 +273,9 @@ export class PackageVersionFileService extends AbstractService {
         file.packageVersionFileId, dist.size, file.path);
     } catch (err) {
       // ignore Duplicate entry
-      if (err.code === 'ER_DUP_ENTRY') return file;
+      if (isDuplicateKeyError(err)) {
+        return file;
+      }
       throw err;
     }
     return file;
@@ -143,4 +287,37 @@ export class PackageVersionFileService extends AbstractService {
       name: basename(path),
     };
   }
+
+  #formatTarEntryFilename(entry: tar.ReadEntry) {
+    if (entry.type !== 'File') return;
+    // ignore hidden dir
+    if (entry.path.includes('/./')) return;
+    // https://github.com/cnpm/cnpmcore/issues/452#issuecomment-1570077310
+    // strip first dir, e.g.: 'package/', 'lodash-es/'
+    const filename = entry.path.split('/').slice(1).join('/');
+    return filename;
+  }
+
+  #matchReadmeFilename(filename: string) {
+    // support README,README.*
+    // https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
+    return (/^README(\.\w{1,20}|$)/i.test(filename));
+  }
+
+  // https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
+  #preferMarkdownReadme(files: string[]) {
+    let fallback = 0;
+    const markdownRE = /\.m?a?r?k?d?o?w?n?$/i;
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      if (markdownRE.test(file)) {
+        return file;
+      } else if (file.toLowerCase() === 'README') {
+        fallback = i;
+      }
+    }
+    // prefer README.md, followed by README; otherwise, return
+    // the first filename (which could be README)
+    return files[fallback];
+  }
 }

app/core/service/PackageVersionService.ts

@@ -0,0 +1,134 @@
+import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
+import semver, { Range } from 'semver';
+import { Result, AliasResult } from 'npm-package-arg';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
+import { getScopeAndName } from '../../common/PackageUtil';
+import { SqlRange } from '../entity/SqlRange';
+import { BugVersionService } from './BugVersionService';
+import type { PackageJSONType, PackageRepository } from '../../repository/PackageRepository';
+import { DistRepository } from '../../repository/DistRepository';
+import { BugVersionAdvice } from '../entity/BugVersion';
+import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class PackageVersionService {
+  @Inject()
+  private packageVersionRepository: PackageVersionRepository;
+
+  @Inject()
+  private packageRepository: PackageRepository;
+
+  @Inject()
+  private packageVersionBlockRepository: PackageVersionBlockRepository;
+
+  @Inject()
+  private readonly bugVersionService: BugVersionService;
+
+  @Inject()
+  private readonly distRepository: DistRepository;
+
+  async readManifest(pkgId: string, spec: Result, isFullManifests: boolean, withBugVersion = true): Promise<PackageJSONType | undefined> {
+    const realSpec = this.findRealSpec(spec);
+    let version = await this.getVersion(realSpec, false);
+    if (!version) {
+      return undefined;
+    }
+    let bugVersionAdvice: {
+      advice: BugVersionAdvice,
+      version: string,
+    } | undefined;
+    if (withBugVersion) {
+      const bugVersion = await this.bugVersionService.getBugVersion();
+      if (bugVersion) {
+        const advice = bugVersion.fixVersion(spec.name!, version);
+        if (advice) {
+          bugVersionAdvice = {
+            advice,
+            version,
+          };
+          version = advice.version;
+        }
+      }
+    }
+    let manifest;
+    if (isFullManifests) {
+      manifest = await this.distRepository.findPackageVersionManifest(pkgId, version);
+    } else {
+      manifest = await this.distRepository.findPackageAbbreviatedManifest(pkgId, version);
+    }
+    if (manifest && bugVersionAdvice) {
+      manifest.deprecated = `[WARNING] Use ${bugVersionAdvice.advice.version} instead of ${bugVersionAdvice.version}, reason: ${bugVersionAdvice.advice.reason}`;
+      manifest.version = bugVersionAdvice.version;
+    }
+    return manifest;
+  }
+
+  private findRealSpec(spec: Result) {
+    let realSpec: Result;
+    switch (spec.type) {
+      case 'alias':
+        realSpec = (spec as AliasResult).subSpec;
+        break;
+      case 'version':
+      case 'tag':
+      case 'range':
+        realSpec = spec;
+        break;
+      default:
+        throw new Error(`npmcore not support spec: ${spec.raw}`);
+    }
+    return realSpec;
+  }
+
+  async getVersion(spec: Result, withBugVersion = true): Promise<string | undefined | null> {
+    let version: string | undefined | null;
+    const [ scope, name ] = getScopeAndName(spec.name!);
+    // 优先通过 tag 来进行判断
+    if (spec.type === 'tag') {
+      version = await this.packageVersionRepository.findVersionByTag(scope, name, spec.fetchSpec!);
+    } else if (spec.type === 'version') {
+      // 1.0.0
+      // '=1.0.0' => '1.0.0'
+      // https://github.com/npm/npm-package-arg/blob/main/lib/npa.js#L392
+      version = semver.valid(spec.fetchSpec!, true);
+    } else if (spec.type === 'range') {
+      // a@1.1 情况下，1.1 会解析为 range，如果有对应的 distTag 时会失效
+      // 这里需要进行兼容
+      // 仅当 spec 不为 version 时才查询，减少请求次数
+      const versionMatchTag = await this.packageVersionRepository.findVersionByTag(scope, name, spec.fetchSpec!);
+      if (versionMatchTag) {
+        version = versionMatchTag;
+      } else {
+        const range = new Range(spec.fetchSpec!);
+        const paddingSemVer = new SqlRange(range);
+        if (paddingSemVer.containPreRelease) {
+          const versions = await this.packageVersionRepository.findSatisfyVersionsWithPrerelease(scope, name, paddingSemVer);
+          version = semver.maxSatisfying(versions, range);
+        } else {
+          version = await this.packageVersionRepository.findMaxSatisfyVersion(scope, name, paddingSemVer);
+        }
+      }
+    }
+    if (version && withBugVersion) {
+      const bugVersion = await this.bugVersionService.getBugVersion();
+      if (bugVersion) {
+        const advice = bugVersion.fixVersion(spec.name!, version);
+        if (advice) {
+          version = advice.version;
+        }
+      }
+    }
+    return version;
+  }
+
+  async findBlockInfo(fullname: string) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const packageId = await this.packageRepository.findPackageId(scope, name);
+    if (!packageId) {
+      return null;
+    }
+    return await this.packageVersionBlockRepository.findPackageBlock(packageId);
+  }
+}

app/core/service/ProxyCacheService.ts

@@ -0,0 +1,274 @@
+import { EggHttpClient, HttpClientRequestOptions, HttpClientResponse } from 'egg';
+import { ForbiddenError } from 'egg-errors';
+import { SingletonProto, AccessLevel, Inject, EggContext } from '@eggjs/tegg';
+import { BackgroundTaskHelper } from '@eggjs/tegg-background-task';
+import { valid as semverValid } from 'semver';
+import { AbstractService } from '../../common/AbstractService';
+import { TaskService } from './TaskService';
+import { CacheService } from './CacheService';
+import { RegistryManagerService } from './RegistryManagerService';
+import { NPMRegistry } from '../../common/adapter/NPMRegistry';
+import { NFSAdapter } from '../../common/adapter/NFSAdapter';
+import { ProxyCache } from '../entity/ProxyCache';
+import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task';
+import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
+import { TaskType, TaskState } from '../../common/enum/Task';
+import { calculateIntegrity } from '../../common/PackageUtil';
+import { ABBREVIATED_META_TYPE, PROXY_CACHE_DIR_NAME } from '../../common/constants';
+import { DIST_NAMES } from '../entity/Package';
+import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository';
+
+function isoNow() {
+  return new Date().toISOString();
+}
+
+export function isPkgManifest(fileType: DIST_NAMES) {
+  return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS;
+}
+
+type GetSourceManifestAndCacheReturnType<T> = T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType :
+  T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never;
+
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class ProxyCacheService extends AbstractService {
+  @Inject()
+  private readonly httpclient: EggHttpClient;
+  @Inject()
+  private readonly npmRegistry: NPMRegistry;
+  @Inject()
+  private readonly nfsAdapter: NFSAdapter;
+  @Inject()
+  private readonly proxyCacheRepository: ProxyCacheRepository;
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly taskService: TaskService;
+  @Inject()
+  private readonly cacheService: CacheService;
+  @Inject()
+  private readonly backgroundTaskHelper:BackgroundTaskHelper;
+
+  async getPackageVersionTarResponse(fullname: string, ctx: EggContext): Promise<HttpClientResponse> {
+    if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) {
+      throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`);
+    }
+    return await this.getProxyResponse(ctx);
+  }
+
+  async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise<AbbreviatedPackageManifestType|PackageManifestType> {
+    const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS;
+    const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath;
+    if (cachedStoreKey) {
+      try {
+        const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
+        if (!nfsBytes) throw new Error('not found proxy cache, try again later.');
+
+        const nfsBuffer = Buffer.from(nfsBytes);
+        const { shasum: etag } = await calculateIntegrity(nfsBytes);
+        await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, nfsBuffer);
+
+        const nfsString = nfsBuffer.toString();
+        const nfsPkgManifest = JSON.parse(nfsString);
+        return nfsPkgManifest as AbbreviatedPackageManifestType|PackageManifestType;
+      } catch (error) {
+        /* c8 ignore next 6 */
+        if (error.message.includes('not found proxy cache') || error.message.includes('Unexpected token : in JSON at')) {
+          await this.nfsAdapter.remove(cachedStoreKey);
+          await this.proxyCacheRepository.removeProxyCache(fullname, fileType);
+        }
+        throw error;
+      }
+    }
+
+    const manifest = await this.getRewrittenManifest<typeof fileType>(fullname, fileType);
+    this.backgroundTaskHelper.run(async () => {
+      await this.storeRewrittenManifest(manifest, fullname, fileType);
+      const cachedFiles = ProxyCache.create({ fullname, fileType });
+      await this.proxyCacheRepository.saveProxyCache(cachedFiles);
+    });
+    return manifest;
+  }
+
+  // used by GET /:fullname/:versionOrTag
+  async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST, versionOrTag: string): Promise<AbbreviatedPackageJSONType|PackageJSONType> {
+    let version;
+    if (semverValid(versionOrTag)) {
+      version = versionOrTag;
+    } else {
+      const pkgManifest = await this.getPackageManifest(fullname, DIST_NAMES.ABBREVIATED_MANIFESTS);
+      const distTags = pkgManifest['dist-tags'] || {};
+      version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag;
+    }
+    const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath;
+    if (cachedStoreKey) {
+      try {
+        const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
+        if (!nfsBytes) throw new Error('not found proxy cache, try again later.');
+        const nfsString = Buffer.from(nfsBytes!).toString();
+        return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType;
+      } catch (error) {
+        /* c8 ignore next 6 */
+        if (error.message.includes('not found proxy cache') || error.message.includes('Unexpected token : in JSON at')) {
+          await this.nfsAdapter.remove(cachedStoreKey);
+          await this.proxyCacheRepository.removeProxyCache(fullname, fileType);
+        }
+        throw error;
+      }
+    }
+    const manifest = await this.getRewrittenManifest(fullname, fileType, versionOrTag);
+    this.backgroundTaskHelper.run(async () => {
+      await this.storeRewrittenManifest(manifest, fullname, fileType);
+      const cachedFiles = ProxyCache.create({ fullname, fileType, version });
+      await this.proxyCacheRepository.saveProxyCache(cachedFiles);
+    });
+    return manifest;
+  }
+
+  async removeProxyCache(fullname: string, fileType: DIST_NAMES, version?: string) {
+    const storeKey = isPkgManifest(fileType)
+      ? `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`
+      : `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`;
+    await this.nfsAdapter.remove(storeKey);
+    await this.proxyCacheRepository.removeProxyCache(fullname, fileType, version);
+  }
+
+  replaceTarballUrl<T extends DIST_NAMES>(manifest: GetSourceManifestAndCacheReturnType<T>, fileType: T) {
+    const { sourceRegistry, registry } = this.config.cnpmcore;
+    if (isPkgManifest(fileType)) {
+      // pkg manifest
+      const versionMap = (manifest as AbbreviatedPackageManifestType|PackageManifestType)?.versions;
+      for (const key in versionMap) {
+        const versionItem = versionMap[key];
+        if (versionItem?.dist?.tarball) {
+          versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry);
+        }
+      }
+    } else {
+      // pkg version manifest
+      const distItem = (manifest as AbbreviatedPackageJSONType | PackageJSONType).dist;
+      if (distItem?.tarball) {
+        distItem.tarball = distItem.tarball.replace(sourceRegistry, registry);
+      }
+    }
+    return manifest;
+  }
+
+  async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise<CreateUpdateProxyCacheTask> {
+    return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask;
+  }
+
+  async findExecuteTask() {
+    return await this.taskService.findExecuteTask(TaskType.UpdateProxyCache);
+  }
+
+  async executeTask(task: Task) {
+    const logs: string[] = [];
+    const fullname = (task as CreateUpdateProxyCacheTask).data.fullname;
+    const { fileType, version } = (task as CreateUpdateProxyCacheTask).data;
+    let cachedManifest;
+    logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`);
+    try {
+      const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType);
+      if (!cachedFiles) throw new Error('task params error, can not found record in repo.');
+      cachedManifest = await this.getRewrittenManifest<typeof fileType>(fullname, fileType);
+      await this.storeRewrittenManifest(cachedManifest, fullname, fileType);
+      ProxyCache.update(cachedFiles);
+      await this.proxyCacheRepository.saveProxyCache(cachedFiles);
+    } catch (error) {
+      task.error = error;
+      logs.push(`[${isoNow()}] ❌ ${task.error}`);
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      return;
+    }
+    logs.push(`[${isoNow()}] 🟢 Update Success.`);
+    const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS;
+    const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests);
+    if (cachedKey) {
+      const cacheBytes = Buffer.from(JSON.stringify(cachedManifest));
+      const { shasum: etag } = await calculateIntegrity(cacheBytes);
+      await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes);
+      logs.push(`[${isoNow()}] 🟢 Update Cache Success.`);
+    }
+    await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
+  }
+
+  // only used by schedule task
+  private async getRewrittenManifest<T extends DIST_NAMES>(fullname:string, fileType: T, versionOrTag?:string): Promise<GetSourceManifestAndCacheReturnType<T>> {
+    let responseResult;
+    const USER_AGENT = 'npm_service.cnpmjs.org/cnpmcore';
+    switch (fileType) {
+      case DIST_NAMES.FULL_MANIFESTS: {
+        const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`;
+        responseResult = await this.getProxyResponse({ url, headers: { accept: 'application/json', 'user-agent': USER_AGENT } }, { dataType: 'json' });
+        break;
+      }
+      case DIST_NAMES.ABBREVIATED_MANIFESTS: {
+        const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`;
+        responseResult = await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE, 'user-agent': USER_AGENT } }, { dataType: 'json' });
+        break;
+      }
+      case DIST_NAMES.MANIFEST: {
+        const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag!)}`;
+        responseResult = await this.getProxyResponse({ url, headers: { accept: 'application/json', 'user-agent': USER_AGENT } }, { dataType: 'json' });
+        break;
+      }
+      case DIST_NAMES.ABBREVIATED: {
+        const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag!)}`;
+        responseResult = await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE, 'user-agent': USER_AGENT } }, { dataType: 'json' });
+        break;
+      }
+      default:
+        break;
+    }
+
+    // replace tarball url
+    const manifest = this.replaceTarballUrl(responseResult.data, fileType);
+    return manifest;
+  }
+
+  private async storeRewrittenManifest(manifest, fullname: string, fileType: DIST_NAMES) {
+    let storeKey: string;
+    if (isPkgManifest(fileType)) {
+      storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`;
+    } else {
+      const version = manifest.version;
+      storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`;
+    }
+    const nfsBytes = Buffer.from(JSON.stringify(manifest));
+    await this.nfsAdapter.uploadBytes(storeKey, nfsBytes);
+  }
+
+  async getProxyResponse(ctx: Partial<EggContext>, options?: HttpClientRequestOptions): Promise<HttpClientResponse> {
+    const registry = this.npmRegistry.registry;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
+    const authorization = this.npmRegistry.genAuthorizationHeader(remoteAuthToken);
+
+    const url = `${this.npmRegistry.registry}${ctx.url}`;
+
+    const res = await this.httpclient.request(url, {
+      timing: true,
+      followRedirect: true,
+      // once redirection is also count as a retry
+      retry: 7,
+      dataType: 'stream',
+      timeout: 10000,
+      compressed: true,
+      ...options,
+      headers: {
+        accept: ctx.headers?.accept,
+        'user-agent': ctx.headers?.['user-agent'],
+        authorization,
+        'x-forwarded-for': ctx?.ip,
+        via: `1.1, ${this.config.cnpmcore.registry}`,
+      },
+    }) as HttpClientResponse;
+    this.logger.info('[ProxyCacheService:getProxyStreamResponse] %s, status: %s', url, res.status);
+    return res;
+  }
+}

app/core/service/RegistryManagerService.ts

@@ -11,13 +11,13 @@ import { PageOptions, PageResult } from '../util/EntityUtil';
 import { ScopeManagerService } from './ScopeManagerService';
 import { TaskService } from './TaskService';
 import { Task } from '../entity/Task';
-import { PresetRegistryName } from '../../common/constants';
+import { ChangesStreamMode, PresetRegistryName } from '../../common/constants';
 import { RegistryType } from '../../common/enum/Registry';
 
-export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name'> {
+export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
-export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'registryId'> {
+export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
 export interface RemoveRegistryCmd extends Pick<Registry, 'registryId'> {
@@ -61,7 +61,7 @@ export class RegistryManagerService extends AbstractService {
   }
 
   async createRegistry(createCmd: CreateRegistryCmd): Promise<Registry> {
-    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-' } = createCmd;
+    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-', authToken } = createCmd;
     this.logger.info('[RegistryManagerService.createRegistry:prepare] operatorId: %s, createCmd: %j', operatorId, createCmd);
     const registry = Registry.create({
       name,
@@ -69,6 +69,7 @@ export class RegistryManagerService extends AbstractService {
       host,
       userPrefix,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
     return registry;
@@ -76,8 +77,8 @@ export class RegistryManagerService extends AbstractService {
 
   // 更新部分 registry 信息
   // 不允许 userPrefix 字段变更
-  async updateRegistry(updateCmd: UpdateRegistryCmd) {
-    const { name, changeStream, host, type, registryId, operatorId = '-' } = updateCmd;
+  async updateRegistry(registryId: string, updateCmd: UpdateRegistryCmd) {
+    const { name, changeStream, host, type, operatorId = '-', authToken } = updateCmd;
     this.logger.info('[RegistryManagerService.updateRegistry:prepare] operatorId: %s, updateCmd: %j', operatorId, updateCmd);
     const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
     if (!registry) {
@@ -88,6 +89,7 @@ export class RegistryManagerService extends AbstractService {
       changeStream,
       host,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
   }
@@ -105,6 +107,10 @@ export class RegistryManagerService extends AbstractService {
     return await this.registryRepository.findRegistry(registryName);
   }
 
+  async findByRegistryHost(host?: string): Promise<Registry | null> {
+    return host ? await this.registryRepository.findRegistryByRegistryHost(host) : null;
+  }
+
   // 删除 Registry 方法
   // 可选传入 operatorId 作为参数，用于记录操作人员
   // 同时删除对应的 scope 数据
@@ -143,7 +149,7 @@ export class RegistryManagerService extends AbstractService {
 
     // 从配置文件默认生成
     const { changesStreamRegistryMode, changesStreamRegistry: changesStreamHost, sourceRegistry: host } = this.config.cnpmcore;
-    const type = changesStreamRegistryMode === 'json' ? RegistryType.Cnpmcore : RegistryType.Npm;
+    const type = changesStreamRegistryMode === ChangesStreamMode.json ? RegistryType.Cnpmcore : RegistryType.Npm;
     const registry = await this.createRegistry({
       name: PresetRegistryName.default,
       type,
@@ -156,4 +162,12 @@ export class RegistryManagerService extends AbstractService {
 
   }
 
+  async getAuthTokenByRegistryHost(host: string): Promise<string|undefined> {
+    const registry = await this.findByRegistryHost(host);
+    if (!registry) {
+      return undefined;
+    }
+    return registry.authToken;
+  }
+
 }

app/core/service/TaskService.ts

@@ -7,7 +7,7 @@ import { NFSAdapter } from '../../common/adapter/NFSAdapter';
 import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { Task } from '../entity/Task';
+import { Task, CreateSyncPackageTaskData } from '../entity/Task';
 import { QueueAdapter } from '../../common/typing';
 
 @SingletonProto({
@@ -27,10 +27,27 @@ export class TaskService extends AbstractService {
 
   public async createTask(task: Task, addTaskQueueOnExists: boolean) {
     const existsTask = await this.taskRepository.findTaskByTargetName(task.targetName, task.type);
-    if (existsTask) {
-      // 如果任务还未被触发，就不继续重复创建
+
+    // 只在包同步场景下做任务合并，其余场景通过 bizId 来进行任务幂等
+    if (existsTask && Task.needMergeWhenWaiting(task.type)) {
+      // 在包同步场景，如果任务还未被触发，就不继续重复创建
       // 如果任务正在执行，可能任务状态已更新，这种情况需要继续创建
       if (existsTask.state === TaskState.Waiting) {
+        if (task.type === TaskType.SyncPackage) {
+          // 如果是specificVersions的任务则可能可以和存量任务进行合并
+          const specificVersions = (task as Task<CreateSyncPackageTaskData>).data?.specificVersions;
+          const existsTaskSpecificVersions = (existsTask as Task<CreateSyncPackageTaskData>).data?.specificVersions;
+          if (existsTaskSpecificVersions) {
+            if (specificVersions) {
+              // 存量的任务和新增任务都是同步指定版本的任务，合并两者版本至存量任务
+              await this.taskRepository.updateSpecificVersionsOfWaitingTask(existsTask, specificVersions);
+            } else {
+              // 新增任务是全量同步任务，移除存量任务中的指定版本使其成为全量同步任务
+              await this.taskRepository.updateSpecificVersionsOfWaitingTask(existsTask);
+            }
+          }
+          // 存量任务是全量同步任务，直接提高任务优先级
+        }
         // 提高任务的优先级
         if (addTaskQueueOnExists) {
           const queueLength = await this.getTaskQueueLength(task.type);

app/core/service/TokenService.ts

@@ -13,6 +13,8 @@ import { ModelConvertor } from '../../../app/repository/util/ModelConvertor';
 import { Package as PackageEntity } from '../entity/Package';
 import { ForbiddenError, UnauthorizedError } from 'egg-errors';
 import { getScopeAndName } from '../../../app/common/PackageUtil';
+import { sha512 } from '../../../app/common/UserUtil';
+import { UserRepository } from '../../../app/repository/UserRepository';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -22,6 +24,8 @@ export class TokenService extends AbstractService {
   private readonly TokenPackage: typeof TokenPackageModel;
   @Inject()
   private readonly Package: typeof PackageModel;
+  @Inject()
+  private readonly userRepository: UserRepository;
 
   public async listTokenPackages(token: Token) {
     if (isGranularToken(token)) {
@@ -32,17 +36,17 @@ export class TokenService extends AbstractService {
     return null;
   }
 
-  public async checkGranularTokenAccess(token: Token, fullname: string) {
-    // skip classic token
-    if (!isGranularToken(token)) {
-      return true;
-    }
-
+  public async checkTokenStatus(token: Token) {
     // check for expires
-    if (dayjs(token.expiredAt).isBefore(new Date())) {
+    if (isGranularToken(token) && dayjs(token.expiredAt).isBefore(new Date())) {
       throw new UnauthorizedError('Token expired');
     }
 
+    token.lastUsedAt = new Date();
+    this.userRepository.saveToken(token);
+  }
+
+  public async checkGranularTokenAccess(token: Token, fullname: string) {
     // check for scope whitelist
     const [ scope, name ] = getScopeAndName(fullname);
     // check for packages whitelist
@@ -67,4 +71,14 @@ export class TokenService extends AbstractService {
 
   }
 
+  async getUserAndToken(authorization: string) {
+    if (!authorization) return null;
+    const matchs = /^Bearer ([\w\.]+?)$/.exec(authorization);
+    if (!matchs) return null;
+    const tokenValue = matchs[1];
+    const tokenKey = sha512(tokenValue);
+    const authorizedUserAndToken = await this.userRepository.findUserAndTokenByTokenKey(tokenKey);
+    return authorizedUserAndToken;
+  }
+
 }

app/core/service/UserService.ts

@@ -12,6 +12,9 @@ import { WebauthnCredential as WebauthnCredentialEntity } from '../entity/Webaut
 import { LoginResultCode } from '../../common/enum/User';
 import { integrity, checkIntegrity, randomToken, sha512 } from '../../common/UserUtil';
 import { AbstractService } from '../../common/AbstractService';
+import { RegistryManagerService } from './RegistryManagerService';
+import { getPrefixedName } from '../../common/PackageUtil';
+import { Registry } from '../entity/Registry';
 
 type Optional<T, K extends keyof T> = Omit < T, K > & Partial<T> ;
 
@@ -59,12 +62,36 @@ type CreateWebauthnCredentialOptions = {
 export class UserService extends AbstractService {
   @Inject()
   private readonly userRepository: UserRepository;
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
 
   checkPassword(user: UserEntity, password: string): boolean {
     const plain = `${user.passwordSalt}${password}`;
     return checkIntegrity(plain, user.passwordIntegrity);
   }
 
+  async findUserByNameOrDisplayName(name: string) {
+    const hasPrefix = name.includes(':');
+    if (hasPrefix) {
+      return await this.findUserByName(name);
+    }
+
+    const selfRegistry = await this.registryManagerService.ensureSelfRegistry();
+    const selfUser = await this.findUserByName(getPrefixedName(selfRegistry.userPrefix, name));
+    if (selfUser) {
+      return selfUser;
+    }
+
+    const defaultRegistry = await this.registryManagerService.ensureDefaultRegistry();
+    const defaultUser = await this.findUserByName(getPrefixedName(defaultRegistry.userPrefix, name));
+
+    return defaultUser;
+  }
+
+  async findInRegistry(registry:Registry, name: string): Promise<UserEntity | null> {
+    return await this.findUserByName(getPrefixedName(registry.userPrefix, name));
+  }
+
   async findUserByName(name: string): Promise<UserEntity | null> {
     return await this.userRepository.findUserByName(name);
   }
@@ -79,19 +106,23 @@ export class UserService extends AbstractService {
     return { code: LoginResultCode.Success, user, token };
   }
 
-  async ensureTokenByUser({ name, email, password = crypto.randomUUID(), ip }: Optional<CreateUser, 'password'>) {
+  async findOrCreateUser({ name, email, ip, password = crypto.randomUUID() }: Optional<CreateUser, 'password'>) {
     let user = await this.userRepository.findUserByName(name);
     if (!user) {
       const createRes = await this.create({
         name,
         email,
-        // Authentication via sso
-        // should use token instead of password
         password,
         ip,
       });
       user = createRes.user;
     }
+
+    return user;
+  }
+
+  async ensureTokenByUser(opts: Optional<CreateUser, 'password'>) {
+    const user = await this.findOrCreateUser(opts);
     const token = await this.createToken(user.userId);
     return { user, token };
   }
@@ -172,14 +203,14 @@ export class UserService extends AbstractService {
     await this.userRepository.removeToken(token.tokenId);
   }
 
-  async findWebauthnCredential(userId: string, browserType?: string) {
+  async findWebauthnCredential(userId: string, browserType: string | undefined | null) {
     const credential = await this.userRepository.findCredentialByUserIdAndBrowserType(userId, browserType || null);
     return credential;
   }
 
-  async createWebauthnCredential(userId: string, options: CreateWebauthnCredentialOptions) {
+  async createWebauthnCredential(userId: string | undefined, options: CreateWebauthnCredentialOptions) {
     const credentialEntity = WebauthnCredentialEntity.create({
-      userId,
+      userId: userId as string,
       credentialId: options.credentialId,
       publicKey: options.publicKey,
       browserType: options.browserType,
@@ -188,10 +219,11 @@ export class UserService extends AbstractService {
     return credentialEntity;
   }
 
-  async removeWebauthnCredential(userId: string, browserType?: string) {
+  async removeWebauthnCredential(userId?: string, browserType?: string) {
     const credential = await this.userRepository.findCredentialByUserIdAndBrowserType(userId, browserType || null);
     if (credential) {
       await this.userRepository.removeCredential(credential.wancId);
     }
   }
+
 }

app/infra/NFSClientAdapter.ts

@@ -1,10 +1,8 @@
 import {
   AccessLevel,
-  EggObjectLifecycle,
+  LifecycleInit,
   Inject,
   SingletonProto,
-  EggQualifier,
-  EggType,
 } from '@eggjs/tegg';
 import { EggAppConfig, EggLogger } from 'egg';
 import FSClient from 'fs-cnpm';
@@ -15,9 +13,8 @@ import { Readable } from 'stream';
   name: 'nfsClient',
   accessLevel: AccessLevel.PUBLIC,
 })
-export class NFSClientAdapter implements EggObjectLifecycle, NFSClient {
+export class NFSClientAdapter implements NFSClient {
   @Inject()
-  @EggQualifier(EggType.APP)
   private logger: EggLogger;
 
   @Inject()
@@ -31,7 +28,8 @@ export class NFSClientAdapter implements EggObjectLifecycle, NFSClient {
 
   url?(key: string): string;
 
-  async init() {
+  @LifecycleInit()
+  protected async init() {
     // NFS interface https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide
     if (this.config.nfs.client) {
       this._client = this.config.nfs.client;

app/infra/SearchAdapter.ts

@@ -0,0 +1,52 @@
+import {
+  AccessLevel,
+  Inject,
+  SingletonProto,
+} from '@eggjs/tegg';
+import { EggAppConfig } from 'egg';
+
+import { Client as ElasticsearchClient, estypes } from '@elastic/elasticsearch';
+import { SearchAdapter } from '../common/typing';
+
+/**
+ * Use elasticsearch to search the huge npm packages.
+ */
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+  name: 'searchAdapter',
+})
+export class ESSearchAdapter implements SearchAdapter {
+  @Inject()
+  private config: EggAppConfig;
+
+  @Inject()
+  private readonly elasticsearch: ElasticsearchClient; // 由 elasticsearch 插件引入
+
+  async search<T>(query: any): Promise<estypes.SearchHitsMetadata<T>> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const result = await this.elasticsearch.search<T>({
+      index,
+      ...query,
+    });
+    return result.hits;
+  }
+
+  async upsert<T>(id: string, document: T): Promise<string> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const res = await this.elasticsearch.index({
+      id,
+      index,
+      document,
+    });
+    return res._id;
+  }
+
+  async delete(id: string): Promise<string> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const res = await this.elasticsearch.delete({
+      index,
+      id,
+    });
+    return res._id;
+  }
+}

app/port/UserRoleManager.ts

@@ -1,17 +1,15 @@
 import {
   AccessLevel,
-  ContextProto,
   Inject,
   EggContext,
+  ContextProto,
 } from '@eggjs/tegg';
 import { EggAppConfig, EggLogger } from 'egg';
 import { UnauthorizedError, ForbiddenError } from 'egg-errors';
-import { UserRepository } from '../repository/UserRepository';
 import { PackageRepository } from '../repository/PackageRepository';
 import { Package as PackageEntity } from '../core/entity/Package';
 import { User as UserEntity } from '../core/entity/User';
 import { Token as TokenEntity } from '../core/entity/Token';
-import { sha512 } from '../common/UserUtil';
 import { getScopeAndName } from '../common/PackageUtil';
 import { RegistryManagerService } from '../core/service/RegistryManagerService';
 import { TokenService } from '../core/service/TokenService';
@@ -24,8 +22,6 @@ export type TokenRole = 'read' | 'publish' | 'setting';
   accessLevel: AccessLevel.PRIVATE,
 })
 export class UserRoleManager {
-  @Inject()
-  private readonly userRepository: UserRepository;
   @Inject()
   private readonly packageRepository: PackageRepository;
   @Inject()
@@ -108,20 +104,19 @@ export class UserRoleManager {
         user: this.currentAuthorizedUser,
       };
     }
-
     this.handleAuthorized = true;
     const authorization = ctx.get('authorization');
     if (!authorization) return null;
-    const matchs = /^Bearer ([\w\.]+?)$/.exec(authorization);
-    if (!matchs) return null;
-    const tokenValue = matchs[1];
-    const tokenKey = sha512(tokenValue);
-    const authorizedUserAndToken = await this.userRepository.findUserAndTokenByTokenKey(tokenKey);
-    if (authorizedUserAndToken) {
-      this.currentAuthorizedToken = authorizedUserAndToken.token;
-      this.currentAuthorizedUser = authorizedUserAndToken.user;
-      ctx.userId = authorizedUserAndToken.user.userId;
+    const authorizedUserAndToken = await this.tokenService.getUserAndToken(authorization);
+    if (!authorizedUserAndToken) {
+      return null;
     }
+
+    // check token expired & set lastUsedAt
+    await this.tokenService.checkTokenStatus(authorizedUserAndToken.token);
+    this.currentAuthorizedToken = authorizedUserAndToken.token;
+    this.currentAuthorizedUser = authorizedUserAndToken.user;
+    ctx.userId = authorizedUserAndToken.user.userId;
     return authorizedUserAndToken;
   }
 

app/port/config.ts

@@ -0,0 +1,186 @@
+import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants';
+import { DATABASE_TYPE } from '../../config/database';
+
+export { cnpmcoreConfig } from '../../config/config.default';
+
+export type CnpmcoreConfig = {
+  name: string,
+  /**
+   * enable hook or not
+   */
+  hookEnable: boolean,
+  /**
+   * mac custom hooks count
+   */
+  hooksLimit: number,
+  /**
+   * upstream registry url
+   */
+  sourceRegistry: string,
+  /**
+   * upstream registry is base on `cnpmcore` or not
+   * if your upstream is official npm registry, please turn it off
+   */
+  sourceRegistryIsCNpm: boolean,
+  /**
+   * sync upstream first
+   */
+  syncUpstreamFirst: boolean,
+  /**
+   * sync upstream timeout, default is 3mins
+   */
+  sourceRegistrySyncTimeout: number,
+  /**
+   * sync task high water size, default is 100
+   */
+  taskQueueHighWaterSize: number,
+  /**
+   * sync mode
+   * - none: don't sync npm package
+   * - admin: don't sync npm package,only admin can create sync task by sync controller.
+   * - all: sync all npm packages
+   * - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled
+   */
+  syncMode: SyncMode,
+  syncDeleteMode: SyncDeleteMode,
+  syncPackageWorkerMaxConcurrentTasks: number,
+  triggerHookWorkerMaxConcurrentTasks: number,
+  createTriggerHookWorkerMaxConcurrentTasks: number,
+  /**
+   * stop syncing these packages in future
+   */
+  syncPackageBlockList: string[],
+  /**
+   * check recently from https://www.npmjs.com/browse/updated, if use set changesStreamRegistry to cnpmcore,
+   * maybe you should disable it
+   */
+  enableCheckRecentlyUpdated: boolean,
+  /**
+   * mirror binary, default is false
+   */
+  enableSyncBinary: boolean,
+  /**
+   * sync binary source api, default is `${sourceRegistry}/-/binary`
+   */
+  syncBinaryFromAPISource: string,
+  /**
+   * enable sync downloads data from source registry https://github.com/cnpm/cnpmcore/issues/108
+   * all three parameters must be configured at the same time to take effect
+   */
+  enableSyncDownloadData: boolean,
+  syncDownloadDataSourceRegistry: string,
+  /**
+   * should be YYYY-MM-DD format
+   */
+  syncDownloadDataMaxDate: string,
+  /**
+   * @see https://github.com/npm/registry-follower-tutorial
+   */
+  enableChangesStream: boolean,
+  checkChangesStreamInterval: number,
+  changesStreamRegistry: string,
+  /**
+   * handle _changes request mode, default is 'streaming', please set it to 'json' when on cnpmcore registry
+   */
+  changesStreamRegistryMode: ChangesStreamMode,
+  /**
+   * registry url
+   */
+  registry: string,
+  /**
+   * https://docs.npmjs.com/cli/v6/using-npm/config#always-auth npm <= 6
+   * if `alwaysAuth=true`, all api request required access token
+   */
+  alwaysAuth: boolean,
+  /**
+   * white scope list
+   */
+  allowScopes: string[],
+  /**
+   * allow publish non-scope package, disable by default
+   */
+  allowPublishNonScopePackage: boolean,
+  /**
+   * Public registration is allowed, otherwise only admins can login
+   */
+  allowPublicRegistration: boolean,
+  /**
+   * default system admins
+   */
+  admins: Record<string, string>,
+  /**
+   * use webauthn for login, https://webauthn.guide/
+   * only support platform authenticators, browser support: https://webauthn.me/browser-support
+   */
+  enableWebAuthn: boolean,
+  /**
+   * http response cache control header
+   */
+  enableCDN: boolean,
+  /**
+   * if you are using CDN, can override it
+   * it meaning cache 300s on CDN server and client side.
+   */
+  cdnCacheControlHeader: string,
+  /**
+   * if you are using CDN, can set it to 'Accept, Accept-Encoding'
+   */
+  cdnVaryHeader: string,
+  /**
+   * store full package version manifests data to database table(package_version_manifests), default is false
+   */
+  enableStoreFullPackageVersionManifestsToDatabase: boolean,
+  /**
+   * only support npm as client and npm >= 7.0.0 allow publish action
+   */
+  enableNpmClientAndVersionCheck: boolean,
+  /**
+   * sync when package not found, only effect when syncMode = all/exist
+   */
+  syncNotFound: boolean,
+  /**
+   * redirect to source registry when package not found
+   */
+  redirectNotFound: boolean,
+  /**
+   * enable unpkg features, https://github.com/cnpm/cnpmcore/issues/452
+   */
+  enableUnpkg: boolean,
+  /**
+   * enable sync unpkg files
+   */
+  enableSyncUnpkgFiles: boolean;
+  /**
+   * enable sync unpkg files from the white list, https://github.com/cnpm/unpkg-white-list
+   */
+  enableSyncUnpkgFilesWhiteList: boolean;
+  /**
+   * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag.
+   * in most cases, you should set to false to keep the same behavior as source registry.
+   */
+  strictSyncSpecivicVersion: boolean,
+  /**
+  * enable elasticsearch
+  */
+  enableElasticsearch: boolean,
+  /**
+  * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc.
+  */
+  elasticsearchIndex: string,
+  /**
+   * strictly enforces/validates manifest and tgz when publish, https://github.com/cnpm/cnpmcore/issues/542
+   */
+  strictValidateTarballPkg?: boolean,
+
+  /**
+   * strictly enforces/validates dependencies version when publish or sync
+   */
+  strictValidatePackageDeps?: boolean,
+
+  /**
+   * database config
+   */
+  database: {
+    type: DATABASE_TYPE | string,
+  },
+};

app/port/controller/AbstractController.ts

@@ -146,25 +146,25 @@ export abstract class AbstractController extends MiddlewareController {
     return new UnavailableForLegalReasonsError(`${message}, reason: ${reason}`);
   }
 
-  protected async getPackageEntityByFullname(fullname: string): Promise<PackageEntity> {
+  protected async getPackageEntityByFullname(fullname: string, allowSync?: boolean): Promise<PackageEntity> {
     const [ scope, name ] = getScopeAndName(fullname);
-    return await this.getPackageEntity(scope, name);
+    return await this.getPackageEntity(scope, name, allowSync);
   }
 
   // try to get package entity, throw NotFoundError when package not exists
-  protected async getPackageEntity(scope: string, name: string): Promise<PackageEntity> {
+  protected async getPackageEntity(scope: string, name: string, allowSync?:boolean): Promise<PackageEntity> {
     const packageEntity = await this.packageRepository.findPackage(scope, name);
     if (!packageEntity) {
       const fullname = getFullname(scope, name);
-      throw this.createPackageNotFoundErrorWithRedirect(fullname);
+      throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
     }
     return packageEntity;
   }
 
-  protected async getPackageVersionEntity(pkg: PackageEntity, version: string): Promise<PackageVersionEntity> {
+  protected async getPackageVersionEntity(pkg: PackageEntity, version: string, allowSync?: boolean): Promise<PackageVersionEntity> {
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
     if (!packageVersion) {
-      throw new NotFoundError(`${pkg.fullname}@${version} not found`);
+      throw this.createPackageNotFoundErrorWithRedirect(pkg.fullname, version, allowSync);
     }
     return packageVersion;
   }

app/port/controller/DownloadController.ts

@@ -28,15 +28,15 @@ export class DownloadController extends AbstractController {
     const pkg = await this.packageRepository.findPackage(scope, name);
     if (!pkg) throw new NotFoundError(`${fullname} not found`);
     const entities = await this.packageVersionDownloadRepository.query(pkg.packageId, startDate.toDate(), endDate.toDate());
-    const days = {};
-    const versions = {};
+    const days: Record<string, number> = {};
+    const versions: Record<string, { day: string, downloads: number }[]> = {};
     for (const entity of entities) {
       const yearMonth = String(entity.yearMonth);
       const prefix = yearMonth.substring(0, 4) + '-' + yearMonth.substring(4, 6);
       for (let i = 1; i <= 31; i++) {
         const day = String(i).padStart(2, '0');
-        const field = `d${day}`;
-        const counter = entity[field];
+        const field = `d${day}` as keyof typeof entity;
+        const counter = entity[field] as number;
         if (!counter) continue;
         const date = `${prefix}-${day}`;
         days[date] = (days[date] || 0) + counter;
@@ -66,14 +66,14 @@ export class DownloadController extends AbstractController {
   async showTotalDownloads(@HTTPParam() scope: string, @HTTPParam() range: string) {
     const [ startDate, endDate ] = this.checkAndGetRange(range);
     const entities = await this.packageVersionDownloadRepository.query(scope, startDate.toDate(), endDate.toDate());
-    const days = {};
+    const days: Record<string, number> = {};
     for (const entity of entities) {
       const yearMonth = String(entity.yearMonth);
       const prefix = yearMonth.substring(0, 4) + '-' + yearMonth.substring(4, 6);
       for (let i = 1; i <= 31; i++) {
         const day = String(i).padStart(2, '0');
-        const field = `d${day}`;
-        const counter = entity[field];
+        const field = `d${day}` as keyof typeof entity;
+        const counter = entity[field] as number;
         if (!counter) continue;
         const date = `${prefix}-${day}`;
         days[date] = (days[date] || 0) + counter;
@@ -115,4 +115,3 @@ export class DownloadController extends AbstractController {
     return [ startDate, endDate ];
   }
 }
-

app/port/controller/HomeController.ts

@@ -9,6 +9,7 @@ import {
 } from '@eggjs/tegg';
 import { AbstractController } from './AbstractController';
 import { CacheService, DownloadInfo, UpstreamRegistryInfo } from '../../core/service/CacheService';
+import { HomeService } from '../../core/service/HomeService';
 
 const startTime = new Date();
 
@@ -27,7 +28,7 @@ type LegacyInfo = {
 
 type SiteEnvInfo = {
   sync_model: string;
-  sync_binary: string;
+  sync_binary: boolean;
   instance_start_time: Date;
   node_version: string;
   app_version: string;
@@ -51,6 +52,9 @@ export class HomeController extends AbstractController {
   @Inject()
   private readonly cacheService: CacheService;
 
+  @Inject()
+  private readonly homeService: HomeService;
+
   @HTTPMethod({
     // GET /
     // https://github.com/cnpm/cnpmjs.org/blob/master/docs/registry-api.md#schema
@@ -97,4 +101,23 @@ export class HomeController extends AbstractController {
       use: performance.now() - ctx.performanceStarttime!,
     };
   }
+
+  @HTTPMethod({
+    path: '/*',
+    method: HTTPMethodEnum.POST,
+    priority: -Infinity,
+  })
+  async miscPost(@Context() ctx: EggContext) {
+    await this.homeService.misc(ctx.path);
+  }
+
+  @HTTPMethod({
+    path: '/*',
+    method: HTTPMethodEnum.GET,
+    priority: -Infinity,
+  })
+  async miscGet(@Context() ctx: EggContext) {
+    await this.homeService.misc(ctx.path);
+  }
+
 }

app/port/controller/PackageSyncController.ts

@@ -67,13 +67,13 @@ export class PackageSyncController extends AbstractController {
 
     const params = {
       fullname,
-      remoteAuthToken: data.remoteAuthToken,
       tips,
       skipDependencies: !!data.skipDependencies,
       syncDownloadData: !!data.syncDownloadData,
       force: !!data.force,
       // only admin allow to sync history version
       forceSyncHistory: !!data.forceSyncHistory && isAdmin,
+      specificVersions: data.specificVersions,
     };
     ctx.tValidate(SyncPackageTaskRule, params);
     const [ scope, name ] = getScopeAndName(params.fullname);
@@ -96,12 +96,12 @@ export class PackageSyncController extends AbstractController {
     const task = await this.packageSyncerService.createTask(params.fullname, {
       authorIp: ctx.ip,
       authorId: authorized?.user.userId,
-      remoteAuthToken: params.remoteAuthToken,
       tips: params.tips,
       skipDependencies: params.skipDependencies,
       syncDownloadData: params.syncDownloadData,
       forceSyncHistory: params.forceSyncHistory,
       registryId: registry?.registryId,
+      specificVersions: params.specificVersions && JSON.parse(params.specificVersions),
     });
     ctx.logger.info('[PackageSyncController.createSyncTask:success] taskId: %s, fullname: %s',
       task.taskId, fullname);

app/port/controller/PackageTagController.ts

@@ -29,7 +29,7 @@ export class PackageTagController extends AbstractController {
   async showTags(@HTTPParam() fullname: string) {
     const packageEntity = await this.getPackageEntityByFullname(fullname);
     const tagEntities = await this.packageRepository.listPackageTags(packageEntity.packageId);
-    const tags = {};
+    const tags: Record<string, string> = {};
     for (const entity of tagEntities) {
       tags[entity.tag] = entity.version;
     }

app/port/controller/PackageVersionFileController.ts

@@ -19,6 +19,7 @@ import { PackageManagerService } from '../../core/service/PackageManagerService'
 import { PackageVersionFile } from '../../core/entity/PackageVersionFile';
 import { PackageVersion } from '../../core/entity/PackageVersion';
 import { DistRepository } from '../../repository/DistRepository';
+import { Spec } from '../typebox';
 
 type FileItem = {
   path: string,
@@ -65,85 +66,99 @@ export class PackageVersionFileController extends AbstractController {
   }
 
   @HTTPMethod({
-    // PUT /:fullname/:versionOrTag/files
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files`,
+    // PUT /:fullname/:versionSpec/files
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`,
     method: HTTPMethodEnum.PUT,
   })
   @Middleware(AdminAccess)
-  async sync(@HTTPParam() fullname: string, @HTTPParam() versionOrTag: string) {
+  async sync(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionSpec: string) {
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     this.#requireUnpkgEnable();
     const [ scope, name ] = getScopeAndName(fullname);
     const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
-      scope, name, versionOrTag);
+      scope, name, versionSpec);
     if (!packageVersion) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+      throw new NotFoundError(`${fullname}@${versionSpec} not found`);
     }
     const files = await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
     return files.map(file => formatFileItem(file));
   }
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag/files => /:fullname/:versionOrTag/files/${pkg.main}
-    // GET /:fullname/:versionOrTag/files?meta
-    // GET /:fullname/:versionOrTag/files/
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files`,
+    // GET /:fullname/:versionSpec/files => /:fullname/:versionSpec/files/${pkg.main}
+    // GET /:fullname/:versionSpec/files?meta
+    // GET /:fullname/:versionSpec/files/
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`,
     method: HTTPMethodEnum.GET,
   })
   async listFiles(@Context() ctx: EggContext,
       @HTTPParam() fullname: string,
-      @HTTPParam() versionOrTag: string,
+      @HTTPParam() versionSpec: string,
       @HTTPQuery() meta: string) {
     this.#requireUnpkgEnable();
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     ctx.vary(this.config.cnpmcore.cdnVaryHeader);
     const [ scope, name ] = getScopeAndName(fullname);
-    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionOrTag);
+    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionSpec);
     ctx.set('cache-control', META_CACHE_CONTROL);
     const hasMeta = typeof meta === 'string' || ctx.path.endsWith('/files/');
     // meta request
     if (hasMeta) {
       const files = await this.#listFilesByDirectory(packageVersion, '/');
       if (!files) {
-        throw new NotFoundError(`${fullname}@${versionOrTag}/files not found`);
+        throw new NotFoundError(`${fullname}@${versionSpec}/files not found`);
       }
       return files;
     }
-    const { manifest } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionOrTag);
+    const { manifest } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, false, true);
     // GET /foo/1.0.0/files => /foo/1.0.0/files/{main}
-    const indexFile = manifest?.main ?? 'index.js';
+    // ignore empty entry exp: @types/node@20.2.5/
+    const indexFile = manifest?.main || 'index.js';
     ctx.redirect(join(ctx.path, indexFile));
   }
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag/files/:path
-    // GET /:fullname/:versionOrTag/files/:path?meta
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files/:path(.+)`,
+    // GET /:fullname/:versionSpec/files/:path
+    // GET /:fullname/:versionSpec/files/:path?meta
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files/:path(.+)`,
     method: HTTPMethodEnum.GET,
   })
   async raw(@Context() ctx: EggContext,
       @HTTPParam() fullname: string,
-      @HTTPParam() versionOrTag: string,
+      @HTTPParam() versionSpec: string,
       @HTTPParam() path: string,
       @HTTPQuery() meta: string) {
     this.#requireUnpkgEnable();
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     ctx.vary(this.config.cnpmcore.cdnVaryHeader);
     const [ scope, name ] = getScopeAndName(fullname);
     path = `/${path}`;
-    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionOrTag);
+    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionSpec);
     if (path.endsWith('/')) {
       const directory = path.substring(0, path.length - 1);
       const files = await this.#listFilesByDirectory(packageVersion, directory);
       if (!files) {
-        throw new NotFoundError(`${fullname}@${versionOrTag}/files${directory} not found`);
+        throw new NotFoundError(`${fullname}@${versionSpec}/files${directory} not found`);
       }
       ctx.set('cache-control', META_CACHE_CONTROL);
       return files;
     }
 
+    await this.packageVersionFileService.checkPackageVersionInUnpkgWhiteList(scope, name, packageVersion.version);
     const file = await this.packageVersionFileService.showPackageVersionFile(packageVersion, path);
-    if (!file) {
-      throw new NotFoundError(`File ${fullname}@${versionOrTag}${path} not found`);
-    }
     const hasMeta = typeof meta === 'string';
+
+    if (!file) {
+      const possibleFile = await this.#searchPossibleEntries(packageVersion, path);
+      if (possibleFile) {
+        const route = `/${fullname}/${versionSpec}/files${possibleFile.path}${hasMeta ? '?meta' : ''}`;
+        ctx.redirect(route);
+        return;
+      }
+
+      throw new NotFoundError(`File ${fullname}@${versionSpec}${path} not found`);
+    }
+
     if (hasMeta) {
       ctx.set('cache-control', META_CACHE_CONTROL);
       return formatFileItem(file);
@@ -156,51 +171,64 @@ export class PackageVersionFileController extends AbstractController {
     return await this.distRepository.getDistStream(file.dist);
   }
 
-  async #getPackageVersion(ctx: EggContext, fullname: string, scope: string, name: string, versionOrTag: string) {
+  /**
+   * compatibility with unpkg
+   * 1. try to match alias entry. e.g. accessing `index.js` or `index.json` using /index
+   * 2. if given path is directory and has `index.js` file, redirect to it. e.g. using `lib` alias to access `lib/index.js` or `lib/index.json`
+   * @param {PackageVersion} packageVersion packageVersion
+   * @param {String} path  filepath
+   * @return {Promise<PackageVersionFile | undefined>} return packageVersionFile or null
+   */
+  async #searchPossibleEntries(packageVersion: PackageVersion, path: string) {
+    const possiblePath = [ `${path}.js`, `${path}.json`, `${path}/index.js`, `${path}/index.json` ];
+
+    for (const pathItem of possiblePath) {
+      const file = await this.packageVersionFileService.showPackageVersionFile(packageVersion, pathItem);
+
+      if (file) {
+        return file;
+      }
+    }
+  }
+
+  async #getPackageVersion(ctx: EggContext, fullname: string, scope: string, name: string, versionSpec: string) {
     const { blockReason, packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
-      scope, name, versionOrTag);
+      scope, name, versionSpec);
     if (blockReason) {
       this.setCDNHeaders(ctx);
-      throw this.createPackageBlockError(blockReason, fullname, versionOrTag);
+      throw this.createPackageBlockError(blockReason, fullname, versionSpec);
     }
     if (!packageVersion) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+      throw new NotFoundError(`${fullname}@${versionSpec} not found`);
     }
-    if (packageVersion.version !== versionOrTag) {
+    if (packageVersion.version !== versionSpec) {
       ctx.set('cache-control', META_CACHE_CONTROL);
-      const location = ctx.url.replace(`/${fullname}/${versionOrTag}/files`, `/${fullname}/${packageVersion.version}/files`);
+      let location = ctx.url.replace(`/${fullname}/${versionSpec}/files`, `/${fullname}/${packageVersion.version}/files`);
+      location = location.replace(`/${fullname}/${encodeURIComponent(versionSpec)}/files`, `/${fullname}/${packageVersion.version}/files`);
       throw this.createControllerRedirectError(location);
     }
     return packageVersion;
   }
 
   async #listFilesByDirectory(packageVersion: PackageVersion, directory: string) {
-    const files = await this.packageVersionFileService.listPackageVersionFiles(packageVersion, directory);
-    if (!files || files.length === 0) return null;
-    // convert files to directory and file
-    const directories = new Map<string, DirectoryItem>();
+    const { files, directories } = await this.packageVersionFileService.listPackageVersionFiles(packageVersion, directory);
+    if (files.length === 0 && directories.length === 0) return null;
+
+    const info: DirectoryItem = {
+      path: directory,
+      type: 'directory',
+      files: [],
+    };
     for (const file of files) {
-      // make sure parent directories exists
-      const splits = file.directory.split('/');
-      for (const [ index, name ] of splits.entries()) {
-        const parentPath = index === 0 ? '' : `/${splits.slice(1, index).join('/')}`;
-        const directoryPath = parentPath !== '/' ? `${parentPath}/${name}` : `/${name}`;
-        let directoryItem = directories.get(directoryPath);
-        if (!directoryItem) {
-          directoryItem = {
-            path: directoryPath,
-            type: 'directory',
-            files: [],
-          };
-          directories.set(directoryPath, directoryItem);
-          if (parentPath) {
-            // only set the first time
-            directories.get(parentPath!)!.files.push(directoryItem);
-          }
-        }
-      }
-      directories.get(file.directory)!.files.push(formatFileItem(file));
+      info.files.push(formatFileItem(file));
     }
-    return directories.get(directory);
+    for (const name of directories) {
+      info.files.push({
+        path: name,
+        type: 'directory',
+        files: [],
+      } as DirectoryItem);
+    }
+    return info;
   }
 }

app/port/controller/ProxyCacheController.ts

@@ -0,0 +1,143 @@
+import {
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  Inject,
+  HTTPQuery,
+  HTTPParam,
+  Context,
+  EggContext,
+} from '@eggjs/tegg';
+import { ForbiddenError, NotFoundError, UnauthorizedError, NotImplementedError } from 'egg-errors';
+import { AbstractController } from './AbstractController';
+import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
+import { Static } from 'egg-typebox-validate/typebox';
+import { QueryPageOptions } from '../typebox';
+import { FULLNAME_REG_STRING } from '../../common/PackageUtil';
+import {
+  ProxyCacheService,
+  isPkgManifest,
+} from '../../core/service/ProxyCacheService';
+import { SyncMode } from '../../common/constants';
+import { CacheService } from '../../core/service/CacheService';
+
+@HTTPController()
+export class ProxyCacheController extends AbstractController {
+  @Inject()
+  private readonly proxyCacheRepository: ProxyCacheRepository;
+  @Inject()
+  private readonly proxyCacheService: ProxyCacheService;
+  @Inject()
+  private readonly cacheService: CacheService;
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.GET,
+    path: '/-/proxy-cache',
+  })
+  async listProxyCache(
+    @HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'],
+    @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex'],
+  ) {
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) {
+      throw new ForbiddenError('proxy mode is not enabled');
+    }
+    return await this.proxyCacheRepository.listCachedFiles({
+      pageSize,
+      pageIndex,
+    });
+  }
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.GET,
+    path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`,
+  })
+  async showProxyCaches(@HTTPQuery() pageSize: Static<typeof QueryPageOptions>['pageSize'],
+  @HTTPQuery() pageIndex: Static<typeof QueryPageOptions>['pageIndex'], @HTTPParam() fullname: string) {
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) {
+      throw new ForbiddenError('proxy mode is not enabled');
+    }
+    return await this.proxyCacheRepository.listCachedFiles({
+      pageSize,
+      pageIndex,
+    }, fullname);
+  }
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.PATCH,
+    path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`,
+  })
+  async refreshProxyCaches(@HTTPParam() fullname: string) {
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) {
+      throw new ForbiddenError('proxy mode is not enabled');
+    }
+
+    const refreshList = await this.proxyCacheRepository.findProxyCaches(
+      fullname,
+    );
+    if (refreshList.length === 0) {
+      throw new NotFoundError();
+    }
+    await this.cacheService.removeCache(fullname);
+    const taskList = refreshList
+      // only refresh package.json and abbreviated.json
+      .filter(i => isPkgManifest(i.fileType))
+      .map(item => {
+        const task = this.proxyCacheService.createTask(
+          `${item.fullname}/${item.fileType}`,
+          {
+            fullname: item.fullname,
+            fileType: item.fileType,
+          },
+        );
+        return task;
+      });
+    const tasks = await Promise.all(taskList);
+    return {
+      ok: true,
+      tasks,
+    };
+  }
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.DELETE,
+    path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`,
+  })
+  async removeProxyCaches(@HTTPParam() fullname: string) {
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) {
+      throw new ForbiddenError('proxy mode is not enabled');
+    }
+
+    const proxyCachesList = await this.proxyCacheRepository.findProxyCaches(
+      fullname,
+    );
+    if (proxyCachesList.length === 0) {
+      throw new NotFoundError();
+    }
+    await this.cacheService.removeCache(fullname);
+    const removingList = proxyCachesList.map(item => {
+      return this.proxyCacheService.removeProxyCache(item.fullname, item.fileType, item.version);
+    });
+    await Promise.all(removingList);
+    return {
+      ok: true,
+      result: proxyCachesList,
+    };
+  }
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.DELETE,
+    path: '/-/proxy-cache',
+  })
+  async truncateProxyCaches(@Context() ctx: EggContext) {
+    const isAdmin = await this.userRoleManager.isAdmin(ctx);
+    if (!isAdmin) {
+      throw new UnauthorizedError('only admin can do this');
+    }
+
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) {
+      throw new ForbiddenError('proxy mode is not enabled');
+    }
+
+    throw new NotImplementedError('not implemented yet');
+  }
+}

app/port/controller/RegistryController.ts

@@ -13,10 +13,10 @@ import {
 import { NotFoundError } from 'egg-errors';
 import { AbstractController } from './AbstractController';
 import { Static } from 'egg-typebox-validate/typebox';
-import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { RegistryManagerService, UpdateRegistryCmd } from '../../core/service/RegistryManagerService';
 import { AdminAccess } from '../middleware/AdminAccess';
 import { ScopeManagerService } from '../../core/service/ScopeManagerService';
-import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions } from '../typebox';
+import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions, RegistryUpdateOptions } from '../typebox';
 
 @HTTPController()
 export class RegistryController extends AbstractController {
@@ -67,7 +67,7 @@ export class RegistryController extends AbstractController {
   async createRegistry(@Context() ctx: EggContext, @HTTPBody() registryOptions: Static<typeof RegistryCreateOptions>) {
     ctx.tValidate(RegistryCreateOptions, registryOptions);
     const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
-    const { name, changeStream, host, userPrefix = '', type } = registryOptions;
+    const { name, changeStream, host, userPrefix = '', type, authToken } = registryOptions;
     await this.registryManagerService.createRegistry({
       name,
       changeStream,
@@ -75,6 +75,7 @@ export class RegistryController extends AbstractController {
       userPrefix,
       operatorId: authorizedUser.userId,
       type,
+      authToken,
     });
     return { ok: true };
   }
@@ -106,4 +107,29 @@ export class RegistryController extends AbstractController {
     await this.registryManagerService.remove({ registryId: id, operatorId: authorizedUser.userId });
     return { ok: true };
   }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.PATCH,
+  })
+  @Middleware(AdminAccess)
+  async updateRegistry(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() updateRegistryOptions: Partial<UpdateRegistryCmd>) {
+    ctx.tValidate(RegistryUpdateOptions, updateRegistryOptions);
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    } else {
+      const { name, changeStream, host, type, authToken } = registry;
+      const _updateRegistryOptions = {
+        name,
+        changeStream,
+        host,
+        type,
+        authToken,
+        ...updateRegistryOptions,
+      };
+      await this.registryManagerService.updateRegistry(registry.registryId, _updateRegistryOptions);
+    }
+    return { ok: true };
+  }
 }

app/port/controller/TokenController.ts

@@ -13,8 +13,6 @@ import {
 import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from './AbstractController';
 import { TokenType, isGranularToken } from '../../core/entity/Token';
-import { TokenService } from '../../../app/core/service/TokenService';
-import { getFullname } from '../../../app/common/PackageUtil';
 
 // Creating and viewing access tokens
 // https://docs.npmjs.com/creating-and-viewing-access-tokens#viewing-access-tokens
@@ -44,8 +42,6 @@ type GranularTokenOptions = Static<typeof GranularTokenOptionsRule>;
 export class TokenController extends AbstractController {
   @Inject()
   private readonly authAdapter: AuthAdapter;
-  @Inject()
-  private readonly tokenService: TokenService;
   // https://github.com/npm/npm-profile/blob/main/lib/index.js#L233
   @HTTPMethod({
     path: '/-/npm/v1/tokens',
@@ -127,6 +123,7 @@ export class TokenController extends AbstractController {
           readonly: token.isReadonly,
           automation: token.isAutomation,
           created: token.createdAt,
+          lastUsedAt: token.lastUsedAt,
           updated: token.updatedAt,
         };
       });
@@ -134,15 +131,12 @@ export class TokenController extends AbstractController {
     return { objects, total: objects.length, urls: {} };
   }
 
-  private async ensureWebUser() {
+  private async ensureWebUser(ip = '') {
     const userRes = await this.authAdapter.ensureCurrentUser();
     if (!userRes?.name || !userRes?.email) {
       throw new ForbiddenError('need login first');
     }
-    const user = await this.userService.findUserByName(userRes.name);
-    if (!user?.userId) {
-      throw new ForbiddenError('invalid user info');
-    }
+    const user = await this.userService.findOrCreateUser({ name: userRes.name, email: userRes.email, ip });
     return user;
   }
 
@@ -158,7 +152,7 @@ export class TokenController extends AbstractController {
   // 3. Need to implement ensureCurrentUser method in AuthAdapter, or pass in this.user
   async createGranularToken(@Context() ctx: EggContext, @HTTPBody() tokenOptions: GranularTokenOptions) {
     ctx.tValidate(GranularTokenOptionsRule, tokenOptions);
-    const user = await this.ensureWebUser();
+    const user = await this.ensureWebUser(ctx.ip);
 
     // 生成 Token
     const { name, description, allowedPackages, allowedScopes, cidr_whitelist, automation, readonly, expires } = tokenOptions;
@@ -197,19 +191,14 @@ export class TokenController extends AbstractController {
     const tokens = await this.userRepository.listTokens(user.userId);
     const granularTokens = tokens.filter(token => isGranularToken(token));
 
-    for (const token of granularTokens) {
-      const packages = await this.tokenService.listTokenPackages(token);
-      if (Array.isArray(packages)) {
-        token.allowedPackages = packages.map(p => getFullname(p.scope, p.name));
-      }
-    }
     const objects = granularTokens.map(token => {
-      const { name, description, expiredAt, allowedPackages, allowedScopes } = token;
+      const { name, description, expiredAt, allowedPackages, allowedScopes, lastUsedAt, type } = token;
       return {
         name,
         description,
         allowedPackages,
         allowedScopes,
+        lastUsedAt,
         expiredAt,
         token: token.tokenMark,
         key: token.tokenKey,
@@ -217,6 +206,7 @@ export class TokenController extends AbstractController {
         readonly: token.isReadonly,
         created: token.createdAt,
         updated: token.updatedAt,
+        type,
       };
     });
     return { objects, total: granularTokens.length, urls: {} };

app/port/controller/UserController.ts

@@ -17,6 +17,7 @@ import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from './AbstractController';
 import { LoginResultCode } from '../../common/enum/User';
 import { sha512 } from '../../common/UserUtil';
+import { isGranularToken } from '../../core/entity/Token';
 
 // body: {
 //   _id: 'org.couchdb.user:dddd',
@@ -89,7 +90,7 @@ export class UserController extends AbstractController {
       ctx.status = 201;
       return {
         ok: true,
-        id: `org.couchdb.user:${result.user?.name}`,
+        id: `org.couchdb.user:${result.user?.displayName}`,
         rev: result.user?.userId,
         token: result.token?.token,
       };
@@ -112,7 +113,7 @@ export class UserController extends AbstractController {
     ctx.status = 201;
     return {
       ok: true,
-      id: `org.couchdb.user:${userEntity.name}`,
+      id: `org.couchdb.user:${userEntity.displayName}`,
       rev: userEntity.userId,
       token: token.token,
     };
@@ -139,14 +140,14 @@ export class UserController extends AbstractController {
     method: HTTPMethodEnum.GET,
   })
   async showUser(@Context() ctx: EggContext, @HTTPParam() username: string) {
-    const user = await this.userRepository.findUserByName(username);
+    const user = await this.userService.findUserByNameOrDisplayName(username);
     if (!user) {
       throw new NotFoundError(`User "${username}" not found`);
     }
     const authorized = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
     return {
-      _id: `org.couchdb.user:${user.name}`,
-      name: user.name,
+      _id: `org.couchdb.user:${user.displayName}`,
+      name: user.displayName,
       email: authorized ? user.email : undefined,
     };
   }
@@ -157,10 +158,34 @@ export class UserController extends AbstractController {
     method: HTTPMethodEnum.GET,
   })
   async whoami(@Context() ctx: EggContext) {
-    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const authorizedRes = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
+    const { token, user } = authorizedRes!;
+
+    if (isGranularToken(token)) {
+      const { name, description, expiredAt, allowedPackages, allowedScopes, lastUsedAt, type } = token;
+      return {
+        username: user.displayName,
+        name,
+        description,
+        allowedPackages,
+        allowedScopes,
+        lastUsedAt,
+        expiredAt,
+        // do not return token value
+        // token: token.token,
+        key: token.tokenKey,
+        cidr_whitelist: token.cidrWhitelist,
+        readonly: token.isReadonly,
+        created: token.createdAt,
+        updated: token.updatedAt,
+        type,
+      };
+    }
     return {
-      username: authorizedUser.displayName,
+      username: user.displayName,
     };
+
   }
 
   // https://github.com/cnpm/cnpmcore/issues/64
@@ -184,7 +209,7 @@ export class UserController extends AbstractController {
       //   "pending": false,
       //   "mode": "auth-only"
       // },
-      name: authorizedUser.name,
+      name: authorizedUser.displayName,
       email: authorizedUser.email,
       email_verified: false,
       created: authorizedUser.createdAt,

app/port/controller/admin/PaddingVersionController.ts

@@ -0,0 +1,43 @@
+import {
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  Inject,
+  HTTPQuery, Context, EggContext,
+} from '@eggjs/tegg';
+import { AbstractController } from '../AbstractController';
+import { FixNoPaddingVersionService } from '../../../core/service/FixNoPaddingVersionService';
+
+@HTTPController()
+export class PaddingVersionController extends AbstractController {
+  @Inject()
+  private readonly fixNoPaddingVersionService: FixNoPaddingVersionService;
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.PUT,
+    path: '/-/admin/npm/fixPaddingVersion',
+  })
+  async fixNoPaddingVersion(@Context() ctx: EggContext, @HTTPQuery() id: string) {
+    const isAdmin = await this.userRoleManager.isAdmin(ctx);
+    if (!isAdmin) {
+      return {
+        ok: false,
+        error: 'only admin can do this',
+      };
+    }
+    let idNum: number | undefined;
+    if (id) {
+      idNum = parseInt(id);
+      if (Number.isNaN(idNum)) {
+        return {
+          ok: false,
+          error: `id is not a number ${id}`,
+        };
+      }
+    }
+    await this.fixNoPaddingVersionService.fixPaddingVersion(idNum);
+    return {
+      ok: true,
+    };
+  }
+}

app/port/controller/package/DownloadPackageVersionTar.ts

@@ -1,3 +1,4 @@
+import { PassThrough } from 'node:stream';
 import {
   NotFoundError,
 } from 'egg-errors';
@@ -12,36 +13,95 @@ import {
 } from '@eggjs/tegg';
 import { AbstractController } from '../AbstractController';
 import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil';
+import { SyncMode } from '../../../common/constants';
 import { NFSAdapter } from '../../../common/adapter/NFSAdapter';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
+import { PackageSyncerService } from '../../../core/service/PackageSyncerService';
+import { RegistryManagerService } from '../../../core/service/RegistryManagerService';
+import { PackageVersionService } from '../../../core/service/PackageVersionService';
 
 @HTTPController()
 export class DownloadPackageVersionTarController extends AbstractController {
   @Inject()
   private packageManagerService: PackageManagerService;
   @Inject()
+  registryManagerService: RegistryManagerService;
+  @Inject()
+  private proxyCacheService: ProxyCacheService;
+  @Inject()
+  private packageSyncerService: PackageSyncerService;
+  @Inject()
+  private packageVersionService: PackageVersionService;
+  @Inject()
   private nfsAdapter: NFSAdapter;
 
+  // Support OPTIONS Request on tgz download
+  @HTTPMethod({
+    // GET /:fullname/-/:filenameWithVersion.tgz
+    path: `/:fullname(${FULLNAME_REG_STRING})/-/:filenameWithVersion.tgz`,
+    method: HTTPMethodEnum.OPTIONS,
+  })
+  async downloadForOptions(@Context() ctx: EggContext) {
+    ctx.set('access-control-allow-origin', '*');
+    ctx.set('access-control-allow-methods', 'GET,HEAD');
+    ctx.status = 204;
+  }
+
   @HTTPMethod({
     // GET /:fullname/-/:filenameWithVersion.tgz
     path: `/:fullname(${FULLNAME_REG_STRING})/-/:filenameWithVersion.tgz`,
     method: HTTPMethodEnum.GET,
   })
   async download(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() filenameWithVersion: string) {
-    // try nfs url first, avoid db query
     // tgz file storeKey: `/packages/${this.fullname}/${version}/${filename}`
     const version = this.getAndCheckVersionFromFilename(ctx, fullname, filenameWithVersion);
     const storeKey = `/packages/${fullname}/${version}/${filenameWithVersion}.tgz`;
     const downloadUrl = await this.nfsAdapter.getDownloadUrl(storeKey);
-    if (downloadUrl) {
+
+    // block tgz only all versions have been blocked
+    const blockInfo = await this.packageVersionService.findBlockInfo(fullname);
+    if (blockInfo?.reason) {
+      this.setCDNHeaders(ctx);
+      ctx.logger.info('[PackageController:downloadVersionTar] %s@%s, block for %s',
+        fullname, version, blockInfo.reason);
+      throw this.createPackageBlockError(blockInfo.reason, fullname, version);
+    }
+
+    if (this.config.cnpmcore.syncMode === SyncMode.all && downloadUrl) {
+      // try nfs url first, avoid db query
       this.packageManagerService.plusPackageVersionCounter(fullname, version);
       ctx.redirect(downloadUrl);
       return;
     }
 
+    // check package version in database
+    const allowSync = this.getAllowSync(ctx);
+    let pkg;
+    let packageVersion;
+    try {
+      pkg = await this.getPackageEntityByFullname(fullname, allowSync);
+      packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync);
+    } catch (error) {
+      if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
+        // proxy mode package version not found.
+        const tgzStream = await this.getTgzProxyStream(ctx, fullname, version);
+        this.packageManagerService.plusPackageVersionCounter(fullname, version);
+        const passThroughRemoteStream = new PassThrough();
+        tgzStream.pipe(passThroughRemoteStream);
+        ctx.attachment(`${filenameWithVersion}.tgz`);
+        return passThroughRemoteStream;
+      }
+      throw error;
+    }
+
+    // read by nfs url
+    if (downloadUrl) {
+      this.packageManagerService.plusPackageVersionCounter(fullname, version);
+      ctx.redirect(downloadUrl);
+      return;
+    }
     // read from database
-    const pkg = await this.getPackageEntityByFullname(fullname);
-    const packageVersion = await this.getPackageVersionEntity(pkg, version);
     ctx.logger.info('[PackageController:downloadVersionTar] %s@%s, packageVersionId: %s',
       pkg.fullname, version, packageVersion.packageVersionId);
     const urlOrStream = await this.packageManagerService.downloadPackageVersionTar(packageVersion);
@@ -68,4 +128,42 @@ export class DownloadPackageVersionTarController extends AbstractController {
     const filenameWithVersion = getScopeAndName(fullnameWithVersion)[1];
     return await this.download(ctx, fullname, filenameWithVersion);
   }
+
+  private async getTgzProxyStream(ctx: EggContext, fullname: string, version: string) {
+    const { headers, status, res } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx);
+    ctx.status = status;
+    ctx.set(headers as { [key: string]: string | string[] });
+    ctx.runInBackground(async () => {
+      const task = await this.packageSyncerService.createTask(fullname, {
+        authorIp: ctx.ip,
+        authorId: `pid_${process.pid}`,
+        tips: `Sync specific version in proxy mode cause by "${ctx.href}"`,
+        skipDependencies: true,
+        specificVersions: [ version ],
+      });
+      ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s',
+        task.taskId, fullname);
+    });
+    return res;
+  }
+
+  // Compatible Verdaccio path style
+
+  @HTTPMethod({
+    // GET /:fullname/-/:scope/:filenameWithVersion.tgz
+    path: `/:fullname(${FULLNAME_REG_STRING})/-/:scope/:filenameWithVersion.tgz`,
+    method: HTTPMethodEnum.OPTIONS,
+  })
+  async downloadVerdaccioPathStyleorOptions(@Context() ctx: EggContext) {
+    return this.downloadForOptions(ctx);
+  }
+
+  @HTTPMethod({
+    // GET /:fullname/-/:scope/:filenameWithVersion.tgz
+    path: `/:fullname(${FULLNAME_REG_STRING})/-/:scope/:filenameWithVersion.tgz`,
+    method: HTTPMethodEnum.GET,
+  })
+  async downloadVerdaccioPathStyle(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() filenameWithVersion: string) {
+    return this.download(ctx, fullname, filenameWithVersion);
+  }
 }

app/port/controller/package/SavePackageVersionController.ts

@@ -1,7 +1,9 @@
 import { PackageJson, Simplify } from 'type-fest';
+import { isEqual } from 'lodash';
 import {
   UnprocessableEntityError,
   ForbiddenError,
+  ConflictError,
 } from 'egg-errors';
 import {
   HTTPController,
@@ -17,8 +19,9 @@ import * as ssri from 'ssri';
 import validateNpmPackageName from 'validate-npm-package-name';
 import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from '../AbstractController';
-import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil';
+import { getScopeAndName, FULLNAME_REG_STRING, extractPackageJSON } from '../../../common/PackageUtil';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { PackageVersion as PackageVersionEntity } from '../../../core/entity/PackageVersion';
 import {
   VersionRule,
   TagWithVersionRule,
@@ -27,6 +30,9 @@ import {
 } from '../../typebox';
 import { RegistryManagerService } from '../../../core/service/RegistryManagerService';
 import { PackageJSONType } from '../../../repository/PackageRepository';
+import { CacheAdapter } from '../../../common/adapter/CacheAdapter';
+
+const STRICT_CHECK_TARBALL_FIELDS: (keyof PackageJson)[] = [ 'name', 'version', 'scripts', 'dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies', 'license', 'licenses', 'bin' ];
 
 type PackageVersion = Simplify<PackageJson.PackageJsonStandard & {
   name: 'string';
@@ -71,6 +77,9 @@ export class SavePackageVersionController extends AbstractController {
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
 
+  @Inject()
+  private readonly cacheAdapter: CacheAdapter;
+
   // https://github.com/cnpm/cnpmjs.org/blob/master/docs/registry-api.md#publish-a-new-package
   // https://github.com/npm/libnpmpublish/blob/main/publish.js#L43
   @HTTPMethod({
@@ -87,11 +96,17 @@ export class SavePackageVersionController extends AbstractController {
     if (fullname !== pkg.name) {
       throw new UnprocessableEntityError(`fullname(${fullname}) not match package.name(${pkg.name})`);
     }
+
     // Using https://github.com/npm/validate-npm-package-name to validate package name
     const validateResult = validateNpmPackageName(pkg.name);
     if (!validateResult.validForNewPackages) {
-      const errors = (validateResult.errors || validateResult.warnings).join(', ');
-      throw new UnprocessableEntityError(`package.name invalid, errors: ${errors}`);
+      // if pkg already exists, still allow to publish
+      const [ scope, name ] = getScopeAndName(fullname);
+      const pkg = await this.packageRepository.findPackage(scope, name);
+      if (!pkg) {
+        const errors = (validateResult.errors || validateResult.warnings || []).join(', ');
+        throw new UnprocessableEntityError(`package.name invalid, errors: ${errors}`);
+      }
     }
     const versions = Object.values(pkg.versions);
     if (versions.length === 0) {
@@ -123,11 +138,25 @@ export class SavePackageVersionController extends AbstractController {
 
     const attachment = attachments[attachmentFilename];
     const distTags = pkg['dist-tags'] ?? {};
-    const tagName = Object.keys(distTags)[0];
-    if (!tagName) {
+    let tagNames = Object.keys(distTags);
+    if (tagNames.length === 0) {
       throw new UnprocessableEntityError('dist-tags is empty');
     }
-    const tagWithVersion = { tag: tagName, version: distTags[tagName] };
+
+    const [ scope, name ] = getScopeAndName(fullname);
+    // see @https://github.com/cnpm/cnpmcore/issues/574
+    // add default latest tag
+    if (!pkg['dist-tags']!.latest) {
+      const existsPkg = await this.packageRepository.findPackage(scope, name);
+      const existsLatestTag = existsPkg && await this.packageRepository.findPackageTag(existsPkg?.packageId, 'latest');
+      if (!existsPkg || !existsLatestTag) {
+        this.logger.warn('[package:version:add] add default latest tag');
+        pkg['dist-tags']!.latest = pkg['dist-tags']![tagNames[0]];
+        tagNames = [ ...tagNames, 'latest' ];
+      }
+    }
+
+    const tagWithVersion = { tag: tagNames[0], version: distTags[tagNames[0]] };
     ctx.tValidate(TagWithVersionRule, tagWithVersion);
     if (tagWithVersion.version !== packageVersion.version) {
       throw new UnprocessableEntityError(`dist-tags version "${tagWithVersion.version}" not match package version "${packageVersion.version}"`);
@@ -165,7 +194,20 @@ export class SavePackageVersionController extends AbstractController {
       }
     }
 
-    const [ scope, name ] = getScopeAndName(fullname);
+    // https://github.com/cnpm/cnpmcore/issues/542
+    // check tgz & manifests
+    if (this.config.cnpmcore.strictValidateTarballPkg) {
+      const tarballPkg = await extractPackageJSON(tarballBytes);
+      const versionManifest = pkg.versions[tarballPkg.version];
+      const diffKeys = STRICT_CHECK_TARBALL_FIELDS.filter(key => {
+        const targetKey = key as unknown as keyof typeof versionManifest;
+        return !isEqual(tarballPkg[key], versionManifest[targetKey]);
+      });
+      if (diffKeys.length > 0) {
+        throw new UnprocessableEntityError(`${diffKeys} mismatch between tarball and manifest`);
+      }
+    }
+
 
     // make sure readme is string
     const readme = typeof packageVersion.readme === 'string' ? packageVersion.readme : '';
@@ -177,28 +219,39 @@ export class SavePackageVersionController extends AbstractController {
     }
 
     const registry = await this.registryManagerService.ensureSelfRegistry();
-    const packageVersionEntity = await this.packageManagerService.publish({
-      scope,
-      name,
-      version: packageVersion.version,
-      description: packageVersion.description,
-      packageJson: packageVersion as PackageJSONType,
-      readme,
-      dist: {
-        content: tarballBytes,
-      },
-      tag: tagWithVersion.tag,
-      registryId: registry.registryId,
-      isPrivate: true,
-    }, user);
+
+    let packageVersionEntity: PackageVersionEntity | undefined;
+    const lockName = `${pkg.name}:publish`;
+    const lockRes = await this.cacheAdapter.usingLock(`${pkg.name}:publish`, 60, async () => {
+      packageVersionEntity = await this.packageManagerService.publish({
+        scope,
+        name,
+        version: packageVersion.version,
+        description: packageVersion.description as string,
+        packageJson: packageVersion as PackageJSONType,
+        readme,
+        dist: {
+          content: tarballBytes,
+        },
+        tags: tagNames,
+        registryId: registry.registryId,
+        isPrivate: true,
+      }, user);
+    });
+
+    // lock fail
+    if (!lockRes) {
+      this.logger.warn('[package:version:add] check lock:%s fail', lockName);
+      throw new ConflictError('Unable to create the publication lock, please try again later.');
+    }
 
     this.logger.info('[package:version:add] %s@%s, packageVersionId: %s, tag: %s, userId: %s',
-      packageVersion.name, packageVersion.version, packageVersionEntity.packageVersionId,
-      tagWithVersion.tag, user.userId);
+      packageVersion.name, packageVersion.version, packageVersionEntity?.packageVersionId,
+      tagWithVersion.tag, user?.userId);
     ctx.status = 201;
     return {
       ok: true,
-      rev: `${packageVersionEntity.id}-${packageVersionEntity.packageVersionId}`,
+      rev: `${packageVersionEntity?.id}-${packageVersionEntity?.packageVersionId}`,
     };
   }
 

app/port/controller/package/SearchPackageController.ts

@@ -0,0 +1,69 @@
+import {
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  HTTPQuery,
+  Inject,
+  Middleware,
+  Context,
+  EggContext,
+} from '@eggjs/tegg';
+import { Static } from 'egg-typebox-validate/typebox';
+import { E451 } from 'egg-errors';
+
+import { AbstractController } from '../AbstractController';
+import { SearchQueryOptions } from '../../typebox';
+import { PackageSearchService } from '../../../core/service/PackageSearchService';
+import { FULLNAME_REG_STRING } from '../../../common/PackageUtil';
+import { AdminAccess } from '../../middleware/AdminAccess';
+
+@HTTPController()
+export class SearchPackageController extends AbstractController {
+  @Inject()
+  private readonly packageSearchService: PackageSearchService;
+
+  @HTTPMethod({
+    // GET /-/v1/search?text=react&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
+    path: '/-/v1/search',
+    method: HTTPMethodEnum.GET,
+  })
+  async search(
+    @Context() ctx: EggContext,
+    @HTTPQuery() text: Static<typeof SearchQueryOptions>['text'],
+    @HTTPQuery() from: Static<typeof SearchQueryOptions>['from'],
+    @HTTPQuery() size: Static<typeof SearchQueryOptions>['size'],
+  ) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const data = await this.packageSearchService.searchPackage(text, from, size);
+    this.setCDNHeaders(ctx);
+    return data;
+  }
+
+  @HTTPMethod({
+    path: `/-/v1/search/sync/:fullname(${FULLNAME_REG_STRING})`,
+    method: HTTPMethodEnum.PUT,
+  })
+  async sync(@HTTPParam() fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const name = await this.packageSearchService.syncPackage(fullname, true);
+    return { package: name };
+  }
+
+  @HTTPMethod({
+    path: `/-/v1/search/sync/:fullname(${FULLNAME_REG_STRING})`,
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async delete(@HTTPParam() fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const name = await this.packageSearchService.removePackage(fullname);
+    return { package: name };
+  }
+}

app/port/controller/package/ShowPackageController.ts

@@ -12,6 +12,10 @@ import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUti
 import { isSyncWorkerRequest } from '../../../common/SyncUtil';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
 import { CacheService } from '../../../core/service/CacheService';
+import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants';
+import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
+import { calculateIntegrity } from '../../../common/PackageUtil';
+import { DIST_NAMES } from '../../../core/entity/Package';
 
 @HTTPController()
 export class ShowPackageController extends AbstractController {
@@ -19,6 +23,8 @@ export class ShowPackageController extends AbstractController {
   private packageManagerService: PackageManagerService;
   @Inject()
   private cacheService: CacheService;
+  @Inject()
+  private proxyCacheService: ProxyCacheService;
 
   @HTTPMethod({
     // GET /:fullname
@@ -29,8 +35,7 @@ export class ShowPackageController extends AbstractController {
   async show(@Context() ctx: EggContext, @HTTPParam() fullname: string) {
     const [ scope, name ] = getScopeAndName(fullname);
     const isSync = isSyncWorkerRequest(ctx);
-    const abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
-    const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType;
+    const isFullManifests = ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE;
 
     // handle cache
     // fallback to db when cache error
@@ -64,10 +69,22 @@ export class ShowPackageController extends AbstractController {
 
     // handle cache miss
     let result: { etag: string; data: any, blockReason: string };
-    if (isFullManifests) {
-      result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync);
+    if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
+      // proxy mode
+      const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS;
+      const { data: sourceManifest } = await this.proxyCacheService.getProxyResponse(ctx, { dataType: 'json' });
+      const pkgManifest = this.proxyCacheService.replaceTarballUrl(sourceManifest, fileType);
+
+      const nfsBytes = Buffer.from(JSON.stringify(pkgManifest));
+      const { shasum: etag } = await calculateIntegrity(nfsBytes);
+      result = { data: pkgManifest, etag, blockReason: '' };
     } else {
-      result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync);
+      // sync mode
+      if (isFullManifests) {
+        result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync);
+      } else {
+        result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync);
+      }
     }
     const { etag, data, blockReason } = result;
     // 404, no data

app/port/controller/package/ShowPackageVersionController.ts

@@ -7,38 +7,78 @@ import {
   Context,
   EggContext,
 } from '@eggjs/tegg';
-import { NotFoundError } from 'egg-errors';
 import { AbstractController } from '../AbstractController';
-import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil';
+import {
+  getScopeAndName,
+  FULLNAME_REG_STRING,
+} from '../../../common/PackageUtil';
 import { isSyncWorkerRequest } from '../../../common/SyncUtil';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { ProxyCacheService } from '../../../core/service/ProxyCacheService';
+import { Spec } from '../../../port/typebox';
+import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants';
+import { DIST_NAMES } from '../../../core/entity/Package';
+import { NotFoundError } from 'egg-errors';
 
 @HTTPController()
 export class ShowPackageVersionController extends AbstractController {
   @Inject()
   private packageManagerService: PackageManagerService;
+  @Inject()
+  private proxyCacheService: ProxyCacheService;
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag`,
+    // GET /:fullname/:versionSpec
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec`,
     method: HTTPMethodEnum.GET,
   })
-  async show(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionOrTag: string) {
+  async show(
+    @Context() ctx: EggContext,
+    @HTTPParam() fullname: string,
+    @HTTPParam() versionSpec: string,
+  ) {
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     const [ scope, name ] = getScopeAndName(fullname);
     const isSync = isSyncWorkerRequest(ctx);
-    const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionOrTag, isSync);
-    if (!pkg) {
-      const allowSync = this.getAllowSync(ctx);
-      throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
-    }
+    const isFullManifests =
+      ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE;
+
+    const { blockReason, manifest, pkg } =
+      await this.packageManagerService.showPackageVersionManifest(
+        scope,
+        name,
+        versionSpec,
+        isSync,
+        isFullManifests,
+      );
+    const allowSync = this.getAllowSync(ctx);
+
     if (blockReason) {
       this.setCDNHeaders(ctx);
-      throw this.createPackageBlockError(blockReason, fullname, versionOrTag);
+      throw this.createPackageBlockError(blockReason, fullname, versionSpec);
     }
-    if (!manifest) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+
+    if (!pkg || !manifest) {
+      if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
+        const fileType = isFullManifests
+          ? DIST_NAMES.MANIFEST
+          : DIST_NAMES.ABBREVIATED;
+        return await this.proxyCacheService.getPackageVersionManifest(
+          fullname,
+          fileType,
+          versionSpec,
+        );
+      }
+
+      if (!pkg) {
+        throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
+      }
+      if (!manifest) {
+        throw new NotFoundError(`${fullname}@${versionSpec} not found`);
+      }
     }
+
     this.setCDNHeaders(ctx);
     return manifest;
   }

app/port/controller/package/UpdatePackageController.ts

@@ -50,17 +50,21 @@ export class UpdatePackageController extends AbstractController {
     ctx.tValidate(MaintainerDataRule, data);
     const ensureRes = await this.ensurePublishAccess(ctx, fullname, true);
     const pkg = ensureRes.pkg!;
+    const registry = await this.packageManagerService.getSourceRegistry(pkg);
     // make sure all maintainers exists
     const users: UserEntity[] = [];
     for (const maintainer of data.maintainers) {
-      // TODO check userPrefix
+      if (registry?.userPrefix && !maintainer.name.startsWith(registry.userPrefix)) {
+        maintainer.name = `${registry?.userPrefix}${maintainer.name}`;
+      }
       const user = await this.userRepository.findUserByName(maintainer.name);
       if (!user) {
         throw new UnprocessableEntityError(`Maintainer "${maintainer.name}" not exists`);
       }
       users.push(user);
     }
-    await this.packageManagerService.replacePackageMaintainers(pkg, users);
+
+    await this.packageManagerService.replacePackageMaintainersAndDist(pkg, users);
     return { ok: true };
   }
 

app/port/login.html

@@ -175,8 +175,8 @@
         <button id="submit" disabled="true">Sign in</button>
       </div>
     </div>
-    <script src="https://gw.alipayobjects.com/os/lib/jsencrypt/3.3.1/bin/jsencrypt.min.js"></script>
-    <script src="https://gw.alipayobjects.com/os/lib/jquery/3.6.3/dist/jquery.min.js"></script>
+    <script src="https://registry.npmmirror.com/jsencrypt/3.3.2/files/bin/jsencrypt.min.js"></script>
+    <script src="https://registry.npmmirror.com/jquery/3.6.3/files/dist/jquery.min.js"></script>
     <script>
       $(function() {
         const $submitBtn = $('#submit');

app/port/middleware/ErrorHandler.ts

@@ -31,8 +31,9 @@ export async function ErrorHandler(ctx: EggContext, next: Next) {
     }
 
     // http status, default is DEFAULT_SERVER_ERROR_STATUS
-    ctx.status = err.status || DEFAULT_SERVER_ERROR_STATUS;
-    if (ctx.status >= DEFAULT_SERVER_ERROR_STATUS) {
+    ctx.status = (typeof err.status === 'number' && err.status >= 200) ? err.status : DEFAULT_SERVER_ERROR_STATUS;
+    // don't log NotImplementedError
+    if (ctx.status >= DEFAULT_SERVER_ERROR_STATUS && err.name !== 'NotImplementedError') {
       ctx.logger.error(err);
     }
     let message = err.message;

app/port/schedule/CheckProxyCacheUpdateWorker.ts

@@ -0,0 +1,53 @@
+import { EggAppConfig, EggLogger } from 'egg';
+import { CronParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
+import { Inject } from '@eggjs/tegg';
+import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository';
+import { SyncMode } from '../../common/constants';
+import { ProxyCacheService, isPkgManifest } from '../../core/service/ProxyCacheService';
+
+@Schedule<CronParams>({
+  type: ScheduleType.WORKER,
+  scheduleData: {
+    cron: '0 3 * * *', // run every day at 03:00
+  },
+})
+export class CheckProxyCacheUpdateWorker {
+
+  @Inject()
+  private readonly config: EggAppConfig;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  @Inject()
+  private proxyCacheService: ProxyCacheService;
+
+  @Inject()
+  private readonly proxyCacheRepository:ProxyCacheRepository;
+
+  async subscribe() {
+    if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return;
+    let pageIndex = 0;
+    let { data: list } = await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex });
+    while (list.length !== 0) {
+      for (const item of list) {
+        try {
+          if (isPkgManifest(item.fileType)) {
+            // 仅manifests需要更新，指定版本的package.json文件发布后不会改变
+            const task = await this.proxyCacheService.createTask(`${item.fullname}/${item.fileType}`, {
+              fullname: item.fullname,
+              fileType: item.fileType,
+            });
+            this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s',
+              pageIndex, task.taskId, task.targetName);
+          }
+        } catch (err) {
+          this.logger.error(err);
+        }
+      }
+      pageIndex++;
+      list = (await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex })).data;
+    }
+
+  }
+}

app/port/schedule/CheckRecentlyUpdatedPackages.ts

@@ -30,7 +30,7 @@ export class CheckRecentlyUpdatedPackages {
   private readonly httpclient: EggHttpClient;
 
   async subscribe() {
-    const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin ];
+    const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin, SyncMode.proxy ];
     if (notAllowUpdateModeList.includes(this.config.cnpmcore.syncMode) || !this.config.cnpmcore.enableCheckRecentlyUpdated) return;
     const pageSize = 36;
     const pageCount = this.config.env === 'unittest' ? 2 : 5;

app/port/schedule/SyncBinaryWorker.ts

@@ -2,6 +2,7 @@ import { EggAppConfig, EggLogger } from 'egg';
 import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
 import { Inject } from '@eggjs/tegg';
 import { BinarySyncerService } from '../../core/service/BinarySyncerService';
+import { isTimeoutError } from '../../common/ErrorUtil';
 
 @Schedule<IntervalParams>({
   type: ScheduleType.ALL,
@@ -29,7 +30,19 @@ export class SyncBinaryWorker {
     this.logger.info('[SyncBinaryWorker:executeTask:start] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
       task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
       startTime - task.updatedAt.getTime());
-    await this.binarySyncerService.executeTask(task);
+    try {
+      await this.binarySyncerService.executeTask(task);
+    } catch (err) {
+      const use = Date.now() - startTime;
+      this.logger.warn('[SyncBinaryWorker:executeTask:error] taskId: %s, targetName: %s, use %sms, error: %s',
+        task.taskId, task.targetName, use, err.message);
+      if (isTimeoutError(err)) {
+        this.logger.warn(err);
+      } else {
+        this.logger.error(err);
+      }
+      return;
+    }
     const use = Date.now() - startTime;
     this.logger.info('[SyncBinaryWorker:executeTask:success] taskId: %s, targetName: %s, use %sms',
       task.taskId, task.targetName, use);