feat: devcontainer with codespaces

[skip ci]

## [3.48.3](https://github.com/cnpm/cnpmcore/compare/v3.48.2...v3.48.3) (2023-11-06)

### Bug Fixes

* es query script score syntax fix and add error handler for 404 error ([#607](https://github.com/cnpm/cnpmcore/issues/607)) ([8e1f4ca](8e1f4ca880))

.devcontainer/devcontainer.json

@@ -0,0 +1,43 @@
+// Update the VARIANT arg in docker-compose.yml to pick a Node.js version
+{
+	"name": "Node.js && Redis && MySQL",
+	"dockerComposeFile": "../docker-compose.yml",
+	"service": "app",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+			]
+		}
+	},
+
+	// Forwards ports
+	"forwardPorts": [		
+		8080
+	],
+	"portsAttributes": {
+		"8080": {
+			"label": "Adminer",
+			"onAutoForward": "notify"
+		}
+	},
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": [
+		"bash .devcontainer/scripts/postCreateCommand.sh"		
+	],
+
+	// Container Env
+	"containerEnv": {
+		"MYSQL_HOST": "mysql",
+		"MYSQL_USER": "root",
+		"MYSQL_PASSWORD": "root"
+	},
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "node"
+}

.devcontainer/scripts/postCreateCommand.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+pnpm i
+socat TCP4-LISTEN:8080,reuseaddr,fork TCP:adminer:8080 &

.dockerignore

@@ -0,0 +1,6 @@
+logs
+node_modules
+run
+typings
+.cnpmcore*
+coverage

.github/workflows/chatgpt-cr.yml

@@ -1,23 +0,0 @@
-name: 🤖 ChatGPT Code Review
-
-permissions:
-  contents: read
-  pull-requests: write
-
-on:
-  pull_request:
-    types: [opened, reopened, synchronize]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: anc95/ChatGPT-CodeReview@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          # Optional
-          LANGUAGE: Chinese
-          MODEL:
-          top_p: 1
-          temperature: 1

.github/workflows/codeql-analysis.yml

@@ -13,12 +13,9 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
+    branches: [ master ]
   pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '41 13 * * 3'
+    branches: [ master ]
 
 jobs:
   analyze:

.github/workflows/nodejs.yml

@@ -3,7 +3,11 @@
 
 name: Node.js CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
 
 jobs:
   test-mysql57-fs-nfs:

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   release:
     name: Node.js
-    uses: node-modules/github-actions/.github/workflows/node-release.yml@master
+    uses: cnpm/github-actions/.github/workflows/node-release.yml@master
     secrets:
       NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       GIT_TOKEN: ${{ secrets.GIT_TOKEN }}

.gitignore

@@ -8,6 +8,7 @@ lerna-debug.log*
 
 .npmrc
 package-lock.json
+pnpm-lock.yaml
 
 config/config.prod.ts
 config/**/*.js

CHANGELOG.md

@@ -1,5 +1,455 @@
 # Changelog
 
+## [3.48.3](https://github.com/cnpm/cnpmcore/compare/v3.48.2...v3.48.3) (2023-11-06)
+
+
+### Bug Fixes
+
+* es query script score syntax fix and add error handler for 404 error ([#607](https://github.com/cnpm/cnpmcore/issues/607)) ([8e1f4ca](https://github.com/cnpm/cnpmcore/commit/8e1f4ca880c6ad09f766807e6a751b5ae960b550))
+
+## [3.48.2](https://github.com/cnpm/cnpmcore/compare/v3.48.1...v3.48.2) (2023-11-03)
+
+
+### Bug Fixes
+
+* should set OPTIONS on access-control-allow-methods ([#608](https://github.com/cnpm/cnpmcore/issues/608)) ([0179ef3](https://github.com/cnpm/cnpmcore/commit/0179ef364a5bc6aac5eafafee7136bf61405ee43))
+
+## [3.48.1](https://github.com/cnpm/cnpmcore/compare/v3.48.0...v3.48.1) (2023-11-03)
+
+
+### Bug Fixes
+
+* should set access-control-allow-origin and headers ([#606](https://github.com/cnpm/cnpmcore/issues/606)) ([18ef7f4](https://github.com/cnpm/cnpmcore/commit/18ef7f49affd2656cdba26fe21078f46ca1e0cc5))
+
+## [3.48.0](https://github.com/cnpm/cnpmcore/compare/v3.47.2...v3.48.0) (2023-11-03)
+
+
+### Features
+
+* allow OPTIONS request on tgz downlaod url ([#605](https://github.com/cnpm/cnpmcore/issues/605)) ([5bedb25](https://github.com/cnpm/cnpmcore/commit/5bedb25f9dd29d684add0e1f4b827db0c8e0e818))
+
+## [3.47.2](https://github.com/cnpm/cnpmcore/compare/v3.47.1...v3.47.2) (2023-10-28)
+
+
+### Bug Fixes
+
+* ignore BodyTimeoutError ([#603](https://github.com/cnpm/cnpmcore/issues/603)) ([cde4f03](https://github.com/cnpm/cnpmcore/commit/cde4f03c30dea074a32dc32f22f16c16c08fbe0d))
+
+## [3.47.1](https://github.com/cnpm/cnpmcore/compare/v3.47.0...v3.47.1) (2023-10-26)
+
+
+### Bug Fixes
+
+* ignore HttpClientRequestTimeoutError on change stream worker ([#601](https://github.com/cnpm/cnpmcore/issues/601)) ([0791769](https://github.com/cnpm/cnpmcore/commit/079176926dd00c14cbf937d19a8e21dba8376a46))
+
+## [3.47.0](https://github.com/cnpm/cnpmcore/compare/v3.46.0...v3.47.0) (2023-10-26)
+
+
+### Features
+
+* ignore network error to error log ([#600](https://github.com/cnpm/cnpmcore/issues/600)) ([22d401e](https://github.com/cnpm/cnpmcore/commit/22d401ee1f103a9702448d4749f0028a676eddc0))
+
+## [3.46.0](https://github.com/cnpm/cnpmcore/compare/v3.45.1...v3.46.0) (2023-10-11)
+
+
+### Features
+
+* read remote auth token from database ([#595](https://github.com/cnpm/cnpmcore/issues/595)) ([707a1d3](https://github.com/cnpm/cnpmcore/commit/707a1d3809f14cc9a7d613a16f8bea4e5baa7127))
+
+## [3.45.1](https://github.com/cnpm/cnpmcore/compare/v3.45.0...v3.45.1) (2023-10-07)
+
+
+### Bug Fixes
+
+* use oss-cnpm@5.0.1 ([#597](https://github.com/cnpm/cnpmcore/issues/597)) ([413ec56](https://github.com/cnpm/cnpmcore/commit/413ec5685ee54dd90fcfcd5cb59a9b732ec73d84))
+
+## [3.44.0](https://github.com/cnpm/cnpmcore/compare/v3.43.5...v3.44.0) (2023-10-07)
+
+
+### Features
+
+* sync all crhome for test binaries ([#592](https://github.com/cnpm/cnpmcore/issues/592)) ([4596b21](https://github.com/cnpm/cnpmcore/commit/4596b2127119f7c3c31f5fbe786504a7972d62a9))
+* use oss-client v2 ([#596](https://github.com/cnpm/cnpmcore/issues/596)) ([d24e3bd](https://github.com/cnpm/cnpmcore/commit/d24e3bd235fb73b1c145ff3b06dcc168d65b0f9f))
+
+## [3.44.0](https://github.com/cnpm/cnpmcore/compare/v3.43.5...v3.44.0) (2023-09-19)
+
+
+### Features
+
+* sync all crhome for test binaries ([#592](https://github.com/cnpm/cnpmcore/issues/592)) ([4596b21](https://github.com/cnpm/cnpmcore/commit/4596b2127119f7c3c31f5fbe786504a7972d62a9))
+
+## [3.43.5](https://github.com/cnpm/cnpmcore/compare/v3.43.4...v3.43.5) (2023-09-05)
+
+
+### Bug Fixes
+
+* the license may be an object ([#587](https://github.com/cnpm/cnpmcore/issues/587)) ([88b6afb](https://github.com/cnpm/cnpmcore/commit/88b6afb66e77c825d24189f904c448a9cbb86fab)), closes [/github.com/cnpm/cnpmcore/issues/585#issuecomment-1706009496](https://github.com/cnpm//github.com/cnpm/cnpmcore/issues/585/issues/issuecomment-1706009496)
+
+## [3.43.4](https://github.com/cnpm/cnpmcore/compare/v3.43.3...v3.43.4) (2023-09-02)
+
+
+### Bug Fixes
+
+* add CDN cache header on search api ([#583](https://github.com/cnpm/cnpmcore/issues/583)) ([89f6b98](https://github.com/cnpm/cnpmcore/commit/89f6b989c18e714ddd8a1c81fc96778ac53214d7))
+
+## [3.43.3](https://github.com/cnpm/cnpmcore/compare/v3.43.2...v3.43.3) (2023-09-01)
+
+
+### Bug Fixes
+
+* author display in cli ([#582](https://github.com/cnpm/cnpmcore/issues/582)) ([9b2dc41](https://github.com/cnpm/cnpmcore/commit/9b2dc4113485402d5475410e7e8fcd393a562054))
+
+## [3.43.2](https://github.com/cnpm/cnpmcore/compare/v3.43.1...v3.43.2) (2023-09-01)
+
+
+### Bug Fixes
+
+* author info ([#581](https://github.com/cnpm/cnpmcore/issues/581)) ([6dd241d](https://github.com/cnpm/cnpmcore/commit/6dd241d6900bfcebab17aa7814f3c750994a337e))
+
+## [3.43.1](https://github.com/cnpm/cnpmcore/compare/v3.43.0...v3.43.1) (2023-09-01)
+
+
+### Bug Fixes
+
+* use env.CNPMCORE_CONFIG_ENABLE_ES to enable ([#580](https://github.com/cnpm/cnpmcore/issues/580)) ([bcf67c4](https://github.com/cnpm/cnpmcore/commit/bcf67c4cea675793c4804d1892c4a12a2e25c0b8))
+
+## [3.43.0](https://github.com/cnpm/cnpmcore/compare/v3.42.2...v3.43.0) (2023-09-01)
+
+
+### Features
+
+* support npm search command like npmio ([#513](https://github.com/cnpm/cnpmcore/issues/513)) ([7f85848](https://github.com/cnpm/cnpmcore/commit/7f858482f7c26457a37d4e99fb84bd4b9f0ca5da))
+
+## [3.42.2](https://github.com/cnpm/cnpmcore/compare/v3.42.1...v3.42.2) (2023-08-31)
+
+
+### Bug Fixes
+
+* should use NodeNext on module and moduleResolution tsconfig ([#578](https://github.com/cnpm/cnpmcore/issues/578)) ([10d7a84](https://github.com/cnpm/cnpmcore/commit/10d7a8499e5f53663f087e132642018c77783948))
+
+## [3.42.1](https://github.com/cnpm/cnpmcore/compare/v3.42.0...v3.42.1) (2023-08-23)
+
+
+### Bug Fixes
+
+* default latest tag ([#575](https://github.com/cnpm/cnpmcore/issues/575)) ([ffe8fa7](https://github.com/cnpm/cnpmcore/commit/ffe8fa7d190550d9ca340fe62b31397ff018f5d2)), closes [#574](https://github.com/cnpm/cnpmcore/issues/574)
+
+## [3.42.0](https://github.com/cnpm/cnpmcore/compare/v3.41.0...v3.42.0) (2023-08-22)
+
+
+### Features
+
+* revalidate latest version ([#573](https://github.com/cnpm/cnpmcore/issues/573)) ([73b4383](https://github.com/cnpm/cnpmcore/commit/73b4383f5c0805c65abed9c5ac50758402321a53)), closes [#376](https://github.com/cnpm/cnpmcore/issues/376)
+
+## [3.41.0](https://github.com/cnpm/cnpmcore/compare/v3.40.0...v3.41.0) (2023-08-17)
+
+
+### Features
+
+* improved readability of emoji in sync logs ([#572](https://github.com/cnpm/cnpmcore/issues/572)) ([0ac275a](https://github.com/cnpm/cnpmcore/commit/0ac275a348080822e26c66ede81f56c0d3ac94d2))
+
+## [3.40.0](https://github.com/cnpm/cnpmcore/compare/v3.39.5...v3.40.0) (2023-08-08)
+
+
+### Features
+
+* signup on auth ([#567](https://github.com/cnpm/cnpmcore/issues/567)) ([c710600](https://github.com/cnpm/cnpmcore/commit/c7106008d9e5da1760919010899f7a0e24acd051))
+
+## [3.39.5](https://github.com/cnpm/cnpmcore/compare/v3.39.4...v3.39.5) (2023-08-08)
+
+
+### Bug Fixes
+
+* noImplicitAny ts ([#568](https://github.com/cnpm/cnpmcore/issues/568)) ([1932bb9](https://github.com/cnpm/cnpmcore/commit/1932bb9713187bcd4d7e0b0dde410cb0118ab607))
+
+## [3.39.4](https://github.com/cnpm/cnpmcore/compare/v3.39.3...v3.39.4) (2023-08-04)
+
+
+### Bug Fixes
+
+* publisher info ([#565](https://github.com/cnpm/cnpmcore/issues/565)) ([94bcc1a](https://github.com/cnpm/cnpmcore/commit/94bcc1a37ec621a292937b21699ee007c2994974))
+
+## [3.39.3](https://github.com/cnpm/cnpmcore/compare/v3.39.2...v3.39.3) (2023-08-01)
+
+
+### Bug Fixes
+
+* don't log NotImplementedError to error log ([#563](https://github.com/cnpm/cnpmcore/issues/563)) ([bcf3547](https://github.com/cnpm/cnpmcore/commit/bcf3547ff2ee5830fc8cd61bd21b5629d73de316))
+
+## [3.39.2](https://github.com/cnpm/cnpmcore/compare/v3.39.1...v3.39.2) (2023-07-29)
+
+
+### Bug Fixes
+
+* rename libpg-query to libpg-query-node ([#561](https://github.com/cnpm/cnpmcore/issues/561)) ([9483a46](https://github.com/cnpm/cnpmcore/commit/9483a460a395e34c68cb9273ec2e52add4ed1962))
+
+## [3.39.1](https://github.com/cnpm/cnpmcore/compare/v3.39.0...v3.39.1) (2023-07-27)
+
+
+### Bug Fixes
+
+* body parser ignore ([#558](https://github.com/cnpm/cnpmcore/issues/558)) ([81d6455](https://github.com/cnpm/cnpmcore/commit/81d6455ff811b53618c622df74b1f04cf99af3e4))
+
+## [3.39.0](https://github.com/cnpm/cnpmcore/compare/v3.38.2...v3.39.0) (2023-07-27)
+
+
+### Features
+
+* Sync libpg-query binary ([#557](https://github.com/cnpm/cnpmcore/issues/557)) ([8556b5f](https://github.com/cnpm/cnpmcore/commit/8556b5f92f3a6525dd6dde2661a24de9137122d4))
+
+## [3.38.2](https://github.com/cnpm/cnpmcore/compare/v3.38.1...v3.38.2) (2023-07-21)
+
+
+### Bug Fixes
+
+* _npmUser info in fullManifest ([#554](https://github.com/cnpm/cnpmcore/issues/554)) ([4455295](https://github.com/cnpm/cnpmcore/commit/44552959eb8052ac534cc60c1c2820962deed5b8)), closes [#553](https://github.com/cnpm/cnpmcore/issues/553)
+
+## [3.38.1](https://github.com/cnpm/cnpmcore/compare/v3.38.0...v3.38.1) (2023-07-21)
+
+
+### Bug Fixes
+
+* publish lock ([#555](https://github.com/cnpm/cnpmcore/issues/555)) ([ec90ab8](https://github.com/cnpm/cnpmcore/commit/ec90ab85fa529a9666f7f3f03e54d063ddeebc0b))
+
+## [3.38.0](https://github.com/cnpm/cnpmcore/compare/v3.37.1...v3.38.0) (2023-07-17)
+
+
+### Features
+
+* misc router ([#552](https://github.com/cnpm/cnpmcore/issues/552)) ([e9e3a7b](https://github.com/cnpm/cnpmcore/commit/e9e3a7b70f78a13dd7ded2795c62f4d6fcbbe431)), closes [#551](https://github.com/cnpm/cnpmcore/issues/551)
+
+## [3.37.1](https://github.com/cnpm/cnpmcore/compare/v3.37.0...v3.37.1) (2023-07-13)
+
+
+### Bug Fixes
+
+* playwright bianry sync config ([#548](https://github.com/cnpm/cnpmcore/issues/548)) ([166e334](https://github.com/cnpm/cnpmcore/commit/166e3341f424fe514fd7c8c62d0530ac51ef6c12))
+
+## [3.37.0](https://github.com/cnpm/cnpmcore/compare/v3.36.0...v3.37.0) (2023-07-12)
+
+
+### Features
+
+* add lastUsedAt for classic token ([#547](https://github.com/cnpm/cnpmcore/issues/547)) ([e061685](https://github.com/cnpm/cnpmcore/commit/e0616859ffd64f1f273b1dfff711f0d1796b9ec4))
+
+## [3.36.0](https://github.com/cnpm/cnpmcore/compare/v3.35.1...v3.36.0) (2023-07-09)
+
+
+### Features
+
+* support strictValidateTarballPkg ([#546](https://github.com/cnpm/cnpmcore/issues/546)) ([dd3438f](https://github.com/cnpm/cnpmcore/commit/dd3438f470a87ee4f7be058345bbaaa914ea8b2e)), closes [#542](https://github.com/cnpm/cnpmcore/issues/542)
+
+## [3.35.1](https://github.com/cnpm/cnpmcore/compare/v3.35.0...v3.35.1) (2023-06-29)
+
+
+### Bug Fixes
+
+* update source registry ([#537](https://github.com/cnpm/cnpmcore/issues/537)) ([ab2fde7](https://github.com/cnpm/cnpmcore/commit/ab2fde7c809720b1e9692ce2aaf383a9db64e957))
+
+## [3.35.0](https://github.com/cnpm/cnpmcore/compare/v3.34.10...v3.35.0) (2023-06-28)
+
+
+### Features
+
+* adaptive username ([#536](https://github.com/cnpm/cnpmcore/issues/536)) ([dd69606](https://github.com/cnpm/cnpmcore/commit/dd696063652467f7ad1cc94d202f001cdb637906)), closes [/github.com/npm/cli/blob/latest/lib/commands/owner.js#L151](https://github.com/cnpm//github.com/npm/cli/blob/latest/lib/commands/owner.js/issues/L151)
+
+## [3.34.10](https://github.com/cnpm/cnpmcore/compare/v3.34.9...v3.34.10) (2023-06-28)
+
+
+### Bug Fixes
+
+* only syncUpstream in default registry ([#535](https://github.com/cnpm/cnpmcore/issues/535)) ([bb5d993](https://github.com/cnpm/cnpmcore/commit/bb5d9930301426e300ba47d360ab4c92543ab05a))
+
+## [3.34.9](https://github.com/cnpm/cnpmcore/compare/v3.34.8...v3.34.9) (2023-06-27)
+
+
+### Bug Fixes
+
+* console ([#534](https://github.com/cnpm/cnpmcore/issues/534)) ([4141003](https://github.com/cnpm/cnpmcore/commit/4141003e136c82a3a4e68db406b03b3242a1f1fd))
+
+## [3.34.8](https://github.com/cnpm/cnpmcore/compare/v3.34.7...v3.34.8) (2023-06-27)
+
+
+### Bug Fixes
+
+* legacy pkg publish ([#533](https://github.com/cnpm/cnpmcore/issues/533)) ([20ffba8](https://github.com/cnpm/cnpmcore/commit/20ffba8d4115923ddbc9a5407db2110a3c684806))
+
+## [3.34.7](https://github.com/cnpm/cnpmcore/compare/v3.34.6...v3.34.7) (2023-06-25)
+
+
+### Bug Fixes
+
+* sync self pkg ([#532](https://github.com/cnpm/cnpmcore/issues/532)) ([ada3e22](https://github.com/cnpm/cnpmcore/commit/ada3e220a11b6ee0b09b4630ccaf31e5918138c3))
+
+## [3.34.6](https://github.com/cnpm/cnpmcore/compare/v3.34.5...v3.34.6) (2023-06-25)
+
+
+### Bug Fixes
+
+* only auto install s3-cnpmcore on Node.js 18+ ([#531](https://github.com/cnpm/cnpmcore/issues/531)) ([c9d9ce8](https://github.com/cnpm/cnpmcore/commit/c9d9ce8205b115a68a2b7b7184ec58b97721a152))
+
+## [3.34.5](https://github.com/cnpm/cnpmcore/compare/v3.34.4...v3.34.5) (2023-06-21)
+
+
+### Bug Fixes
+
+* create sync task by 'GET /:fullname/-/:filenameWithVersion.tgz' ([#526](https://github.com/cnpm/cnpmcore/issues/526)) ([5ceaa6b](https://github.com/cnpm/cnpmcore/commit/5ceaa6b8dd43aee907e00ac979b55f02a08dba62))
+
+## [3.34.4](https://github.com/cnpm/cnpmcore/compare/v3.34.3...v3.34.4) (2023-06-20)
+
+
+### Bug Fixes
+
+* avoid db query on sync mode all ([#527](https://github.com/cnpm/cnpmcore/issues/527)) ([49855d9](https://github.com/cnpm/cnpmcore/commit/49855d97e5dd08a747f6e999b7eef03362399634)), closes [/github.com/cnpm/cnpmcore/pull/522/files#r1234655574](https://github.com/cnpm//github.com/cnpm/cnpmcore/pull/522/files/issues/r1234655574)
+
+## [3.34.3](https://github.com/cnpm/cnpmcore/compare/v3.34.2...v3.34.3) (2023-06-17)
+
+
+### Bug Fixes
+
+* merge docker ENV into one layer ([#523](https://github.com/cnpm/cnpmcore/issues/523)) ([eb91b83](https://github.com/cnpm/cnpmcore/commit/eb91b834c0edb65248073573840fc8c0511a0ce0))
+
+## [3.34.2](https://github.com/cnpm/cnpmcore/compare/v3.34.1...v3.34.2) (2023-06-17)
+
+
+### Bug Fixes
+
+* should redirect when nfs adapter support url ([#522](https://github.com/cnpm/cnpmcore/issues/522)) ([3d6864c](https://github.com/cnpm/cnpmcore/commit/3d6864c713c3c59905cdf0a4287e0ca2ad291398))
+
+## [3.34.1](https://github.com/cnpm/cnpmcore/compare/v3.34.0...v3.34.1) (2023-06-14)
+
+
+### Bug Fixes
+
+* add block package by packageId and name function ([#514](https://github.com/cnpm/cnpmcore/issues/514)) ([b81b2a0](https://github.com/cnpm/cnpmcore/commit/b81b2a03f85010bb8e94c334e2338ecc8064e833))
+
+## [3.34.0](https://github.com/cnpm/cnpmcore/compare/v3.33.0...v3.34.0) (2023-06-13)
+
+
+### Features
+
+* sync package readme ([#512](https://github.com/cnpm/cnpmcore/issues/512)) ([f64e273](https://github.com/cnpm/cnpmcore/commit/f64e27356691417c2323f9e0951e4f110bae3c6b))
+* use unpkg README.md to update package version readme property ([#511](https://github.com/cnpm/cnpmcore/issues/511)) ([56d8e1a](https://github.com/cnpm/cnpmcore/commit/56d8e1ad877c456042f891b76ff4fe2a771522c4))
+
+## [3.33.0](https://github.com/cnpm/cnpmcore/compare/v3.32.0...v3.33.0) (2023-06-12)
+
+
+### Features
+
+* support Dockerfile and S3 nfs ([#509](https://github.com/cnpm/cnpmcore/issues/509)) ([f61ef1c](https://github.com/cnpm/cnpmcore/commit/f61ef1c0586f668c7c67ca30356e82026afc234d))
+
+## [3.32.0](https://github.com/cnpm/cnpmcore/compare/v3.31.0...v3.32.0) (2023-06-11)
+
+
+### Features
+
+* sync specified versions ([#487](https://github.com/cnpm/cnpmcore/issues/487)) ([a9bb81a](https://github.com/cnpm/cnpmcore/commit/a9bb81adfb0d4eb2ff8b755995272dda22511e58))
+
+## [3.31.0](https://github.com/cnpm/cnpmcore/compare/v3.30.2...v3.31.0) (2023-06-11)
+
+
+### Features
+
+* Support environment variable for config :rocket:  ([#489](https://github.com/cnpm/cnpmcore/issues/489)) ([d4d7a3d](https://github.com/cnpm/cnpmcore/commit/d4d7a3d7c8bb089cd6e00050a9849749a089ae64))
+
+## [3.30.2](https://github.com/cnpm/cnpmcore/compare/v3.30.1...v3.30.2) (2023-06-09)
+
+
+### Bug Fixes
+
+* sync DOWNLOAD_PATHS code update ([#506](https://github.com/cnpm/cnpmcore/issues/506)) ([bf2bf64](https://github.com/cnpm/cnpmcore/commit/bf2bf64532521b7fa78985cf99e7d48efe5261f0))
+
+## [3.30.1](https://github.com/cnpm/cnpmcore/compare/v3.30.0...v3.30.1) (2023-06-07)
+
+
+### Bug Fixes
+
+* type import ([#502](https://github.com/cnpm/cnpmcore/issues/502)) ([27ee3d6](https://github.com/cnpm/cnpmcore/commit/27ee3d61a3a2a555c44e832ca1fdcdd6556cb63b))
+
+## [3.30.0](https://github.com/cnpm/cnpmcore/compare/v3.29.4...v3.30.0) (2023-06-07)
+
+
+### Features
+
+* bulk update maintainers ([#501](https://github.com/cnpm/cnpmcore/issues/501)) ([1b6dce6](https://github.com/cnpm/cnpmcore/commit/1b6dce65acb870c6bb0540b90c5f3c7a5af13a5e))
+
+## [3.29.4](https://github.com/cnpm/cnpmcore/compare/v3.29.3...v3.29.4) (2023-06-07)
+
+
+### Bug Fixes
+
+* validate pkg@version spec ([#500](https://github.com/cnpm/cnpmcore/issues/500)) ([a9d2ff7](https://github.com/cnpm/cnpmcore/commit/a9d2ff7e542b2250dc0e9c9431bb0b8bc97f504d))
+
+## [3.29.3](https://github.com/cnpm/cnpmcore/compare/v3.29.2...v3.29.3) (2023-06-07)
+
+
+### Bug Fixes
+
+* auto fix invalid version to any version ([#499](https://github.com/cnpm/cnpmcore/issues/499)) ([0cf6470](https://github.com/cnpm/cnpmcore/commit/0cf6470993bba77e978be08f234f841efc1c47e9))
+
+## [3.29.2](https://github.com/cnpm/cnpmcore/compare/v3.29.1...v3.29.2) (2023-06-07)
+
+
+### Bug Fixes
+
+* ignore fix invalid version ([#498](https://github.com/cnpm/cnpmcore/issues/498)) ([f5607c0](https://github.com/cnpm/cnpmcore/commit/f5607c0a8723f5ef500cf88399f04b16b5491179))
+
+## [3.29.1](https://github.com/cnpm/cnpmcore/compare/v3.29.0...v3.29.1) (2023-06-07)
+
+
+### Bug Fixes
+
+* fix encoded semver spec ([#497](https://github.com/cnpm/cnpmcore/issues/497)) ([aa39ced](https://github.com/cnpm/cnpmcore/commit/aa39cedf35734528af820e19dab2554ff0c6a125))
+
+## [3.29.0](https://github.com/cnpm/cnpmcore/compare/v3.28.0...v3.29.0) (2023-06-06)
+
+
+### Features
+
+* infer userPrefix when update maintainers  ([#496](https://github.com/cnpm/cnpmcore/issues/496)) ([e02ea2a](https://github.com/cnpm/cnpmcore/commit/e02ea2a3e54b393ffa46cc10e5fc6e29f452e5eb))
+
+## [3.28.0](https://github.com/cnpm/cnpmcore/compare/v3.27.0...v3.28.0) (2023-06-06)
+
+
+### Features
+
+* impl fast semver search ([#495](https://github.com/cnpm/cnpmcore/issues/495)) ([a7fd3a8](https://github.com/cnpm/cnpmcore/commit/a7fd3a8c8ac8692667a3010e3e66f9c139ead0d9))
+
+## [3.27.0](https://github.com/cnpm/cnpmcore/compare/v3.26.0...v3.27.0) (2023-06-04)
+
+
+### Features
+
+* source registry manifest ([#493](https://github.com/cnpm/cnpmcore/issues/493)) ([bbec9a3](https://github.com/cnpm/cnpmcore/commit/bbec9a38bd2a8291913d1e21df1a49c7eb77783a))
+
+## [3.26.0](https://github.com/cnpm/cnpmcore/compare/v3.25.1...v3.26.0) (2023-06-04)
+
+
+### Features
+
+* whoami return granular token info ([#494](https://github.com/cnpm/cnpmcore/issues/494)) ([d0d2f78](https://github.com/cnpm/cnpmcore/commit/d0d2f78d7bf53bcf5b4b75409598f0f7f1625f1a))
+
+## [3.25.1](https://github.com/cnpm/cnpmcore/compare/v3.25.0...v3.25.1) (2023-06-03)
+
+
+### Bug Fixes
+
+* hook enable ([#492](https://github.com/cnpm/cnpmcore/issues/492)) ([40e2f92](https://github.com/cnpm/cnpmcore/commit/40e2f92b944583d609c7eece2391ec5b563866f7))
+
+## [3.25.0](https://github.com/cnpm/cnpmcore/compare/v3.24.0...v3.25.0) (2023-06-02)
+
+
+### Features
+
+* add _npmUser ([#491](https://github.com/cnpm/cnpmcore/issues/491)) ([f7b5d5a](https://github.com/cnpm/cnpmcore/commit/f7b5d5af12b5fcdeaab041061479cc2890101bd2))
+
+## [3.24.0](https://github.com/cnpm/cnpmcore/compare/v3.23.2...v3.24.0) (2023-06-01)
+
+
+### Features
+
+* token last used at ([#488](https://github.com/cnpm/cnpmcore/issues/488)) ([3a8a91a](https://github.com/cnpm/cnpmcore/commit/3a8a91ae0b7370e2f38534e75db75132dcb08151))
+
 ## [3.23.2](https://github.com/cnpm/cnpmcore/compare/v3.23.1...v3.23.2) (2023-05-31)
 
 

Dockerfile

@@ -0,0 +1,22 @@
+FROM node:18
+
+# Create app directory
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY . .
+
+# NPM Mirror
+# npm install -g npminstall --registry=https://registry.npmmirror.com
+# apk add --no-cache socat \  
+RUN apt-get update \
+  && apt-get -y install socat \
+  && npm install -g npminstall \ 
+  && npminstall -c \
+  && npm run tsc
+
+ENV NODE_ENV=production \
+  EGG_SERVER_ENV=prod
+
+EXPOSE 7001
+CMD ["npm", "run", "start:foreground"]

INTEGRATE.md

@@ -1,4 +1,5 @@
 # 🥚 如何在 [tegg](https://github.com/eggjs/tegg) 中集成 cnpmcore
+
 > 文档中的示例项目可以在 [这里](https://github.com/eggjs/examples/commit/bed580fe053ae573f8b63f6788002ff9c6e7a142) 查看，在开始前请确保已阅读 [DEVELOPER.md](DEVELOPER.md) 中的相关文档，完成本地开发环境搭建。
 
 在生产环境中，我们也可以直接部署 cnpmcore 系统，实现完整的 Registry 镜像功能。
@@ -12,7 +13,8 @@
 ## 🚀 快速开始
 
 ### 🆕 新建一个 tegg 应用
-> 我们以 https://github.com/eggjs/examples/tree/master/hello-tegg 为例
+
+> 我们以 <https://github.com/eggjs/examples/tree/master/hello-tegg> 为例
 
 ```shell
 .
@@ -37,6 +39,7 @@
   ```
 
 1. 修改 `ts-config.json` 配置，这是因为 cnpmcore 使用了 [subPath](https://nodejs.org/api/packages.html#subpath-exports)
+
     ```json
     {
       "extends": "@eggjs/tsconfig",
@@ -50,6 +53,7 @@
     ```
 
 2. 修改 `config/plugin.ts` 文件，开启 cnpmcore 依赖的一些插件
+
     ```typescript
     // 开启如下插件
     {
@@ -77,6 +81,7 @@
     ```
 
 3. 修改 `config.default.ts` 文件，可以直接覆盖默认配置
+
 ```typescript
 import { SyncMode } from 'cnpmcore/common/constants';
 import { cnpmcoreConfig } from 'cnpmcore/common/config';
@@ -104,7 +109,7 @@ export default () => {
     │   └── package.json
     ```
 
-  * 添加 `package.json` ，声明 infra 作为一个 eggModule 单元
+* 添加 `package.json` ，声明 infra 作为一个 eggModule 单元
 
     ```JSON
     {
@@ -115,7 +120,7 @@ export default () => {
     }
     ```
 
-  * 添加 `XXXAdapter.ts` 在对应的 Adapter 中继承 cnpmcore 默认的 Adapter，以 AuthAdapter 为例
+* 添加 `XXXAdapter.ts` 在对应的 Adapter 中继承 cnpmcore 默认的 Adapter，以 AuthAdapter 为例
 
     ```typescript
       import { AccessLevel, SingletonProto } from '@eggjs/tegg';
@@ -159,12 +164,14 @@ export default () => {
 我们以 AuthAdapter 为例，来实现 npm cli 的 SSO 登录的功能。
 
 我们需要实现了 getAuthUrl 和 ensureCurrentUser 这两个方法:
+
   1. getAuthUrl 引导用户访问企业内实际的登录中心。
   2. ensureCurrentUser 当用户完成访问后，需要回调到应用进行鉴权流程。
 我们约定通过 `POST /-/v1/login/sso/:sessionId` 这个路由来进行登录验证。
 当然，你也可以任意修改地址和登录回调，只需保证更新 redis 中的 token 状态即可。
 
 修改 AuthAdapter.ts 文件
+
 ```typescript
 import { AccessLevel, EggContext, SingletonProto } from '@eggjs/tegg';
 import { AuthAdapter } from 'cnpmcore/infra/AuthAdapter';
@@ -199,6 +206,7 @@ export class MyAuthAdapter extends AuthAdapter {
 ```
 
 修改 HelloController 的实现，实际也可以通过登录中心回调、页面确认等方式实现
+
 ```typescript
   // 触发回调接口，会自动完成用户创建
   await this.httpclient.request(`${ctx.origin}/-/v1/login/sso/${name}`, { method: 'POST' });
@@ -209,22 +217,24 @@ export class MyAuthAdapter extends AuthAdapter {
 1. 在命令行输入 `npm login --registry=http://127.0.0.1:7001`
 
     ```shell
-    $ npm login --registry=http://127.0.0.1:7001
-    $ npm notice Log in on http://127.0.0.1:7001/
-    $ Login at:
-    $ http://127.0.0.1:7001/hello?name=e44e8c43-211a-4bcd-ae78-c4cbb1a78ae7
-    $ Press ENTER to open in the browser...
+    npm login --registry=http://127.0.0.1:7001
+    npm notice Log in on http://127.0.0.1:7001/
+    Login at:
+    http://127.0.0.1:7001/hello?name=e44e8c43-211a-4bcd-ae78-c4cbb1a78ae7
+    Press ENTER to open in the browser...
     ```
 
 2. 界面提示回车打开浏览器访问登录中心，也就是我们在 getAuthUrl，返回的 loginUrl 配置
 
 3. 由于我们 mock 了对应实现，界面会直接显示登录成功
+
     ```shell
     Logged in on http://127.0.0.1:7001/.
     ```
 
 4. 在命令行输入 `npm whoami --registry=http://127.0.0.1:7001` 验证
+
     ```shell
-    $ npm whoami --registry=http://127.0.0.1:7001
-    $ hello
+    npm whoami --registry=http://127.0.0.1:7001
+    hello
     ```

app/common/FileUtil.ts

@@ -64,7 +64,7 @@ async function _downloadToTempfile(httpclient: EggContextHttpClient,
   try {
     // max 10 mins to download
     // FIXME: should show download progress
-    const authorization = optionalConfig?.remoteAuthToken ? `Bearer ${optionalConfig?.remoteAuthToken}` : '';
+    const authorization = optionalConfig?.remoteAuthToken ? `Bearer ${optionalConfig.remoteAuthToken}` : '';
     const { status, headers, res } = await httpclient.request(url, {
       timeout: 60000 * 10,
       headers: { authorization },
@@ -105,13 +105,13 @@ const WHITE_FILENAME_CONTENT_TYPES = {
   '.eslintignore': PLAIN_TEXT,
   '.jshintrc': 'application/json',
   '.eslintrc': 'application/json',
-};
+} as const;
 
 export function mimeLookup(filepath: string) {
   const filename = path.basename(filepath).toLowerCase();
   if (filename.endsWith('.ts')) return PLAIN_TEXT;
   if (filename.endsWith('.lock')) return PLAIN_TEXT;
   return mime.lookup(filename) ||
-    WHITE_FILENAME_CONTENT_TYPES[filename] ||
+    WHITE_FILENAME_CONTENT_TYPES[filename as keyof typeof WHITE_FILENAME_CONTENT_TYPES] ||
     DEFAULT_CONTENT_TYPE;
 }

app/common/PackageUtil.ts

@@ -3,6 +3,8 @@ import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import * as ssri from 'ssri';
 import tar from 'tar';
+import { AuthorType, PackageJSONType } from '../repository/PackageRepository';
+
 
 // /@cnpm%2ffoo
 // /@cnpm%2Ffoo
@@ -27,6 +29,10 @@ export function cleanUserPrefix(username: string): string {
   return username.replace(/^.*:/, '');
 }
 
+export function getPrefixedName(prefix: string, username: string): string {
+  return prefix ? `${prefix}${username}` : username;
+}
+
 export async function calculateIntegrity(contentOrFile: Uint8Array | string) {
   let integrityObj;
   if (typeof contentOrFile === 'string') {
@@ -98,3 +104,37 @@ export async function hasShrinkWrapInTgz(contentOrFile: Uint8Array | string): Pr
     throw Object.assign(new Error('[hasShrinkWrapInTgz] Fail to parse input file'), { cause: e });
   }
 }
+
+/** 写入 ES 时，格式化 author */
+export function formatAuthor(author: string | AuthorType | undefined): AuthorType | undefined {
+  if (author === undefined) {
+    return author;
+  }
+
+  if (typeof author === 'string') {
+    return { name: author };
+  }
+
+  return author;
+}
+
+export async function extractPackageJSON(tarballBytes: Buffer): Promise<PackageJSONType> {
+  return new Promise((resolve, reject) => {
+    Readable.from(tarballBytes)
+      .pipe(tar.t({
+        filter: name => name === 'package/package.json',
+        onentry: async entry => {
+          const chunks: Buffer[] = [];
+          for await (const chunk of entry) {
+            chunks.push(chunk);
+          }
+          try {
+            const data = Buffer.concat(chunks);
+            return resolve(JSON.parse(data.toString()));
+          } catch (err) {
+            reject(new Error('Error parsing package.json'));
+          }
+        },
+      }));
+  });
+}

app/common/UserUtil.ts

@@ -34,7 +34,7 @@ export function integrity(plain: string): string {
 }
 
 export function checkIntegrity(plain: string, expectedIntegrity: string): boolean {
-  return ssri.checkData(plain, expectedIntegrity);
+  return !!ssri.checkData(plain, expectedIntegrity);
 }
 
 export function sha512(plain: string): string {

app/common/adapter/CacheAdapter.ts

@@ -64,12 +64,13 @@ export class CacheAdapter {
 
   async usingLock(key: string, seconds: number, func: () => Promise<void>) {
     const lockTimestamp = await this.lock(key, seconds);
-    if (!lockTimestamp) return;
+    if (!lockTimestamp) return false;
     try {
       await func();
     } finally {
       await this.unlock(key, lockTimestamp);
     }
+    return true;
   }
 
   private getLockName(key: string) {

app/common/adapter/NPMRegistry.ts

@@ -11,6 +11,7 @@ import {
   HttpClientRequestOptions,
   HttpClientResponse,
 } from 'egg';
+import { PackageManifestType } from '../../repository/PackageRepository';
 
 type HttpMethod = HttpClientRequestOptions['method'];
 
@@ -40,7 +41,7 @@ export class NPMRegistry {
     this.registryHost = registryHost;
   }
 
-  public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise<RegistryResponse> {
+  public async getFullManifests(fullname: string, optionalConfig?: { retries?: number, remoteAuthToken?: string }): Promise<{ method: HttpMethod } & HttpClientResponse<PackageManifestType>> {
     let retries = optionalConfig?.retries || 3;
     // set query t=timestamp, make sure CDN cache disable
     // cache=0 is sync worker request flag

app/common/adapter/binary/AbstractBinary.ts

@@ -17,6 +17,8 @@ export type FetchResult = {
   nextParams?: any;
 };
 
+const platforms = [ 'darwin', 'linux', 'win32' ] as const;
+
 export const BINARY_ADAPTER_ATTRIBUTE = Symbol('BINARY_ADAPTER_ATTRIBUTE');
 
 export abstract class AbstractBinary {
@@ -74,7 +76,7 @@ export abstract class AbstractBinary {
 
   protected listNodePlatforms() {
     // https://nodejs.org/api/os.html#osplatform
-    return [ 'darwin', 'linux', 'win32' ];
+    return platforms;
   }
 
   protected listNodeArchs(binaryConfig?: BinaryTaskConfig) {
@@ -87,11 +89,11 @@ export abstract class AbstractBinary {
     };
   }
 
-  protected listNodeLibcs() {
+  protected listNodeLibcs(): Record<typeof platforms[number], string[]> {
     // https://github.com/lovell/detect-libc/blob/master/lib/detect-libc.js#L42
     return {
-      linux: [ 'glibc', 'musl' ],
       darwin: [ 'unknown' ],
+      linux: [ 'glibc', 'musl' ],
       win32: [ 'unknown' ],
     };
   }

app/common/adapter/binary/ChromeForTestingBinary.ts

@@ -1,3 +1,4 @@
+import { basename } from 'path';
 import { SingletonProto } from '@eggjs/tegg';
 import { BinaryType } from '../../enum/Binary';
 import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './AbstractBinary';
@@ -5,6 +6,8 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.ChromeForTesting)
 export class ChromeForTestingBinary extends AbstractBinary {
+  static lastTimestamp = '';
+
   private dirItems?: {
     [key: string]: BinaryItem[];
   };
@@ -13,57 +16,102 @@ export class ChromeForTestingBinary extends AbstractBinary {
     this.dirItems = undefined;
   }
 
-  async fetch(dir: string): Promise<FetchResult | undefined> {
-    if (!this.dirItems) {
-      this.dirItems = {};
-      this.dirItems['/'] = [];
-      let chromeVersion = '';
+  async #syncDirItems() {
+    this.dirItems = {};
+    this.dirItems['/'] = [];
+    const jsonApiEndpoint = 'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
+    const { data } = await this.httpclient.request(jsonApiEndpoint, {
+      dataType: 'json',
+      timeout: 30000,
+      followRedirect: true,
+      gzip: true,
+    });
+    if (data.timestamp === ChromeForTestingBinary.lastTimestamp) return;
 
-      // exports.PUPPETEER_REVISIONS = Object.freeze({
-      //   chrome: '113.0.5672.63',
-      //   firefox: 'latest',
-      // });
-      const unpkgURL = 'https://unpkg.com/puppeteer-core@latest/lib/cjs/puppeteer/revisions.js';
-      const text = await this.requestXml(unpkgURL);
-      const m = /chrome:\s+\'([\d\.]+)\'\,/.exec(text);
-      if (m) {
-        chromeVersion = m[1];
-      }
-
-      const platforms = [ 'linux64', 'mac-arm64', 'mac-x64', 'win32', 'win64' ];
-      const date = new Date().toISOString();
+    // "timestamp": "2023-09-16T00:21:21.964Z",
+    // "versions": [
+    //   {
+    //     "version": "113.0.5672.0",
+    //     "revision": "1121455",
+    //     "downloads": {
+    //       "chrome": [
+    //         {
+    //           "platform": "linux64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/linux64/chrome-linux64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-arm64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-arm64/chrome-mac-arm64.zip"
+    //         },
+    //         {
+    //           "platform": "mac-x64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/mac-x64/chrome-mac-x64.zip"
+    //         },
+    //         {
+    //           "platform": "win32",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win32/chrome-win32.zip"
+    //         },
+    //         {
+    //           "platform": "win64",
+    //           "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.0/win64/chrome-win64.zip"
+    //         }
+    //       ]
+    //     }
+    //   },
+    const versions = data.versions as {
+      version: string;
+      revision: string;
+      downloads: {
+        [key: string]: {
+          platform: string;
+          url: string;
+        }[];
+      };
+    }[];
+    for (const item of versions) {
       this.dirItems['/'].push({
-        name: `${chromeVersion}/`,
-        date,
+        name: `${item.version}/`,
+        date: item.revision,
         size: '-',
         isDir: true,
         url: '',
       });
-      this.dirItems[`/${chromeVersion}/`] = [];
-
-      for (const platform of platforms) {
-        this.dirItems[`/${chromeVersion}/`].push({
-          name: `${platform}/`,
-          date,
-          size: '-',
-          isDir: true,
-          url: '',
-        });
-
-        // https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/113.0.5672.63/mac-arm64/chrome-mac-arm64.zip
-        const name = `chrome-${platform}.zip`;
-        this.dirItems[`/${chromeVersion}/${platform}/`] = [
-          {
-            name,
-            date,
+      const versionDir = `/${item.version}/`;
+      if (!this.dirItems[versionDir]) {
+        this.dirItems[versionDir] = [];
+      }
+      for (const category in item.downloads) {
+        const downloads = item.downloads[category];
+        for (const download of downloads) {
+          const platformDir = `${versionDir}${download.platform}/`;
+          if (!this.dirItems[platformDir]) {
+            this.dirItems[platformDir] = [];
+            this.dirItems[versionDir].push({
+              name: `${download.platform}/`,
+              date: item.revision,
+              size: '-',
+              isDir: true,
+              url: '',
+            });
+          }
+          this.dirItems[platformDir].push({
+            name: basename(download.url),
+            date: data.timestamp,
             size: '-',
             isDir: false,
-            url: `https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${chromeVersion}/${platform}/${name}`,
-          },
-        ];
+            url: download.url,
+          });
+        }
       }
     }
+    ChromeForTestingBinary.lastTimestamp = data.timestamp;
+  }
 
-    return { items: this.dirItems[dir], nextParams: null };
+  async fetch(dir: string): Promise<FetchResult | undefined> {
+    // use https://github.com/GoogleChromeLabs/chrome-for-testing#json-api-endpoints
+    if (!this.dirItems) {
+      await this.#syncDirItems();
+    }
+    return { items: this.dirItems![dir], nextParams: null };
   }
 }

app/common/adapter/binary/NodePreGypBinary.ts

@@ -15,7 +15,8 @@ export class NodePreGypBinary extends AbstractBinary {
   // https://github.com/mapbox/node-pre-gyp
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     const binaryConfig = binaries[binaryName];
-    const pkgUrl = `https://registry.npmjs.com/${binaryName}`;
+    const npmPackageName = binaryConfig.options?.npmPackageName ?? binaryName;
+    const pkgUrl = `https://registry.npmjs.com/${npmPackageName}`;
     const data = await this.requestJSON(pkgUrl);
     const dirItems: {
       [key: string]: BinaryItem[];

app/common/adapter/binary/PlaywrightBinary.ts

@@ -21,6 +21,10 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
     'ubuntu20.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'debian11': 'builds/chromium/%s/chromium-linux.zip',
+    'debian11-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
+    'debian12': 'builds/chromium/%s/chromium-linux.zip',
+    'debian12-arm64': 'builds/chromium/%s/chromium-linux-arm64.zip',
     'mac10.13': 'builds/chromium/%s/chromium-mac.zip',
     'mac10.14': 'builds/chromium/%s/chromium-mac.zip',
     'mac10.15': 'builds/chromium/%s/chromium-mac.zip',
@@ -28,6 +32,8 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
     'mac12': 'builds/chromium/%s/chromium-mac.zip',
     'mac12-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
+    'mac13': 'builds/chromium/%s/chromium-mac.zip',
+    'mac13-arm64': 'builds/chromium/%s/chromium-mac-arm64.zip',
     'win64': 'builds/chromium/%s/chromium-win64.zip',
   },
   'chromium-tip-of-tree': {
@@ -40,6 +46,10 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
     'ubuntu20.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'debian11': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'debian11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
+    'debian12': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux.zip',
+    'debian12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-linux-arm64.zip',
     'mac10.13': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac10.14': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac10.15': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
@@ -47,6 +57,8 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
     'mac12': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
     'mac12-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
+    'mac13': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac.zip',
+    'mac13-arm64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-mac-arm64.zip',
     'win64': 'builds/chromium-tip-of-tree/%s/chromium-tip-of-tree-win64.zip',
   },
   'chromium-with-symbols': {
@@ -59,6 +71,10 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
     'ubuntu20.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
+    'debian11': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
+    'debian11-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
+    'debian12': 'builds/chromium/%s/chromium-with-symbols-linux.zip',
+    'debian12-arm64': 'builds/chromium/%s/chromium-with-symbols-linux-arm64.zip',
     'mac10.13': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
     'mac10.14': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
     'mac10.15': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
@@ -66,6 +82,8 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/chromium/%s/chromium-with-symbols-mac-arm64.zip',
     'mac12': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
     'mac12-arm64': 'builds/chromium/%s/chromium-with-symbols-mac-arm64.zip',
+    'mac13': 'builds/chromium/%s/chromium-with-symbols-mac.zip',
+    'mac13-arm64': 'builds/chromium/%s/chromium-with-symbols-mac-arm64.zip',
     'win64': 'builds/chromium/%s/chromium-with-symbols-win64.zip',
   },
   'firefox': {
@@ -78,13 +96,19 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/firefox/%s/firefox-ubuntu-20.04-arm64.zip',
     'ubuntu22.04-arm64': 'builds/firefox/%s/firefox-ubuntu-22.04-arm64.zip',
-    'mac10.13': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac10.14': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac10.15': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac11': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac11-arm64': 'builds/firefox/%s/firefox-mac-11-arm64.zip',
-    'mac12': 'builds/firefox/%s/firefox-mac-11.zip',
-    'mac12-arm64': 'builds/firefox/%s/firefox-mac-11-arm64.zip',
+    'debian11': 'builds/firefox/%s/firefox-debian-11.zip',
+    'debian11-arm64': 'builds/firefox/%s/firefox-debian-11-arm64.zip',
+    'debian12': undefined,
+    'debian12-arm64': undefined,
+    'mac10.13': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac10.14': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac10.15': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac11': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac11-arm64': 'builds/firefox/%s/firefox-mac-13-arm64.zip',
+    'mac12': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac12-arm64': 'builds/firefox/%s/firefox-mac-13-arm64.zip',
+    'mac13': 'builds/firefox/%s/firefox-mac-13.zip',
+    'mac13-arm64': 'builds/firefox/%s/firefox-mac-13-arm64.zip',
     'win64': 'builds/firefox/%s/firefox-win64.zip',
   },
   'firefox-beta': {
@@ -97,32 +121,44 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': undefined,
     'ubuntu22.04-arm64': 'builds/firefox-beta/%s/firefox-beta-ubuntu-22.04-arm64.zip',
-    'mac10.13': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac10.14': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac10.15': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac11': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac11-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-11-arm64.zip',
-    'mac12': 'builds/firefox-beta/%s/firefox-beta-mac-11.zip',
-    'mac12-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-11-arm64.zip',
+    'debian11': 'builds/firefox-beta/%s/firefox-beta-debian-11.zip',
+    'debian11-arm64': 'builds/firefox-beta/%s/firefox-beta-debian-11-arm64.zip',
+    'debian12': undefined,
+    'debian12-arm64': undefined,
+    'mac10.13': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac10.14': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac10.15': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac11': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac11-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-13-arm64.zip',
+    'mac12': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac12-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-13-arm64.zip',
+    'mac13': 'builds/firefox-beta/%s/firefox-beta-mac-13.zip',
+    'mac13-arm64': 'builds/firefox-beta/%s/firefox-beta-mac-13-arm64.zip',
     'win64': 'builds/firefox-beta/%s/firefox-beta-win64.zip',
   },
   'webkit': {
     '<unknown>': undefined,
     'generic-linux': 'builds/webkit/%s/webkit-ubuntu-20.04.zip',
     'generic-linux-arm64': 'builds/webkit/%s/webkit-ubuntu-20.04-arm64.zip',
-    'ubuntu18.04': 'builds/webkit/%s/webkit-ubuntu-18.04.zip',
+    'ubuntu18.04': 'builds/deprecated-webkit-ubuntu-18.04/%s/deprecated-webkit-ubuntu-18.04.zip',
     'ubuntu20.04': 'builds/webkit/%s/webkit-ubuntu-20.04.zip',
     'ubuntu22.04': 'builds/webkit/%s/webkit-ubuntu-22.04.zip',
     'ubuntu18.04-arm64': undefined,
     'ubuntu20.04-arm64': 'builds/webkit/%s/webkit-ubuntu-20.04-arm64.zip',
     'ubuntu22.04-arm64': 'builds/webkit/%s/webkit-ubuntu-22.04-arm64.zip',
+    'debian11': 'builds/webkit/%s/webkit-debian-11.zip',
+    'debian11-arm64': 'builds/webkit/%s/webkit-debian-11-arm64.zip',
+    'debian12': undefined,
+    'debian12-arm64': undefined,
     'mac10.13': undefined,
     'mac10.14': 'builds/deprecated-webkit-mac-10.14/%s/deprecated-webkit-mac-10.14.zip',
-    'mac10.15': 'builds/webkit/%s/webkit-mac-10.15.zip',
+    'mac10.15': 'builds/deprecated-webkit-mac-10.15/%s/deprecated-webkit-mac-10.15.zip',
     'mac11': 'builds/webkit/%s/webkit-mac-11.zip',
     'mac11-arm64': 'builds/webkit/%s/webkit-mac-11-arm64.zip',
     'mac12': 'builds/webkit/%s/webkit-mac-12.zip',
     'mac12-arm64': 'builds/webkit/%s/webkit-mac-12-arm64.zip',
+    'mac13': 'builds/webkit/%s/webkit-mac-13.zip',
+    'mac13-arm64': 'builds/webkit/%s/webkit-mac-13-arm64.zip',
     'win64': 'builds/webkit/%s/webkit-win64.zip',
   },
   'ffmpeg': {
@@ -135,6 +171,10 @@ const DOWNLOAD_PATHS = {
     'ubuntu18.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
     'ubuntu20.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
     'ubuntu22.04-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'debian11': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'debian11-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
+    'debian12': 'builds/ffmpeg/%s/ffmpeg-linux.zip',
+    'debian12-arm64': 'builds/ffmpeg/%s/ffmpeg-linux-arm64.zip',
     'mac10.13': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac10.14': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac10.15': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
@@ -142,9 +182,14 @@ const DOWNLOAD_PATHS = {
     'mac11-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
     'mac12': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
     'mac12-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
+    'mac13': 'builds/ffmpeg/%s/ffmpeg-mac.zip',
+    'mac13-arm64': 'builds/ffmpeg/%s/ffmpeg-mac-arm64.zip',
     'win64': 'builds/ffmpeg/%s/ffmpeg-win64.zip',
   },
-};
+  'android': {
+    '<unknown>': 'builds/android/%s/android.zip',
+  },
+} as const;
 
 @SingletonProto()
 @BinaryAdapter(BinaryType.Playwright)
@@ -170,7 +215,7 @@ export class PlaywrightBinary extends AbstractBinary {
         .filter(version => version.match(/^(?:\d+\.\d+\.\d+)(?:-beta-\d+)?$/))
         // select recently update 20 items
         .slice(-20);
-      const browsers: { name: string; revision: string; browserVersion: string; revisionOverrides?: Record<string, string> }[] = [];
+      const browsers: { name: keyof typeof DOWNLOAD_PATHS; revision: string; browserVersion: string; revisionOverrides?: Record<string, string> }[] = [];
       await Promise.all(
         packageVersions.map(version =>
           this.requestJSON(

app/common/adapter/changesStream/CnpmjsorgChangesStream.ts

@@ -6,6 +6,16 @@ import { AbstractChangeStream, RegistryChangesStream } from './AbstractChangesSt
 
 const MAX_LIMIT = 10000;
 
+type FetchResults = {
+  results: {
+    seq: number;
+    type: string;
+    id: string;
+    changes: Record<string, string>[];
+    gmt_modified: Date,
+  }[];
+};
+
 @SingletonProto()
 @RegistryChangesStream(RegistryType.Cnpmjsorg)
 export class CnpmjsorgChangesStream extends AbstractChangeStream {
@@ -18,13 +28,13 @@ export class CnpmjsorgChangesStream extends AbstractChangeStream {
     return since;
   }
 
-  private async tryFetch(registry: Registry, since: string, limit = 1000) {
+  private async tryFetch(registry: Registry, since: string, limit = 1000): Promise<{ data: FetchResults }> {
     if (limit > MAX_LIMIT) {
       throw new E500(`limit too large, current since: ${since}, limit: ${limit}`);
     }
     const db = this.getChangesStreamUrl(registry, since, limit);
     // json mode
-    const res = await this.httpclient.request(db, {
+    const res = await this.httpclient.request<FetchResults>(db, {
       followRedirect: true,
       timeout: 30000,
       dataType: 'json',

app/common/constants.ts

@@ -1,6 +1,7 @@
 export const BUG_VERSIONS = 'bug-versions';
 export const LATEST_TAG = 'latest';
 export const GLOBAL_WORKER = 'GLOBAL_WORKER';
+export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ];
 export enum SyncMode {
   none = 'none',
   admin = 'admin',

app/common/typing.ts

@@ -2,6 +2,7 @@ import { CnpmcoreConfig } from '../port/config';
 import { Readable } from 'stream';
 import { IncomingHttpHeaders } from 'http';
 import { EggContext } from '@eggjs/tegg';
+import { estypes } from '@elastic/elasticsearch';
 
 export interface UploadResult {
   key: string;
@@ -50,6 +51,12 @@ export interface QueueAdapter {
   length(key: string): Promise<number>;
 }
 
+export interface SearchAdapter {
+  search<T>(query: any): Promise<estypes.SearchHitsMetadata<T>>;
+  upsert<T>(id: string, document: T): Promise<string>;
+  delete(id: string): Promise<string>;
+}
+
 export interface AuthUrlResult {
   loginUrl: string;
   doneUrl: string;

app/core/entity/BugVersion.ts

@@ -8,7 +8,7 @@ export type BugVersionPackages = Record<string, BugVersionPackage>;
 export class BugVersion {
   private readonly data: BugVersionPackages;
 
-  constructor(data) {
+  constructor(data: BugVersionPackages) {
     this.data = data;
   }
 

app/core/entity/Hook.ts

@@ -48,7 +48,7 @@ export class Hook extends Entity {
   }
 
   // payload 可能会特别大，如果做多次 stringify 浪费太多 cpu
-  signPayload(payload: object): { digest, payloadStr } {
+  signPayload(payload: object) {
     const payloadStr = JSON.stringify(payload);
     const digest = crypto.createHmac('sha256', this.secret)
       .update(JSON.stringify(payload))

app/core/entity/PackageVersion.ts

@@ -1,6 +1,7 @@
 import { Dist } from './Dist';
 import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
+import { PaddingSemVer } from './PaddingSemVer';
 
 interface PackageVersionData extends EntityData {
   packageId: string;
@@ -11,6 +12,8 @@ interface PackageVersionData extends EntityData {
   tarDist: Dist;
   readmeDist: Dist;
   publishTime: Date;
+  paddingVersion?: string | null;
+  isPreRelease?: boolean | null;
 }
 
 export class PackageVersion extends Entity {
@@ -22,6 +25,8 @@ export class PackageVersion extends Entity {
   tarDist: Dist;
   readmeDist: Dist;
   publishTime: Date;
+  paddingVersion: string;
+  isPreRelease: boolean;
 
   constructor(data: PackageVersionData) {
     super(data);
@@ -33,6 +38,14 @@ export class PackageVersion extends Entity {
     this.tarDist = data.tarDist;
     this.readmeDist = data.readmeDist;
     this.publishTime = data.publishTime;
+    if (data.paddingVersion && typeof data.isPreRelease === 'boolean') {
+      this.paddingVersion = data.paddingVersion;
+      this.isPreRelease = data.isPreRelease;
+    } else {
+      const paddingSemVer = new PaddingSemVer(this.version);
+      this.paddingVersion = paddingSemVer.paddingVersion;
+      this.isPreRelease = paddingSemVer.isPreRelease;
+    }
   }
 
   static create(data: EasyData<PackageVersionData, 'packageVersionId'>): PackageVersion {

app/core/entity/PaddingSemVer.ts

@@ -0,0 +1,50 @@
+import { SemVer, valid } from 'semver';
+
+export class PaddingSemVer {
+  private readonly semver: SemVer;
+  // 跳过 semver 中的 buildInfo, buildInfo 不参与版本比较
+  private _paddingVersion: string;
+  readonly isPreRelease: boolean;
+
+  constructor(semver: string | SemVer) {
+    // ignore invalid version, e.g.: '1000000000000000000.0.0' on https://registry.npmjs.com/latentflip-test
+    if (!valid(semver)) {
+      this.isPreRelease = true;
+      this._paddingVersion = PaddingSemVer.anyVersion();
+      return;
+    }
+    this.semver = new SemVer(semver);
+    if ((this.semver as any).includePrerelease) {
+      this.isPreRelease = true;
+    } else if (this.semver.prerelease && this.semver.prerelease.length) {
+      this.isPreRelease = true;
+    } else {
+      this.isPreRelease = false;
+    }
+  }
+
+  get paddingVersion(): string {
+    if (!this._paddingVersion) {
+      this._paddingVersion = PaddingSemVer.paddingVersion(this.semver.major)
+        + PaddingSemVer.paddingVersion(this.semver.minor)
+        + PaddingSemVer.paddingVersion(this.semver.patch);
+    }
+    return this._paddingVersion;
+  }
+
+  // 版本信息中为纯数字, JS 中支持的最大整型为 16 位
+  // 因此填充成 16 位对齐，如果版本号超过 16 位，则抛出异常
+  static paddingVersion(v: number) {
+    const t = String(v);
+    if (t.length <= 16) {
+      const padding = new Array(16 - t.length).fill(0)
+        .join('');
+      return padding + t;
+    }
+    throw new Error(`v ${v} too long`);
+  }
+
+  static anyVersion() {
+    return '000000000000000000000000000000000000000000000000';
+  }
+}

app/core/entity/Registry.ts

@@ -9,6 +9,7 @@ interface RegistryData extends EntityData {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 }
 
 export type CreateRegistryData = Omit<EasyData<RegistryData, 'registryId'>, 'id'>;
@@ -20,6 +21,7 @@ export class Registry extends Entity {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 
   constructor(data: RegistryData) {
     super(data);
@@ -29,10 +31,11 @@ export class Registry extends Entity {
     this.changeStream = data.changeStream;
     this.userPrefix = data.userPrefix;
     this.type = data.type;
+    this.authToken = data.authToken;
   }
 
   public static create(data: CreateRegistryData): Registry {
-    const newData = EntityUtil.defaultData(data, 'registryId');
+    const newData = EntityUtil.defaultData<RegistryData, 'registryId'>(data, 'registryId');
     return new Registry(newData);
   }
 }

app/core/entity/SqlRange.ts

@@ -0,0 +1,81 @@
+import { Range, Comparator } from 'semver';
+import { PaddingSemVer } from './PaddingSemVer';
+
+const OPERATOR_MAP = {
+  '<': '$lt',
+  '<=': '$lte',
+  '>': '$gt',
+  '>=': '$gte',
+  '': '$eq',
+};
+
+export class SqlRange {
+  private readonly range: Range;
+  private _containPreRelease: boolean;
+  readonly condition: object;
+
+  constructor(range: string | Range) {
+    this.range = new Range(range);
+    this._containPreRelease = false;
+    this.condition = this.generateWhere();
+  }
+
+  private comparatorToSql(comparator: Comparator) {
+    if (comparator.semver === (Comparator as any).ANY) {
+      return {
+        $and: [
+          {
+            isPreRelease: {
+              $lte: 0,
+            },
+          },
+          {
+            paddingVersion: {
+              $gte: PaddingSemVer.anyVersion(),
+            },
+          },
+        ],
+      };
+    }
+    const paddingSemver = new PaddingSemVer(comparator.semver);
+    const operator = OPERATOR_MAP[comparator.operator as keyof typeof OPERATOR_MAP];
+    if (!operator) {
+      throw new Error(`unknown operator ${comparator.operator}`);
+    }
+    this._containPreRelease = this._containPreRelease || paddingSemver.isPreRelease;
+    return {
+      $and: [
+        {
+          isPreRelease: {
+            $lte: paddingSemver.isPreRelease ? 1 : 0,
+          },
+        },
+        {
+          paddingVersion: {
+            [operator]: paddingSemver.paddingVersion,
+          },
+        },
+      ],
+    };
+  }
+
+  private comparatorSetToSql(comparatorSet: Array<Comparator>) {
+    const condition: Array<object> = [];
+    for (const comparator of comparatorSet) {
+      condition.push(this.comparatorToSql(comparator));
+    }
+    return { $and: condition };
+  }
+
+  private generateWhere() {
+    const conditions: Array<object> = [];
+    for (const rangeSet of this.range.set) {
+      conditions.push(this.comparatorSetToSql(rangeSet as Comparator[]));
+    }
+    return { $or: conditions };
+  }
+
+  get containPreRelease(): boolean {
+    return this._containPreRelease;
+  }
+}

app/core/entity/Task.ts

@@ -31,13 +31,13 @@ export interface TaskData<T = TaskBaseData> extends EntityData {
 export type SyncPackageTaskOptions = {
   authorId?: string;
   authorIp?: string;
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
   // force sync history version
   forceSyncHistory?: boolean;
   registryId?: string;
+  specificVersions?: Array<string>;
 };
 
 export interface CreateHookTaskData extends TaskBaseData {
@@ -51,11 +51,11 @@ export interface TriggerHookTaskData extends TaskBaseData {
 }
 
 export interface CreateSyncPackageTaskData extends TaskBaseData {
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
   forceSyncHistory?: boolean;
+  specificVersions?: Array<string>;
 }
 
 export interface ChangesStreamTaskData extends TaskBaseData {
@@ -131,12 +131,12 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
       data: {
         // task execute worker
         taskWorker: '',
-        remoteAuthToken: options?.remoteAuthToken,
         tips: options?.tips,
         registryId: options?.registryId ?? '',
         skipDependencies: options?.skipDependencies,
         syncDownloadData: options?.syncDownloadData,
         forceSyncHistory: options?.forceSyncHistory,
+        specificVersions: options?.specificVersions,
       },
     };
     const task = this.create(data);

app/core/entity/Token.ts

@@ -13,7 +13,8 @@ interface BaseTokenData extends EntityData {
   cidrWhitelist?: string[];
   userId: string;
   isReadonly?: boolean;
-  type?: TokenType;
+  type?: TokenType | string;
+  lastUsedAt?: Date;
 }
 
 interface ClassicTokenData extends BaseTokenData{
@@ -30,7 +31,7 @@ interface GranularTokenData extends BaseTokenData {
 
 type TokenData = ClassicTokenData | GranularTokenData;
 
-export function isGranularToken(data: TokenData): data is GranularTokenData {
+export function isGranularToken(data: TokenData | Token): data is GranularTokenData {
   return data.type === TokenType.granular;
 }
 
@@ -48,6 +49,7 @@ export class Token extends Entity {
   readonly allowedScopes?: string[];
   readonly expiredAt?: Date;
   readonly expires?: number;
+  lastUsedAt: Date | null;
   allowedPackages?: string[];
   token?: string;
 
@@ -59,7 +61,8 @@ export class Token extends Entity {
     this.tokenKey = data.tokenKey;
     this.cidrWhitelist = data.cidrWhitelist || [];
     this.isReadonly = data.isReadonly || false;
-    this.type = data.type || TokenType.classic;
+    this.type = (data.type as TokenType) || TokenType.classic;
+    this.lastUsedAt = data.lastUsedAt || null;
 
     if (isGranularToken(data)) {
       this.name = data.name;
@@ -67,6 +70,7 @@ export class Token extends Entity {
       this.allowedScopes = data.allowedScopes;
       this.expiredAt = data.expiredAt;
       this.allowedPackages = data.allowedPackages;
+      this.isAutomation = false;
     } else {
       this.isAutomation = data.isAutomation || false;
     }

app/core/event/BugVersionFixHandler.ts

@@ -2,15 +2,12 @@ import { Event, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import { PACKAGE_VERSION_ADDED } from './index';
 import { BUG_VERSIONS } from '../../common/constants';
-import { PackageManagerService } from '../service/PackageManagerService';
 import { BugVersionService } from '../service/BugVersionService';
 
 @Event(PACKAGE_VERSION_ADDED)
 export class BugVersionFixHandler {
   @Inject()
   private readonly bugVersionService: BugVersionService;
-  @Inject()
-  private readonly packageManagerService: PackageManagerService;
 
   @Inject()
   private readonly logger: EggLogger;
@@ -18,7 +15,7 @@ export class BugVersionFixHandler {
   async handle(fullname: string) {
     if (fullname !== BUG_VERSIONS) return;
     try {
-      const bugVersion = await this.packageManagerService.getBugVersion();
+      const bugVersion = await this.bugVersionService.getBugVersion();
       if (!bugVersion) return;
       await this.bugVersionService.cleanBugVersionPackageCaches(bugVersion);
     } catch (e) {

app/core/event/ChangesStream.ts

@@ -29,7 +29,7 @@ class ChangesStreamEvent {
   protected readonly config: EggAppConfig;
 
   protected get hookEnable() {
-    return this.config.hookEnable;
+    return this.config.cnpmcore.hookEnable;
   }
 
   protected async addChange(type: string, fullname: string, data: object): Promise<Change> {

app/core/event/SyncESPackage.ts

@@ -0,0 +1,94 @@
+// TODO sync event
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { EggAppConfig } from 'egg';
+import { Event, Inject } from '@eggjs/tegg';
+import {
+  PACKAGE_UNPUBLISHED,
+  PACKAGE_VERSION_ADDED,
+  PACKAGE_VERSION_REMOVED,
+  PACKAGE_TAG_ADDED,
+  PACKAGE_TAG_CHANGED,
+  PACKAGE_TAG_REMOVED,
+  PACKAGE_MAINTAINER_CHANGED,
+  PACKAGE_MAINTAINER_REMOVED,
+  PACKAGE_META_CHANGED,
+} from './index';
+
+import { PackageSearchService } from '../service/PackageSearchService';
+
+class SyncESPackage {
+  @Inject()
+  protected readonly packageSearchService: PackageSearchService;
+
+  @Inject()
+  protected readonly config: EggAppConfig;
+
+  protected async syncPackage(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.syncPackage(fullname, true);
+  }
+}
+
+@Event(PACKAGE_UNPUBLISHED)
+export class PackageUnpublished extends SyncESPackage {
+  async handle(fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) return;
+    await this.packageSearchService.removePackage(fullname);
+  }
+}
+
+@Event(PACKAGE_VERSION_ADDED)
+export class PackageVersionAdded extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_VERSION_REMOVED)
+export class PackageVersionRemoved extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_TAG_ADDED)
+export class PackageTagAdded extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_TAG_CHANGED)
+export class PackageTagChanged extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_TAG_REMOVED)
+export class PackageTagRemoved extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_MAINTAINER_CHANGED)
+export class PackageMaintainerChanged extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_MAINTAINER_REMOVED)
+export class PackageMaintainerRemoved extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}
+
+@Event(PACKAGE_META_CHANGED)
+export class PackageMetaChanged extends SyncESPackage {
+  async handle(fullname: string) {
+    await this.syncPackage(fullname);
+  }
+}

app/core/event/SyncPackageVersionFile.ts

@@ -2,7 +2,7 @@ import { Event, Inject } from '@eggjs/tegg';
 import {
   EggAppConfig,
 } from 'egg';
-import { PACKAGE_VERSION_ADDED } from './index';
+import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { PackageManagerService } from '../service/PackageManagerService';
 import { PackageVersionFileService } from '../service/PackageVersionFileService';
@@ -25,6 +25,14 @@ class SyncPackageVersionFileEvent {
     if (!packageVersion) return;
     await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
   }
+
+  protected async syncPackageReadmeToLatestVersion(fullname: string) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const { pkg, packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
+      scope, name, 'latest');
+    if (!pkg || !packageVersion) return;
+    await this.packageVersionFileService.syncPackageReadme(pkg, packageVersion);
+  }
 }
 
 @Event(PACKAGE_VERSION_ADDED)
@@ -33,3 +41,19 @@ export class PackageVersionAdded extends SyncPackageVersionFileEvent {
     await this.syncPackageVersionFile(fullname, version);
   }
 }
+
+@Event(PACKAGE_TAG_ADDED)
+export class PackageTagAdded extends SyncPackageVersionFileEvent {
+  async handle(fullname: string, tag: string) {
+    if (tag !== 'latest') return;
+    await this.syncPackageReadmeToLatestVersion(fullname);
+  }
+}
+
+@Event(PACKAGE_TAG_CHANGED)
+export class PackageTagChanged extends SyncPackageVersionFileEvent {
+  async handle(fullname: string, tag: string) {
+    if (tag !== 'latest') return;
+    await this.syncPackageReadmeToLatestVersion(fullname);
+  }
+}

app/core/service/BinarySyncerService.ts

@@ -21,6 +21,7 @@ import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/Abstract
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
 import { BinaryType } from '../../common/enum/Binary';
+import { sortBy } from 'lodash';
 
 function isoNow() {
   return new Date().toISOString();
@@ -145,14 +146,22 @@ export class BinarySyncerService extends AbstractService {
       task.error = err.message;
       logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`);
       logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`);
-      this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
-        task.taskId, task.targetName, task.error);
-      this.logger.error(err);
+      if (err.name === 'HttpClientRequestTimeoutError'
+        || err.name === 'ConnectionError'
+        || err.name === 'ConnectTimeoutError') {
+        this.logger.warn('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
+          task.taskId, task.targetName, task.error);
+        this.logger.warn(err);
+      } else {
+        this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s',
+          task.taskId, task.targetName, task.error);
+        this.logger.error(err);
+      }
       await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
     }
   }
 
-  private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '') {
+  private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') {
     const binaryName = task.targetName as BinaryName;
     const result = await binaryAdapter.fetch(dir, binaryName);
     let hasDownloadError = false;
@@ -160,14 +169,15 @@ export class BinarySyncerService extends AbstractService {
     if (result && result.items.length > 0) {
       hasItems = true;
       let logs: string[] = [];
-      const newItems = await this.diff(binaryName, dir, result.items);
+      const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent);
       logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`);
+      // re-check latest version
       for (const [ index, { item, reason }] of newItems.entries()) {
         if (item.isDir) {
           logs.push(`[${isoNow()}][${dir}] 🚧 [${parentIndex}${index}] Start sync dir ${JSON.stringify(item)}, reason: ${reason}`);
           await this.taskService.appendTaskLog(task, logs.join('\n'));
           logs = [];
-          const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`);
+          const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`, latestVersionDir);
           if (hasError) {
             hasDownloadError = true;
           } else {
@@ -208,7 +218,11 @@ export class BinarySyncerService extends AbstractService {
               this.logger.info('Not found %s, skip it', item.sourceUrl);
               logs.push(`[${isoNow()}][${dir}] 🧪️ [${parentIndex}${index}] Download ${item.sourceUrl} not found, skip it`);
             } else {
-              this.logger.error('Download binary %s %s', item.sourceUrl, err);
+              if (err.name === 'DownloadStatusInvalidError') {
+                this.logger.warn('Download binary %s %s', item.sourceUrl, err);
+              } else {
+                this.logger.error('Download binary %s %s', item.sourceUrl, err);
+              }
               hasDownloadError = true;
               logs.push(`[${isoNow()}][${dir}] ❌ [${parentIndex}${index}] Download ${item.sourceUrl} error: ${err}`);
             }
@@ -231,7 +245,12 @@ export class BinarySyncerService extends AbstractService {
     return [ hasDownloadError, hasItems ];
   }
 
-  private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[]) {
+
+  // see https://github.com/cnpm/cnpmcore/issues/556
+  // 上游可能正在发布新版本、同步流程中断，导致同步的时候，文件列表不一致
+  // 如果的当前目录命中 latestVersionParent 父目录，那么就再校验一下当前目录
+  // 如果 existsItems 为空或者经过修改，那么就不需要 revalidate 了
+  private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') {
     const existsItems = await this.binaryRepository.listBinaries(binaryName, dir);
     const existsMap = new Map<string, Binary>();
     for (const item of existsItems) {
@@ -262,9 +281,23 @@ export class BinarySyncerService extends AbstractService {
         existsItem.sourceUrl = item.url;
         existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses;
         existsItem.date = item.date;
+      } else if (dir.endsWith(latestVersionParent)) {
+        const isLatestItem = sortBy(fetchItems, [ 'date' ]).pop()?.name === item.name;
+        if (isLatestItem && existsItem.isDir) {
+          diffItems.push({
+            item: existsItem,
+            reason: `revalidate latest version, latest parent dir is ${latestVersionParent}, current dir is ${dir}, current name is ${existsItem.name}`,
+          });
+          latestVersionParent = `${latestVersionParent}${existsItem.name}`;
+        }
       }
     }
-    return diffItems;
+
+
+    return {
+      newItems: diffItems,
+      latestVersionDir: latestVersionParent,
+    };
   }
 
   private async saveBinaryItem(binary: Binary, tmpfile?: string) {

app/core/service/BugVersionService.ts

@@ -2,10 +2,12 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import pMap from 'p-map';
 import { BugVersion } from '../entity/BugVersion';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageJSONType, PackageRepository } from '../../repository/PackageRepository';
 import { DistRepository } from '../../repository/DistRepository';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { CacheService } from './CacheService';
+import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
+import { BugVersionStore } from '../../common/adapter/BugVersionStore';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -23,6 +25,27 @@ export class BugVersionService {
   @Inject()
   private readonly cacheService: CacheService;
 
+  @Inject()
+  private readonly bugVersionStore: BugVersionStore;
+
+  async getBugVersion(): Promise<BugVersion | undefined> {
+    // TODO performance problem, cache bugVersion and update with schedule
+    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
+    if (!pkg) return;
+    /* c8 ignore next 10 */
+    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
+    if (!tag) return;
+    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
+    if (!bugVersion) {
+      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
+      if (!packageVersionJson) return;
+      const data = packageVersionJson.config?.['bug-versions'];
+      bugVersion = new BugVersion(data || {});
+      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
+    }
+    return bugVersion;
+  }
+
   async cleanBugVersionPackageCaches(bugVersion: BugVersion) {
     const fullnames = bugVersion.listAllPackagesHasBugs();
     await pMap(fullnames, async fullname => {

app/core/service/ChangesStreamService.ts

@@ -101,8 +101,14 @@ export class ChangesStreamService extends AbstractService {
         await setTimeout(this.config.cnpmcore.checkChangesStreamInterval);
       }
     } catch (err) {
-      this.logger.error('[ChangesStreamService.executeTask:error] %s, exit now', err);
-      this.logger.error(err);
+      this.logger.warn('[ChangesStreamService.executeTask:error] %s, exit now', err.message);
+      if (err.name === 'HttpClientRequestTimeoutError'
+        || err.name === 'ConnectTimeoutError'
+        || err.name === 'BodyTimeoutError') {
+        this.logger.warn(err);
+      } else {
+        this.logger.error(err);
+      }
       task.error = `${err}`;
       await this.taskRepository.saveTask(task);
       await this.suspendSync();

app/core/service/FixNoPaddingVersionService.ts

@@ -0,0 +1,33 @@
+import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
+import { EggLogger } from 'egg';
+import pMap from 'p-map';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
+import { PaddingSemVer } from '../entity/PaddingSemVer';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class FixNoPaddingVersionService {
+  @Inject()
+  private readonly packageVersionRepository: PackageVersionRepository;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async fixPaddingVersion(id?: number): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const packageVersions = await this.packageVersionRepository.findHaveNotPaddingVersion(id);
+      if (packageVersions.length === 0) {
+        break;
+      }
+      id = packageVersions[packageVersions.length - 1].id as unknown as number + 1;
+      this.logger.info('[FixNoPaddingVersionService] fix padding version ids %j', packageVersions.map(t => t.id));
+
+      await pMap(packageVersions, async packageVersion => {
+        const paddingSemver = new PaddingSemVer(packageVersion.version);
+        await this.packageVersionRepository.fixPaddingVersion(packageVersion.packageVersionId, paddingSemver);
+      }, { concurrency: 30 });
+    }
+  }
+}

app/core/service/HomeService.ts

@@ -0,0 +1,19 @@
+import {
+  AccessLevel,
+  SingletonProto,
+} from '@eggjs/tegg';
+import { AbstractService } from '../../common/AbstractService';
+import { NOT_IMPLEMENTED_PATH } from '../../common/constants';
+import { NotFoundError, NotImplementedError } from 'egg-errors';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class HomeService extends AbstractService {
+  async misc(path: string) {
+    if (NOT_IMPLEMENTED_PATH.includes(path)) {
+      throw new NotImplementedError(`${path} not implemented yet`);
+    }
+    throw new NotFoundError(`${path} not found`);
+  }
+}

app/core/service/PackageManagerService.ts

@@ -1,12 +1,13 @@
-import { stat } from 'fs/promises';
+import { stat, readFile } from 'node:fs/promises';
 import {
   AccessLevel,
   SingletonProto,
   EventBus,
   Inject,
 } from '@eggjs/tegg';
-import { ForbiddenError } from 'egg-errors';
+import { ForbiddenError, NotFoundError } from 'egg-errors';
 import { RequireAtLeastOne } from 'type-fest';
+import npa from 'npm-package-arg';
 import semver from 'semver';
 import {
   calculateIntegrity,
@@ -17,8 +18,6 @@ import {
   hasShrinkWrapInTgz,
 } from '../../common/PackageUtil';
 import { AbstractService } from '../../common/AbstractService';
-import { BugVersionStore } from '../../common/adapter/BugVersionStore';
-import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
 import { AbbreviatedPackageJSONType, AbbreviatedPackageManifestType, PackageJSONType, PackageManifestType, PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
@@ -46,6 +45,7 @@ import { BugVersionService } from './BugVersionService';
 import { BugVersion } from '../entity/BugVersion';
 import { RegistryManagerService } from './RegistryManagerService';
 import { Registry } from '../entity/Registry';
+import { PackageVersionService } from './PackageVersionService';
 
 export interface PublishPackageCmd {
   // maintainer: Maintainer;
@@ -53,7 +53,7 @@ export interface PublishPackageCmd {
   // name don't include scope
   name: string;
   version: string;
-  description: string;
+  description?: string;
   packageJson: PackageJSONType;
   registryId?: string;
   readme: string;
@@ -64,7 +64,7 @@ export interface PublishPackageCmd {
     // sync worker will use localFile field
     localFile?: string;
   }, 'content' | 'localFile'>;
-  tag?: string;
+  tags?: string[];
   isPrivate: boolean;
   // only use on sync package
   publishTime?: Date;
@@ -91,11 +91,11 @@ export class PackageManagerService extends AbstractService {
   @Inject()
   private readonly bugVersionService: BugVersionService;
   @Inject()
-  private readonly bugVersionStore: BugVersionStore;
-  @Inject()
   private readonly distRepository: DistRepository;
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly packageVersionService: PackageVersionService;
 
   private static downloadCounters = {};
 
@@ -107,14 +107,14 @@ export class PackageManagerService extends AbstractService {
         scope: cmd.scope,
         name: cmd.name,
         isPrivate: cmd.isPrivate,
-        description: cmd.description,
+        description: cmd.description || '',
         registryId: cmd.registryId,
       });
     } else {
       // update description
       // will read database twice to update description by model to entity and entity to model
       if (pkg.description !== cmd.description) {
-        pkg.description = cmd.description;
+        pkg.description = cmd.description || '';
       }
 
       /* c8 ignore next 3 */
@@ -155,17 +155,17 @@ export class PackageManagerService extends AbstractService {
       cmd.packageJson._hasShrinkwrap = await hasShrinkWrapInTgz(cmd.dist.content || cmd.dist.localFile!);
     }
 
+    // set _npmUser field to cmd.packageJson
+    cmd.packageJson._npmUser = {
+      // clean user scope prefix
+      name: publisher.displayName,
+      email: publisher.email,
+    };
+
     // add _registry_name field to cmd.packageJson
-    if (!cmd.packageJson._source_registry_name) {
-      let registry: Registry | null;
-      if (cmd.registryId) {
-        registry = await this.registryManagerService.findByRegistryId(cmd.registryId);
-      } else {
-        registry = await this.registryManagerService.ensureDefaultRegistry();
-      }
-      if (registry) {
-        cmd.packageJson._source_registry_name = registry.name;
-      }
+    const registry = await this.getSourceRegistry(pkg);
+    if (registry) {
+      cmd.packageJson._source_registry_name = registry.name;
     }
 
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
@@ -270,13 +270,27 @@ export class PackageManagerService extends AbstractService {
     if (cmd.skipRefreshPackageManifests !== true) {
       await this.refreshPackageChangeVersionsToDists(pkg, [ pkgVersion.version ]);
     }
-    if (cmd.tag) {
-      await this.savePackageTag(pkg, cmd.tag, cmd.version, true);
+    if (cmd.tags) {
+      for (const tag of cmd.tags) {
+        await this.savePackageTag(pkg, tag, cmd.version, true);
+        this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, tag);
+      }
+    } else {
+      this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, undefined);
     }
-    this.eventBus.emit(PACKAGE_VERSION_ADDED, pkg.fullname, pkgVersion.version, cmd.tag);
+
     return pkgVersion;
   }
 
+  async blockPackageByFullname(name: string, reason: string) {
+    const [ scope, pkgName ] = getScopeAndName(name);
+    const pkg = await this.packageRepository.findPackage(scope, pkgName);
+    if (!pkg) {
+      throw new NotFoundError(`Package name(${name}) not found`);
+    }
+    return await this.blockPackage(pkg, reason);
+  }
+
   async blockPackage(pkg: Package, reason: string) {
     let block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
     if (block) {
@@ -306,6 +320,15 @@ export class PackageManagerService extends AbstractService {
     return block;
   }
 
+  async unblockPackageByFullname(name: string) {
+    const [ scope, pkgName ] = getScopeAndName(name);
+    const pkg = await this.packageRepository.findPackage(scope, pkgName);
+    if (!pkg) {
+      throw new NotFoundError(`Package name(${name}) not found`);
+    }
+    return await this.unblockPackage(pkg);
+  }
+
   async unblockPackage(pkg: Package) {
     const block = await this.packageVersionBlockRepository.findPackageVersionBlock(pkg.packageId, '*');
     if (block) {
@@ -327,9 +350,9 @@ export class PackageManagerService extends AbstractService {
     }
   }
 
-  async replacePackageMaintainers(pkg: Package, maintainers: User[]) {
+  async replacePackageMaintainersAndDist(pkg: Package, maintainers: User[]) {
     await this.packageRepository.replacePackageMaintainers(pkg.packageId, maintainers.map(m => m.userId));
-    await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
+    await this.refreshPackageMaintainersToDists(pkg);
     this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
   }
 
@@ -342,14 +365,12 @@ export class PackageManagerService extends AbstractService {
       }
     }
     if (hasNewRecord) {
-      await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
       this.eventBus.emit(PACKAGE_MAINTAINER_CHANGED, pkg.fullname, maintainers);
     }
   }
 
   async removePackageMaintainer(pkg: Package, maintainer: User) {
     await this.packageRepository.removePackageMaintainer(pkg.packageId, maintainer.userId);
-    await this._refreshPackageManifestRootAttributeOnlyToDists(pkg, 'maintainers');
     this.eventBus.emit(PACKAGE_MAINTAINER_REMOVED, pkg.fullname, maintainer.name);
   }
 
@@ -369,7 +390,7 @@ export class PackageManagerService extends AbstractService {
     return await this._listPackageFullOrAbbreviatedManifests(scope, name, false, isSync);
   }
 
-  async showPackageVersionByVersionOrTag(scope: string, name: string, versionOrTag: string): Promise<{
+  async showPackageVersionByVersionOrTag(scope: string, name: string, spec: string): Promise<{
     blockReason?: string,
     pkg?: Package,
     packageVersion?: PackageVersion | null,
@@ -380,40 +401,27 @@ export class PackageManagerService extends AbstractService {
     if (block) {
       return { blockReason: block.reason, pkg };
     }
-    let version = versionOrTag;
-    if (!semver.valid(versionOrTag)) {
-      // invalid version, versionOrTag is a tag
-      const packageTag = await this.packageRepository.findPackageTag(pkg.packageId, versionOrTag);
-      if (packageTag) {
-        version = packageTag.version;
-      }
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const version = await this.packageVersionService.getVersion(result);
+    if (!version) {
+      return {};
     }
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
     return { packageVersion, pkg };
   }
 
-  async showPackageVersionManifest(scope: string, name: string, versionOrTag: string, isSync = false) {
-    let manifest;
-    const { blockReason, packageVersion, pkg } = await this.showPackageVersionByVersionOrTag(scope, name, versionOrTag);
-    if (blockReason) {
-      return {
-        blockReason,
-        manifest,
-        pkg,
-      };
+  async showPackageVersionManifest(scope: string, name: string, spec: string, isSync = false, isFullManifests = false) {
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) return {};
+    const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
+    if (block) {
+      return { blockReason: block.reason, pkg };
     }
-    if (!packageVersion) return { manifest: null, blockReason, pkg };
-    manifest = await this.distRepository.findPackageVersionManifest(packageVersion.packageId, packageVersion.version);
-    let bugVersion: BugVersion | undefined;
-    // sync mode response no bug version fixed
-    if (!isSync) {
-      bugVersion = await this.getBugVersion();
-    }
-    if (bugVersion) {
-      const fullname = getFullname(scope, name);
-      manifest = await this.bugVersionService.fixPackageBugVersion(bugVersion, fullname, manifest);
-    }
-    return { manifest, blockReason, pkg };
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const manifest = await this.packageVersionService.readManifest(pkg.packageId, result, isFullManifests, !isSync);
+    return { manifest, blockReason: null, pkg };
   }
 
   async downloadPackageVersionTar(packageVersion: PackageVersion) {
@@ -422,7 +430,7 @@ export class PackageManagerService extends AbstractService {
 
   public plusPackageVersionCounter(fullname: string, version: string) {
     // set counter + 1, schedule will store them into database
-    const counters = PackageManagerService.downloadCounters;
+    const counters: Record<string, Record<string, number>> = PackageManagerService.downloadCounters;
     if (!counters[fullname]) counters[fullname] = {};
     counters[fullname][version] = (counters[fullname][version] || 0) + 1;
     // Total
@@ -441,7 +449,7 @@ export class PackageManagerService extends AbstractService {
   // will be call by schedule/SavePackageVersionDownloadCounter.ts
   async savePackageVersionCounters() {
     // { [fullname]: { [version]: number } }
-    const counters = PackageManagerService.downloadCounters;
+    const counters: Record<string, Record<string, number>> = PackageManagerService.downloadCounters;
     const fullnames = Object.keys(counters);
     if (fullnames.length === 0) return;
 
@@ -492,6 +500,25 @@ export class PackageManagerService extends AbstractService {
     await this._mergeManifestDist(pkgVersion.abbreviatedDist, mergeAbbreviated);
   }
 
+  /**
+   * save package version readme
+   */
+  public async savePackageVersionReadme(pkgVersion: PackageVersion, readmeFile: string) {
+    await this.distRepository.saveDist(pkgVersion.readmeDist, readmeFile);
+    this.logger.info('[PackageManagerService.savePackageVersionReadme] save packageVersionId:%s readme:%s to dist:%s',
+      pkgVersion.packageVersionId, readmeFile, pkgVersion.readmeDist.distId);
+  }
+
+  public async savePackageReadme(pkg: Package, readmeFile: string) {
+    if (!pkg.manifestsDist) return;
+    const fullManifests = await this.distRepository.readDistBytesToJSON<PackageManifestType>(pkg.manifestsDist);
+    if (!fullManifests) return;
+    fullManifests.readme = await readFile(readmeFile, 'utf-8');
+    await this._updatePackageManifestsToDists(pkg, fullManifests, null);
+    this.logger.info('[PackageManagerService.savePackageReadme] save packageId:%s readme, size: %s',
+      pkg.packageId, fullManifests.readme.length);
+  }
+
   private async _removePackageVersionAndDist(pkgVersion: PackageVersion) {
     // remove nfs dists
     await Promise.all([
@@ -644,22 +671,14 @@ export class PackageManagerService extends AbstractService {
     await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
   }
 
-  async getBugVersion(): Promise<BugVersion | undefined> {
-    // TODO performance problem, cache bugVersion and update with schedule
-    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
-    if (!pkg) return;
-    /* c8 ignore next 10 */
-    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
-    if (!tag) return;
-    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
-    if (!bugVersion) {
-      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
-      if (!packageVersionJson) return;
-      const data = packageVersionJson.config?.['bug-versions'];
-      bugVersion = new BugVersion(data);
-      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
+  async getSourceRegistry(pkg: Package): Promise<Registry | null> {
+    let registry: Registry | null;
+    if (pkg.registryId) {
+      registry = await this.registryManagerService.findByRegistryId(pkg.registryId);
+    } else {
+      registry = await this.registryManagerService.ensureDefaultRegistry();
     }
-    return bugVersion;
+    return registry;
   }
 
   private async _listPackageDistTags(pkg: Package) {
@@ -710,13 +729,16 @@ export class PackageManagerService extends AbstractService {
     const fieldsFromLatestManifest = [
       'author', 'bugs', 'contributors', 'description', 'homepage', 'keywords', 'license',
       'readmeFilename', 'repository',
-    ];
+    ] as const;
     // the latest version metas
     for (const field of fieldsFromLatestManifest) {
-      fullManifests[field] = latestManifest[field];
+      if (latestManifest[field]) {
+        (fullManifests as Record<string, unknown>)[field] = latestManifest[field];
+      }
     }
   }
 
+
   private async _setPackageDistTagsAndLatestInfos(pkg: Package, fullManifests: PackageManifestType, abbreviatedManifests: AbbreviatedPackageManifestType) {
     const distTags = await this._listPackageDistTags(pkg);
     if (distTags.latest) {
@@ -793,6 +815,7 @@ export class PackageManagerService extends AbstractService {
     let blockReason = '';
     const pkg = await this.packageRepository.findPackage(scope, name);
     if (!pkg) return { etag, data: null, blockReason };
+    const registry = await this.getSourceRegistry(pkg);
 
     const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
     if (block) {
@@ -802,7 +825,7 @@ export class PackageManagerService extends AbstractService {
     let bugVersion: BugVersion | undefined;
     // sync mode response no bug version fixed
     if (!isSync) {
-      bugVersion = await this.getBugVersion();
+      bugVersion = await this.bugVersionService.getBugVersion();
     }
     const fullname = getFullname(scope, name);
 
@@ -814,6 +837,11 @@ export class PackageManagerService extends AbstractService {
       if (bugVersion) {
         await this.bugVersionService.fixPackageBugVersions(bugVersion, fullname, data.versions);
       }
+      // set _source_registry_name in full manifestDist
+      if (registry) {
+        data._source_registry_name = registry?.name;
+      }
+
       const distBytes = Buffer.from(JSON.stringify(data));
       const distIntegrity = await calculateIntegrity(distBytes);
       etag = `"${distIntegrity.shasum}"`;
@@ -854,8 +882,9 @@ export class PackageManagerService extends AbstractService {
 
     const distTags = await this._listPackageDistTags(pkg);
     const maintainers = await this._listPackageMaintainers(pkg);
+    const registry = await this.getSourceRegistry(pkg);
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
-    const data:PackageManifestType = {
+    const data: PackageManifestType = {
       _id: `${pkg.fullname}`,
       _rev: `${pkg.id}-${pkg.packageId}`,
       'dist-tags': distTags,
@@ -888,6 +917,7 @@ export class PackageManagerService extends AbstractService {
       // as given in package.json, for the latest version
       repository: undefined,
       // users: an object whose keys are the npm user names of people who have starred this package
+      _source_registry_name: registry?.name,
     };
 
     let latestTagVersion = '';

app/core/service/PackageSearchService.ts

@@ -0,0 +1,236 @@
+import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
+import { estypes, errors } from '@elastic/elasticsearch';
+import dayjs from 'dayjs';
+
+import { AbstractService } from '../../common/AbstractService';
+import { formatAuthor, getScopeAndName } from '../../common/PackageUtil';
+import { PackageManagerService } from './PackageManagerService';
+import { SearchManifestType, SearchMappingType, SearchRepository } from '../../repository/SearchRepository';
+import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
+import { PackageRepository } from '../../repository/PackageRepository';
+
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class PackageSearchService extends AbstractService {
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
+  @Inject()
+  private readonly searchRepository: SearchRepository;
+  @Inject()
+  private packageVersionDownloadRepository: PackageVersionDownloadRepository;
+  @Inject()
+  protected packageRepository: PackageRepository;
+
+  async syncPackage(fullname: string, isSync = true) {
+    const [ scope, name ] = getScopeAndName(fullname);
+    const fullManifests = await this.packageManagerService.listPackageFullManifests(scope, name, isSync);
+
+    if (!fullManifests.data) {
+      this.logger.warn('[PackageSearchService.syncPackage] save package:%s not found', fullname);
+      return;
+    }
+
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) {
+      this.logger.warn('[PackageSearchService.syncPackage] findPackage:%s not found', fullname);
+      return;
+    }
+
+    // get last year download data
+    const startDate = dayjs().subtract(1, 'year');
+    const endDate = dayjs();
+
+    const entities = await this.packageVersionDownloadRepository.query(pkg.packageId, startDate.toDate(), endDate.toDate());
+    let downloadsAll = 0;
+    for (const entity of entities) {
+      for (let i = 1; i <= 31; i++) {
+        const day = String(i).padStart(2, '0');
+        const field = `d${day}`;
+        const counter = entity[field];
+        if (!counter) continue;
+        downloadsAll += counter;
+      }
+    }
+
+    const { data: manifest } = fullManifests;
+
+    const latestVersion = manifest['dist-tags'].latest;
+    const latestManifest = manifest.versions[latestVersion];
+
+    const packageDoc: SearchMappingType = {
+      name: manifest.name,
+      version: latestVersion,
+      _rev: manifest._rev,
+      scope: scope ? scope.replace('@', '') : 'unscoped',
+      keywords: manifest.keywords || [],
+      versions: Object.keys(manifest.versions),
+      description: manifest.description,
+      license: typeof manifest.license === 'object' ? manifest.license?.type : manifest.license,
+      maintainers: manifest.maintainers,
+      author: formatAuthor(manifest.author),
+      'dist-tags': manifest['dist-tags'],
+      date: manifest.time[latestVersion],
+      created: manifest.time.created,
+      modified: manifest.time.modified,
+      // 归属 registry，keywords 枚举值
+      _source_registry_name: manifest._source_registry_name,
+      // 最新版本发布人 _npmUser:
+      _npmUser: latestManifest?._npmUser,
+      // 最新版本发布信息
+      publish_time: latestManifest?.publish_time,
+    };
+
+    // http://npmmirror.com/package/npm/files/lib/utils/format-search-stream.js#L147-L148
+    // npm cli 使用 username 字段
+    if (packageDoc.maintainers) {
+      packageDoc.maintainers = packageDoc.maintainers.map(maintainer => {
+        return {
+          username: maintainer.name,
+          ...maintainer,
+        };
+      });
+    }
+
+    const document: SearchManifestType = {
+      package: packageDoc,
+      downloads: {
+        all: downloadsAll,
+      },
+    };
+
+    return await this.searchRepository.upsertPackage(document);
+  }
+
+  async searchPackage(text: string, from: number, size: number): Promise<{ objects: (SearchManifestType | undefined)[], total: number }> {
+    const matchQueries = this._buildMatchQueries(text);
+    const scriptScore = this._buildScriptScore({
+      text,
+      scoreEffect: 0.25,
+    });
+
+    const res = await this.searchRepository.searchPackage({
+      body: {
+        size,
+        from,
+        query: {
+          function_score: {
+            boost_mode: 'replace',
+            query: {
+              bool: {
+                should: matchQueries,
+                minimum_should_match: matchQueries.length ? 1 : 0,
+              },
+            },
+            script_score: scriptScore,
+          },
+        },
+      },
+    });
+    const { hits, total } = res;
+    return {
+      objects: hits?.map(item => {
+        return item._source;
+      }),
+      total: (total as estypes.SearchTotalHits).value,
+    };
+  }
+
+  async removePackage(fullname: string) {
+    try {
+      return await this.searchRepository.removePackage(fullname);
+    } catch (error) {
+      // if the package does not exist, returns success
+      if (error instanceof errors.ResponseError && error?.statusCode === 404) {
+        this.logger.warn('[PackageSearchService.removePackage] remove package:%s not found', fullname);
+        return fullname;
+      }
+      throw error;
+    }
+  }
+
+  // https://github.com/npms-io/queries/blob/master/lib/search.js#L8C1-L78C2
+  private _buildMatchQueries(text: string) {
+    return [
+      // Standard match using cross_fields
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.standard^4',
+            'package.description.standard',
+            'package.keywords.standard^2',
+          ],
+          type: 'cross_fields',
+          boost: 6,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Partial match using edge-ngram
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.edge_ngram^4',
+            'package.description.edge_ngram',
+            'package.keywords.edge_ngram^2',
+          ],
+          type: 'phrase',
+          slop: 3,
+          boost: 3,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Normal term match with an english stemmer
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.english_docs^4',
+            'package.description.english_docs',
+            'package.keywords.english_docs^2',
+          ],
+          type: 'cross_fields',
+          boost: 3,
+          tie_breaker: 0.5,
+        },
+      },
+
+      // Normal term match with a more aggressive english stemmer (not so important)
+      {
+        multi_match: {
+          query: text,
+          operator: 'and',
+          fields: [
+            'package.name.english_aggressive_docs^4',
+            'package.description.english_aggressive_docs',
+            'package.keywords.english_aggressive_docs^2',
+          ],
+          type: 'cross_fields',
+          tie_breaker: 0.5,
+        },
+      },
+    ];
+  }
+
+  private _buildScriptScore(params: { text: string | undefined, scoreEffect: number }) {
+    // keep search simple, only download(popularity)
+    const downloads = 'doc["downloads.all"].value';
+    const source = `doc["package.name.raw"].value.equals(params.text) ? 100000 + ${downloads} : _score * Math.pow(${downloads}, params.scoreEffect)`;
+    return {
+      script: {
+        source,
+        params: {
+          text: params.text || '',
+          scoreEffect: params.scoreEffect,
+        },
+      },
+    };
+  }
+}

app/core/service/PackageSyncerService.ts

@@ -5,19 +5,20 @@ import {
   Inject,
 } from '@eggjs/tegg';
 import { Pointcut } from '@eggjs/tegg/aop';
-import {
-  EggContextHttpClient,
-} from 'egg';
+import { EggHttpClient } from 'egg';
 import { setTimeout } from 'timers/promises';
 import { rm } from 'fs/promises';
+import { isEqual, isEmpty } from 'lodash';
 import semver from 'semver';
+import semverRcompare from 'semver/functions/rcompare';
+import semverPrerelease from 'semver/functions/prerelease';
 import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
 import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
 import { downloadToTempfile } from '../../common/FileUtil';
 import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageJSONType, PackageManifestType, PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { UserRepository } from '../../repository/UserRepository';
 import { Task, SyncPackageTaskOptions, CreateSyncPackageTask } from '../entity/Task';
@@ -32,7 +33,7 @@ import { Registry } from '../entity/Registry';
 import { BadRequestError } from 'egg-errors';
 import { ScopeManagerService } from './ScopeManagerService';
 import { EventCorkAdvice } from './EventCorkerAdvice';
-import { SyncDeleteMode } from '../../common/constants';
+import { PresetRegistryName, SyncDeleteMode } from '../../common/constants';
 
 type syncDeletePkgOptions = {
   task: Task,
@@ -73,7 +74,7 @@ export class PackageSyncerService extends AbstractService {
   @Inject()
   private readonly cacheService: CacheService;
   @Inject()
-  private readonly httpclient: EggContextHttpClient;
+  private readonly httpclient: EggHttpClient;
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
   @Inject()
@@ -115,17 +116,18 @@ export class PackageSyncerService extends AbstractService {
     if (!this.allowSyncDownloadData) {
       return;
     }
+
     const fullname = pkg.fullname;
     const start = '2011-01-01';
     const end = this.config.cnpmcore.syncDownloadDataMaxDate;
     const registry = this.config.cnpmcore.syncDownloadDataSourceRegistry;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     const logs: string[] = [];
     let downloads: { day: string; downloads: number }[];
 
     logs.push(`[${isoNow()}][DownloadData] 🚧🚧🚧🚧🚧 Syncing "${fullname}" download data "${start}:${end}" on ${registry} 🚧🚧🚧🚧🚧`);
     const failEnd = '❌❌❌❌❌ 🚮 give up 🚮 ❌❌❌❌❌';
     try {
-      const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
       const { data, status, res } = await this.npmRegistry.getDownloadRanges(registry, fullname, start, end, { remoteAuthToken });
       downloads = data.downloads || [];
       logs.push(`[${isoNow()}][DownloadData] 🚧 HTTP [${status}] timing: ${JSON.stringify(res.timing)}, downloads: ${downloads.length}`);
@@ -162,7 +164,7 @@ export class PackageSyncerService extends AbstractService {
   private async syncUpstream(task: Task) {
     const registry = this.npmRegistry.registry;
     const fullname = task.targetName;
-    const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     let logs: string[] = [];
     let logId = '';
     logs.push(`[${isoNow()}][UP] 🚧🚧🚧🚧🚧 Waiting sync "${fullname}" task on ${registry} 🚧🚧🚧🚧🚧`);
@@ -202,8 +204,8 @@ export class PackageSyncerService extends AbstractService {
         const log = data && data.log || '';
         offset += log.length;
         if (data && data.syncDone) {
-          logs.push(`[${isoNow()}][UP] 🟢 Sync ${fullname} success [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
-          logs.push(`[${isoNow()}][UP] 🟢🟢🟢🟢🟢 ${registry}/${fullname} 🟢🟢🟢🟢🟢`);
+          logs.push(`[${isoNow()}][UP] 🎉 Sync ${fullname} success [${useTime}ms], log: ${logUrl}, offset: ${offset}`);
+          logs.push(`[${isoNow()}][UP] 🔗 ${registry}/${fullname}`);
           await this.taskService.appendTaskLog(task, logs.join('\n'));
           return;
         }
@@ -297,8 +299,8 @@ export class PackageSyncerService extends AbstractService {
     }
 
     // update log
-    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    logs.push(`[${isoNow()}] 📝 Log URL: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🔗 ${url}`);
     await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
     this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s',
       task.taskId, task.targetName);
@@ -310,7 +312,7 @@ export class PackageSyncerService extends AbstractService {
   // 1. 其次从 task.data.registryId (创建单包同步任务时传入)
   // 2. 接着根据 scope 进行计算 (作为子包依赖同步时候，无 registryId)
   // 3. 最后返回 default registryId (可能 default registry 也不存在)
-  public async initSpecRegistry(task: Task, pkg: Package | null = null, scope?: string): Promise<Registry | null> {
+  public async initSpecRegistry(task: Task, pkg: Package | null = null, scope?: string): Promise<Registry> {
     const registryId = pkg?.registryId || (task.data as SyncPackageTaskOptions).registryId;
     let targetHost: string = this.config.cnpmcore.sourceRegistry;
     let registry: Registry | null = null;
@@ -349,10 +351,11 @@ export class PackageSyncerService extends AbstractService {
   public async executeTask(task: Task) {
     const fullname = task.targetName;
     const [ scope, name ] = getScopeAndName(fullname);
-    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, remoteAuthToken } = task.data as SyncPackageTaskOptions;
+    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, specificVersions } = task.data as SyncPackageTaskOptions;
     let pkg = await this.packageRepository.findPackage(scope, name);
     const registry = await this.initSpecRegistry(task, pkg, scope);
     const registryHost = this.npmRegistry.registry;
+    const remoteAuthToken = registry.authToken;
     let logs: string[] = [];
     if (tips) {
       logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`);
@@ -361,13 +364,24 @@ export class PackageSyncerService extends AbstractService {
     const taskQueueHighWaterSize = this.config.cnpmcore.taskQueueHighWaterSize;
     const taskQueueInHighWaterState = taskQueueLength >= taskQueueHighWaterSize;
     const skipDependencies = taskQueueInHighWaterState ? true : !!originSkipDependencies;
-    const syncUpstream = !!(!taskQueueInHighWaterState && this.config.cnpmcore.sourceRegistryIsCNpm && this.config.cnpmcore.syncUpstreamFirst);
+    const syncUpstream = !!(!taskQueueInHighWaterState && this.config.cnpmcore.sourceRegistryIsCNpm && this.config.cnpmcore.syncUpstreamFirst && registry.name === PresetRegistryName.default);
     const logUrl = `${this.config.cnpmcore.registry}/-/package/${fullname}/syncs/${task.taskId}/log`;
     this.logger.info('[PackageSyncerService.executeTask:start] taskId: %s, targetName: %s, attempts: %s, taskQueue: %s/%s, syncUpstream: %s, log: %s',
       task.taskId, task.targetName, task.attempts, taskQueueLength, taskQueueHighWaterSize, syncUpstream, logUrl);
     logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Syncing from ${registryHost}/${fullname}, skipDependencies: ${skipDependencies}, syncUpstream: ${syncUpstream}, syncDownloadData: ${!!syncDownloadData}, forceSyncHistory: ${!!forceSyncHistory} attempts: ${task.attempts}, worker: "${os.hostname()}/${process.pid}", taskQueue: ${taskQueueLength}/${taskQueueHighWaterSize} 🚧🚧🚧🚧🚧`);
+    if (specificVersions) {
+      logs.push(`[${isoNow()}] 👉 syncing specific versions: ${specificVersions.join(' | ')} 👈`);
+    }
     logs.push(`[${isoNow()}] 🚧 log: ${logUrl}`);
 
+    if (registry?.name === PresetRegistryName.self) {
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} has been published to the self registry, skip sync ❌❌❌❌❌`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[PackageSyncerService.executeTask:fail] taskId: %s, targetName: %s, invalid registryId',
+        task.taskId, task.targetName);
+      return;
+    }
+
     if (pkg && pkg?.registryId !== registry?.registryId) {
       if (pkg.registryId) {
         logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} registry is ${pkg.registryId} not belong to ${registry?.registryId}, skip sync ❌❌❌❌❌`);
@@ -472,7 +486,7 @@ export class PackageSyncerService extends AbstractService {
     //   { name: 'jasonlaster11', email: 'jason.laster.11@gmail.com' }
     // ],
     let maintainers = data.maintainers;
-    const maintainersMap = {};
+    const maintainersMap: Record<string, PackageManifestType['maintainers']> = {};
     const users: User[] = [];
     let changedUserCount = 0;
     if (!Array.isArray(maintainers) || maintainers.length === 0) {
@@ -544,10 +558,22 @@ export class PackageSyncerService extends AbstractService {
     const existsVersionCount = Object.keys(existsVersionMap).length;
     const abbreviatedVersionMap = abbreviatedManifests?.versions ?? {};
     // 2. save versions
-    const versions = Object.values<any>(versionMap);
+    if (specificVersions && !this.config.cnpmcore.strictSyncSpecivicVersion && !specificVersions.includes(distTags.latest)) {
+      logs.push(`[${isoNow()}] 📦 Add latest tag version "${fullname}: ${distTags.latest}"`);
+      specificVersions.push(distTags.latest);
+    }
+    const versions: PackageJSONType[] = specificVersions ? Object.values<any>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) : Object.values<any>(versionMap);
     logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
+    if (specificVersions) {
+      const availableVersionList = versions.map(item => item.version);
+      let notAvailableVersionList = specificVersions.filter(i => !availableVersionList.includes(i));
+      if (notAvailableVersionList.length > 0) {
+        notAvailableVersionList = Array.from(new Set(notAvailableVersionList));
+        logs.push(`[${isoNow()}] 🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
+      }
+    }
     const updateVersions: string[] = [];
-    const differentMetas: any[] = [];
+    const differentMetas: [PackageJSONType, Partial<PackageJSONType>][] = [];
     let syncIndex = 0;
     for (const item of versions) {
       const version: string = item.version;
@@ -581,10 +607,10 @@ export class PackageSyncerService extends AbstractService {
         // check metaDataKeys, if different value, override exists one
         // https://github.com/cnpm/cnpmjs.org/issues/1667
         // need libc field https://github.com/cnpm/cnpmcore/issues/187
-        const metaDataKeys = [
-          'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', 'deprecated',
-        ];
-        let diffMeta: any;
+        // fix _npmUser field since https://github.com/cnpm/cnpmcore/issues/553
+        const metaDataKeys = [ 'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript', 'deprecated', '_npmUser' ];
+        const ignoreInAbbreviated = [ '_npmUser' ];
+        const diffMeta: Partial<PackageJSONType> = {};
         for (const key of metaDataKeys) {
           let remoteItemValue = item[key];
           // make sure hasInstallScript exists
@@ -593,34 +619,30 @@ export class PackageSyncerService extends AbstractService {
               remoteItemValue = true;
             }
           }
-          const remoteItemDiffValue = JSON.stringify(remoteItemValue);
-          if (remoteItemDiffValue !== JSON.stringify(existsItem[key])) {
-            if (!diffMeta) diffMeta = {};
+          if (!isEqual(remoteItemValue, existsItem[key])) {
             diffMeta[key] = remoteItemValue;
-          } else if (existsAbbreviatedItem && remoteItemDiffValue !== JSON.stringify(existsAbbreviatedItem[key])) {
+          } else if (!ignoreInAbbreviated.includes(key) && existsAbbreviatedItem && !isEqual(remoteItemValue, (existsAbbreviatedItem as Record<string, unknown>)[key])) {
             // should diff exists abbreviated item too
-            if (!diffMeta) diffMeta = {};
             diffMeta[key] = remoteItemValue;
           }
         }
         // should delete readme
         if (shouldDeleteReadme) {
-          if (!diffMeta) diffMeta = {};
           diffMeta.readme = undefined;
         }
-        if (diffMeta) {
+        if (!isEmpty(diffMeta)) {
           differentMetas.push([ existsItem, diffMeta ]);
         }
         continue;
       }
       syncIndex++;
-      const description: string = item.description;
+      const description = item.description;
       // "dist": {
       //   "shasum": "943e0ec03df00ebeb6273a5b94b916ba54b47581",
       //   "tarball": "https://registry.npmjs.org/foo/-/foo-1.0.0.tgz"
       // },
       const dist = item.dist;
-      const tarball: string = dist && dist.tarball;
+      const tarball = dist && dist.tarball;
       if (!tarball) {
         lastErrorMessage = `missing tarball, dist: ${JSON.stringify(dist)}`;
         logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
@@ -639,7 +661,11 @@ export class PackageSyncerService extends AbstractService {
         localFile = tmpfile;
         logs.push(`[${isoNow()}] 🚧 [${syncIndex}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)} => ${localFile}`);
       } catch (err: any) {
-        this.logger.error('Download tarball %s error: %s', tarball, err);
+        if (err.name === 'DownloadNotFoundError' || err.name === 'DownloadStatusInvalidError') {
+          this.logger.warn('Download tarball %s error: %s', tarball, err);
+        } else {
+          this.logger.error('Download tarball %s error: %s', tarball, err);
+        }
         lastErrorMessage = `download tarball error: ${err}`;
         logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} fail, ${lastErrorMessage}`);
         await this.taskService.appendTaskLog(task, logs.join('\n'));
@@ -667,9 +693,10 @@ export class PackageSyncerService extends AbstractService {
       };
       try {
         // 当 version 记录已经存在时，还需要校验一下 pkg.manifests 是否存在
-        const pkgVersion = await this.packageManagerService.publish(publishCmd, users[0]);
+        const publisher = users.find(user => user.displayName === item._npmUser?.name) || users[0];
+        const pkgVersion = await this.packageManagerService.publish(publishCmd, publisher);
         updateVersions.push(pkgVersion.version);
-        logs.push(`[${isoNow()}] 🟢 [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
+        logs.push(`[${isoNow()}] 🎉 [${syncIndex}] Synced version ${version} success, packageVersionId: ${pkgVersion.packageVersionId}, db id: ${pkgVersion.id}`);
       } catch (err: any) {
         if (err.name === 'ForbiddenError') {
           logs.push(`[${isoNow()}] 🐛 [${syncIndex}] Synced version ${version} already exists, skip publish, try to set in local manifest`);
@@ -787,6 +814,24 @@ export class PackageSyncerService extends AbstractService {
         }
       }
     }
+    // 3.2 shoud add latest tag
+    // 在同步 sepcific version 时如果没有同步 latestTag 的版本会出现 latestTag 丢失或指向版本不正确的情况
+    if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
+      // 不允许自动同步 latest 版本，从已同步版本中选出 latest
+      let latestStableVersion: string;
+      const sortedVersionList = specificVersions.sort(semverRcompare);
+      latestStableVersion = sortedVersionList.filter(i => !semverPrerelease(i))[0];
+      // 所有版本都不是稳定版本则指向非稳定版本保证 latest 存在
+      if (!latestStableVersion) {
+        latestStableVersion = sortedVersionList[0];
+      }
+      if (!existsDistTags.latest || semverRcompare(existsDistTags.latest, latestStableVersion) === 1) {
+        logs.push(`[${isoNow()}] 🚧 patch latest tag from specific versions 🚧`);
+        changedTags.push({ action: 'change', tag: 'latest', version: latestStableVersion });
+        await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);
+      }
+    }
+
     if (changedTags.length > 0) {
       logs.push(`[${isoNow()}] 🟢 Synced ${changedTags.length} tags: ${JSON.stringify(changedTags)}`);
     }
@@ -814,6 +859,15 @@ export class PackageSyncerService extends AbstractService {
       logs.push(`[${isoNow()}] 🟢 Removed ${removedMaintainers.length} maintainers: ${JSON.stringify(removedMaintainers)}`);
     }
 
+    // 4.2 update package maintainers in dist
+    // The event is initialized in the repository and distributed after uncork.
+    // maintainers' information is updated in bulk to ensure consistency.
+    if (!isEqual(maintainers, existsMaintainers)) {
+      logs.push(`[${isoNow()}] 🚧 Syncing maintainers to package manifest, from: ${JSON.stringify(maintainers)} to: ${JSON.stringify(existsMaintainers)}`);
+      await this.packageManagerService.refreshPackageMaintainersToDists(pkg);
+      logs.push(`[${isoNow()}] 🟢 Syncing maintainers to package manifest done`);
+    }
+
     // 5. add deps sync task
     for (const dependencyName of dependenciesSet) {
       const existsTask = await this.taskRepository.findTaskByTargetName(dependencyName, TaskType.SyncPackage, TaskState.Waiting);
@@ -836,9 +890,9 @@ export class PackageSyncerService extends AbstractService {
 
     // clean cache
     await this.cacheService.removeCache(fullname);
-    logs.push(`[${isoNow()}] 🟢 Clean cache`);
-    logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`);
-    logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`);
+    logs.push(`[${isoNow()}] 🗑️ Clean cache`);
+    logs.push(`[${isoNow()}] 📝 Log URL: ${logUrl}`);
+    logs.push(`[${isoNow()}] 🔗 ${url}`);
     task.error = lastErrorMessage;
     await this.taskService.finishTask(task, TaskState.Success, logs.join('\n'));
     this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s',

app/core/service/PackageVersionFileService.ts

@@ -20,6 +20,7 @@ import { DistRepository } from '../../repository/DistRepository';
 import { PackageVersionFile } from '../entity/PackageVersionFile';
 import { PackageVersion } from '../entity/PackageVersion';
 import { Package } from '../entity/Package';
+import { PackageManagerService } from './PackageManagerService';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -31,6 +32,8 @@ export class PackageVersionFileService extends AbstractService {
   private readonly packageVersionFileRepository: PackageVersionFileRepository;
   @Inject()
   private readonly distRepository: DistRepository;
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
 
   async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) {
     await this.#ensurePackageVersionFilesSync(pkgVersion);
@@ -51,6 +54,51 @@ export class PackageVersionFileService extends AbstractService {
     }
   }
 
+  // 基于 latest version 同步 package readme
+  async syncPackageReadme(pkg: Package, latestPkgVersion: PackageVersion) {
+    const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${latestPkgVersion.version}_latest_readme_${randomUUID()}`;
+    const tmpdir = await createTempDir(this.config.dataDir, dirname);
+    const tarFile = `${tmpdir}.tgz`;
+    const readmeFilenames: string[] = [];
+    try {
+      this.logger.info('[PackageVersionFileService.syncPackageReadme:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
+        latestPkgVersion.tarDist.distId, latestPkgVersion.tarDist.path, latestPkgVersion.tarDist.size, tarFile);
+      await this.distRepository.downloadDistToFile(latestPkgVersion.tarDist, tarFile);
+      this.logger.info('[PackageVersionFileService.syncPackageReadme:extract-start] tmpdir:%s', tmpdir);
+      await tar.extract({
+        file: tarFile,
+        cwd: tmpdir,
+        strip: 1,
+        onentry: entry => {
+          const filename = this.#formatTarEntryFilename(entry);
+          if (!filename) return;
+          if (this.#matchReadmeFilename(filename)) {
+            readmeFilenames.push(filename);
+          }
+        },
+      });
+      if (readmeFilenames.length > 0) {
+        const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
+        const readmeFile = join(tmpdir, readmeFilename);
+        await this.packageManagerService.savePackageReadme(pkg, readmeFile);
+      }
+    } catch (err) {
+      this.logger.warn('[PackageVersionFileService.syncPackageReadme:error] packageVersionId: %s, readmeFilenames: %j, tmpdir: %s, error: %s',
+        latestPkgVersion.packageVersionId, readmeFilenames, tmpdir, err);
+      // ignore TAR_BAD_ARCHIVE error
+      if (err.code === 'TAR_BAD_ARCHIVE') return;
+      throw err;
+    } finally {
+      try {
+        await fs.rm(tarFile, { force: true });
+        await fs.rm(tmpdir, { recursive: true, force: true });
+      } catch (err) {
+        this.logger.warn('[PackageVersionFileService.syncPackageReadme:warn] remove tmpdir: %s, error: %s',
+          tmpdir, err);
+      }
+    }
+  }
+
   async syncPackageVersionFiles(pkgVersion: PackageVersion) {
     const files: PackageVersionFile[] = [];
     const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
@@ -59,6 +107,7 @@ export class PackageVersionFileService extends AbstractService {
     const tmpdir = await createTempDir(this.config.dataDir, dirname);
     const tarFile = `${tmpdir}.tgz`;
     const paths: string[] = [];
+    const readmeFilenames: string[] = [];
     try {
       this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
         pkgVersion.tarDist.distId, pkgVersion.tarDist.path, pkgVersion.tarDist.size, tarFile);
@@ -69,12 +118,12 @@ export class PackageVersionFileService extends AbstractService {
         cwd: tmpdir,
         strip: 1,
         onentry: entry => {
-          if (entry.type !== 'File') return;
-          // ignore hidden dir
-          if (entry.path.includes('/./')) return;
-          // https://github.com/cnpm/cnpmcore/issues/452#issuecomment-1570077310
-          // strip first dir, e.g.: 'package/', 'lodash-es/'
-          paths.push('/' + entry.path.split('/').slice(1).join('/'));
+          const filename = this.#formatTarEntryFilename(entry);
+          if (!filename) return;
+          paths.push('/' + filename);
+          if (this.#matchReadmeFilename(filename)) {
+            readmeFilenames.push(filename);
+          }
         },
       });
       for (const path of paths) {
@@ -84,6 +133,11 @@ export class PackageVersionFileService extends AbstractService {
       }
       this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:success] packageVersionId: %s, %d paths, %d files, tmpdir: %s',
         pkgVersion.packageVersionId, paths.length, files.length, tmpdir);
+      if (readmeFilenames.length > 0) {
+        const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
+        const readmeFile = join(tmpdir, readmeFilename);
+        await this.packageManagerService.savePackageVersionReadme(pkgVersion, readmeFile);
+      }
       return files;
     } catch (err) {
       this.logger.warn('[PackageVersionFileService.syncPackageVersionFiles:error] packageVersionId: %s, %d paths, tmpdir: %s, error: %s',
@@ -144,4 +198,37 @@ export class PackageVersionFileService extends AbstractService {
       name: basename(path),
     };
   }
+
+  #formatTarEntryFilename(entry: tar.ReadEntry) {
+    if (entry.type !== 'File') return;
+    // ignore hidden dir
+    if (entry.path.includes('/./')) return;
+    // https://github.com/cnpm/cnpmcore/issues/452#issuecomment-1570077310
+    // strip first dir, e.g.: 'package/', 'lodash-es/'
+    const filename = entry.path.split('/').slice(1).join('/');
+    return filename;
+  }
+
+  #matchReadmeFilename(filename: string) {
+    // support README,README.*
+    // https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
+    return (/^README(\.\w{1,20}|$)/i.test(filename));
+  }
+
+  // https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
+  #preferMarkdownReadme(files: string[]) {
+    let fallback = 0;
+    const markdownRE = /\.m?a?r?k?d?o?w?n?$/i;
+    for (let i = 0; i < files.length; i++) {
+      const file = files[i];
+      if (markdownRE.test(file)) {
+        return file;
+      } else if (file.toLowerCase() === 'README') {
+        fallback = i;
+      }
+    }
+    // prefer README.md, followed by README; otherwise, return
+    // the first filename (which could be README)
+    return files[fallback];
+  }
 }

app/core/service/PackageVersionService.ts

@@ -0,0 +1,118 @@
+import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
+import semver, { Range } from 'semver';
+import { Result, AliasResult } from 'npm-package-arg';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
+import { getScopeAndName } from '../../common/PackageUtil';
+import { SqlRange } from '../entity/SqlRange';
+import { BugVersionService } from './BugVersionService';
+import type { PackageJSONType } from '../../repository/PackageRepository';
+import { DistRepository } from '../../repository/DistRepository';
+import { BugVersionAdvice } from '../entity/BugVersion';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class PackageVersionService {
+  @Inject()
+  private packageVersionRepository: PackageVersionRepository;
+
+  @Inject()
+  private readonly bugVersionService: BugVersionService;
+
+  @Inject()
+  private readonly distRepository: DistRepository;
+
+  async readManifest(pkgId: string, spec: Result, isFullManifests: boolean, withBugVersion = true): Promise<PackageJSONType | undefined> {
+    const realSpec = this.findRealSpec(spec);
+    let version = await this.getVersion(realSpec, false);
+    if (!version) {
+      return undefined;
+    }
+    let bugVersionAdvice: {
+      advice: BugVersionAdvice,
+      version: string,
+    } | undefined;
+    if (withBugVersion) {
+      const bugVersion = await this.bugVersionService.getBugVersion();
+      if (bugVersion) {
+        const advice = bugVersion.fixVersion(spec.name!, version);
+        if (advice) {
+          bugVersionAdvice = {
+            advice,
+            version,
+          };
+          version = advice.version;
+        }
+      }
+    }
+    let manifest;
+    if (isFullManifests) {
+      manifest = await this.distRepository.findPackageVersionManifest(pkgId, version);
+    } else {
+      manifest = await this.distRepository.findPackageAbbreviatedManifest(pkgId, version);
+    }
+    if (manifest && bugVersionAdvice) {
+      manifest.deprecated = `[WARNING] Use ${bugVersionAdvice.advice.version} instead of ${bugVersionAdvice.version}, reason: ${bugVersionAdvice.advice.reason}`;
+      manifest.version = bugVersionAdvice.version;
+    }
+    return manifest;
+  }
+
+  private findRealSpec(spec: Result) {
+    let realSpec: Result;
+    switch (spec.type) {
+      case 'alias':
+        realSpec = (spec as AliasResult).subSpec;
+        break;
+      case 'version':
+      case 'tag':
+      case 'range':
+        realSpec = spec;
+        break;
+      default:
+        throw new Error(`npmcore not support spec: ${spec.raw}`);
+    }
+    return realSpec;
+  }
+
+  async getVersion(spec: Result, withBugVersion = true): Promise<string | undefined | null> {
+    let version: string | undefined | null;
+    const [ scope, name ] = getScopeAndName(spec.name!);
+    // 优先通过 tag 来进行判断
+    if (spec.type === 'tag') {
+      version = await this.packageVersionRepository.findVersionByTag(scope, name, spec.fetchSpec!);
+    } else if (spec.type === 'version') {
+      // 1.0.0
+      // '=1.0.0' => '1.0.0'
+      // https://github.com/npm/npm-package-arg/blob/main/lib/npa.js#L392
+      version = semver.valid(spec.fetchSpec!, true);
+    } else if (spec.type === 'range') {
+      // a@1.1 情况下，1.1 会解析为 range，如果有对应的 distTag 时会失效
+      // 这里需要进行兼容
+      // 仅当 spec 不为 version 时才查询，减少请求次数
+      const versionMatchTag = await this.packageVersionRepository.findVersionByTag(scope, name, spec.fetchSpec!);
+      if (versionMatchTag) {
+        version = versionMatchTag;
+      } else {
+        const range = new Range(spec.fetchSpec!);
+        const paddingSemVer = new SqlRange(range);
+        if (paddingSemVer.containPreRelease) {
+          const versions = await this.packageVersionRepository.findSatisfyVersionsWithPrerelease(scope, name, paddingSemVer);
+          version = semver.maxSatisfying(versions, range);
+        } else {
+          version = await this.packageVersionRepository.findMaxSatisfyVersion(scope, name, paddingSemVer);
+        }
+      }
+    }
+    if (version && withBugVersion) {
+      const bugVersion = await this.bugVersionService.getBugVersion();
+      if (bugVersion) {
+        const advice = bugVersion.fixVersion(spec.name!, version);
+        if (advice) {
+          version = advice.version;
+        }
+      }
+    }
+    return version;
+  }
+}

app/core/service/RegistryManagerService.ts

@@ -14,10 +14,10 @@ import { Task } from '../entity/Task';
 import { ChangesStreamMode, PresetRegistryName } from '../../common/constants';
 import { RegistryType } from '../../common/enum/Registry';
 
-export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name'> {
+export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
-export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'registryId'> {
+export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
 export interface RemoveRegistryCmd extends Pick<Registry, 'registryId'> {
@@ -61,7 +61,7 @@ export class RegistryManagerService extends AbstractService {
   }
 
   async createRegistry(createCmd: CreateRegistryCmd): Promise<Registry> {
-    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-' } = createCmd;
+    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-', authToken } = createCmd;
     this.logger.info('[RegistryManagerService.createRegistry:prepare] operatorId: %s, createCmd: %j', operatorId, createCmd);
     const registry = Registry.create({
       name,
@@ -69,6 +69,7 @@ export class RegistryManagerService extends AbstractService {
       host,
       userPrefix,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
     return registry;
@@ -76,8 +77,8 @@ export class RegistryManagerService extends AbstractService {
 
   // 更新部分 registry 信息
   // 不允许 userPrefix 字段变更
-  async updateRegistry(updateCmd: UpdateRegistryCmd) {
-    const { name, changeStream, host, type, registryId, operatorId = '-' } = updateCmd;
+  async updateRegistry(registryId: string, updateCmd: UpdateRegistryCmd) {
+    const { name, changeStream, host, type, operatorId = '-', authToken } = updateCmd;
     this.logger.info('[RegistryManagerService.updateRegistry:prepare] operatorId: %s, updateCmd: %j', operatorId, updateCmd);
     const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
     if (!registry) {
@@ -88,6 +89,7 @@ export class RegistryManagerService extends AbstractService {
       changeStream,
       host,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
   }
@@ -105,6 +107,10 @@ export class RegistryManagerService extends AbstractService {
     return await this.registryRepository.findRegistry(registryName);
   }
 
+  async findByRegistryHost(host?: string): Promise<Registry | null> {
+    return host ? await this.registryRepository.findRegistryByRegistryHost(host) : null;
+  }
+
   // 删除 Registry 方法
   // 可选传入 operatorId 作为参数，用于记录操作人员
   // 同时删除对应的 scope 数据
@@ -156,4 +162,12 @@ export class RegistryManagerService extends AbstractService {
 
   }
 
+  async getAuthTokenByRegistryHost(host: string): Promise<string|undefined> {
+    const registry = await this.findByRegistryHost(host);
+    if (!registry) {
+      return undefined;
+    }
+    return registry.authToken;
+  }
+
 }

app/core/service/TaskService.ts

@@ -7,7 +7,7 @@ import { NFSAdapter } from '../../common/adapter/NFSAdapter';
 import { TaskState, TaskType } from '../../common/enum/Task';
 import { AbstractService } from '../../common/AbstractService';
 import { TaskRepository } from '../../repository/TaskRepository';
-import { Task } from '../entity/Task';
+import { Task, CreateSyncPackageTaskData } from '../entity/Task';
 import { QueueAdapter } from '../../common/typing';
 
 @SingletonProto({
@@ -31,6 +31,21 @@ export class TaskService extends AbstractService {
       // 如果任务还未被触发，就不继续重复创建
       // 如果任务正在执行，可能任务状态已更新，这种情况需要继续创建
       if (existsTask.state === TaskState.Waiting) {
+        if (task.type === TaskType.SyncPackage) {
+          // 如果是specificVersions的任务则可能可以和存量任务进行合并
+          const specificVersions = (task as Task<CreateSyncPackageTaskData>).data?.specificVersions;
+          const existsTaskSpecificVersions = (existsTask as Task<CreateSyncPackageTaskData>).data?.specificVersions;
+          if (existsTaskSpecificVersions) {
+            if (specificVersions) {
+              // 存量的任务和新增任务都是同步指定版本的任务，合并两者版本至存量任务
+              await this.taskRepository.updateSpecificVersionsOfWaitingTask(existsTask, specificVersions);
+            } else {
+              // 新增任务是全量同步任务，移除存量任务中的指定版本使其成为全量同步任务
+              await this.taskRepository.updateSpecificVersionsOfWaitingTask(existsTask);
+            }
+          }
+          // 存量任务是全量同步任务，直接提高任务优先级
+        }
         // 提高任务的优先级
         if (addTaskQueueOnExists) {
           const queueLength = await this.getTaskQueueLength(task.type);

app/core/service/TokenService.ts

@@ -36,17 +36,17 @@ export class TokenService extends AbstractService {
     return null;
   }
 
-  public async checkGranularTokenAccess(token: Token, fullname: string) {
-    // skip classic token
-    if (!isGranularToken(token)) {
-      return true;
-    }
-
+  public async checkTokenStatus(token: Token) {
     // check for expires
-    if (dayjs(token.expiredAt).isBefore(new Date())) {
+    if (isGranularToken(token) && dayjs(token.expiredAt).isBefore(new Date())) {
       throw new UnauthorizedError('Token expired');
     }
 
+    token.lastUsedAt = new Date();
+    this.userRepository.saveToken(token);
+  }
+
+  public async checkGranularTokenAccess(token: Token, fullname: string) {
     // check for scope whitelist
     const [ scope, name ] = getScopeAndName(fullname);
     // check for packages whitelist

app/core/service/UserService.ts

@@ -12,6 +12,9 @@ import { WebauthnCredential as WebauthnCredentialEntity } from '../entity/Webaut
 import { LoginResultCode } from '../../common/enum/User';
 import { integrity, checkIntegrity, randomToken, sha512 } from '../../common/UserUtil';
 import { AbstractService } from '../../common/AbstractService';
+import { RegistryManagerService } from './RegistryManagerService';
+import { getPrefixedName } from '../../common/PackageUtil';
+import { Registry } from '../entity/Registry';
 
 type Optional<T, K extends keyof T> = Omit < T, K > & Partial<T> ;
 
@@ -59,12 +62,36 @@ type CreateWebauthnCredentialOptions = {
 export class UserService extends AbstractService {
   @Inject()
   private readonly userRepository: UserRepository;
+  @Inject()
+  private readonly registryManagerService: RegistryManagerService;
 
   checkPassword(user: UserEntity, password: string): boolean {
     const plain = `${user.passwordSalt}${password}`;
     return checkIntegrity(plain, user.passwordIntegrity);
   }
 
+  async findUserByNameOrDisplayName(name: string) {
+    const hasPrefix = name.includes(':');
+    if (hasPrefix) {
+      return await this.findUserByName(name);
+    }
+
+    const selfRegistry = await this.registryManagerService.ensureSelfRegistry();
+    const selfUser = await this.findUserByName(getPrefixedName(selfRegistry.userPrefix, name));
+    if (selfUser) {
+      return selfUser;
+    }
+
+    const defaultRegistry = await this.registryManagerService.ensureDefaultRegistry();
+    const defaultUser = await this.findUserByName(getPrefixedName(defaultRegistry.userPrefix, name));
+
+    return defaultUser;
+  }
+
+  async findInRegistry(registry:Registry, name: string): Promise<UserEntity | null> {
+    return await this.findUserByName(getPrefixedName(registry.userPrefix, name));
+  }
+
   async findUserByName(name: string): Promise<UserEntity | null> {
     return await this.userRepository.findUserByName(name);
   }
@@ -79,19 +106,23 @@ export class UserService extends AbstractService {
     return { code: LoginResultCode.Success, user, token };
   }
 
-  async ensureTokenByUser({ name, email, password = crypto.randomUUID(), ip }: Optional<CreateUser, 'password'>) {
+  async findOrCreateUser({ name, email, ip, password = crypto.randomUUID() }: Optional<CreateUser, 'password'>) {
     let user = await this.userRepository.findUserByName(name);
     if (!user) {
       const createRes = await this.create({
         name,
         email,
-        // Authentication via sso
-        // should use token instead of password
         password,
         ip,
       });
       user = createRes.user;
     }
+
+    return user;
+  }
+
+  async ensureTokenByUser(opts: Optional<CreateUser, 'password'>) {
+    const user = await this.findOrCreateUser(opts);
     const token = await this.createToken(user.userId);
     return { user, token };
   }
@@ -172,14 +203,14 @@ export class UserService extends AbstractService {
     await this.userRepository.removeToken(token.tokenId);
   }
 
-  async findWebauthnCredential(userId: string, browserType?: string) {
+  async findWebauthnCredential(userId: string, browserType: string | undefined | null) {
     const credential = await this.userRepository.findCredentialByUserIdAndBrowserType(userId, browserType || null);
     return credential;
   }
 
-  async createWebauthnCredential(userId: string, options: CreateWebauthnCredentialOptions) {
+  async createWebauthnCredential(userId: string | undefined, options: CreateWebauthnCredentialOptions) {
     const credentialEntity = WebauthnCredentialEntity.create({
-      userId,
+      userId: userId as string,
       credentialId: options.credentialId,
       publicKey: options.publicKey,
       browserType: options.browserType,
@@ -188,7 +219,7 @@ export class UserService extends AbstractService {
     return credentialEntity;
   }
 
-  async removeWebauthnCredential(userId: string, browserType?: string) {
+  async removeWebauthnCredential(userId?: string, browserType?: string) {
     const credential = await this.userRepository.findCredentialByUserIdAndBrowserType(userId, browserType || null);
     if (credential) {
       await this.userRepository.removeCredential(credential.wancId);

app/infra/SearchAdapter.ts

@@ -0,0 +1,52 @@
+import {
+  AccessLevel,
+  Inject,
+  SingletonProto,
+} from '@eggjs/tegg';
+import { EggAppConfig } from 'egg';
+
+import { Client as ElasticsearchClient, estypes } from '@elastic/elasticsearch';
+import { SearchAdapter } from '../common/typing';
+
+/**
+ * Use elasticsearch to search the huge npm packages.
+ */
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+  name: 'searchAdapter',
+})
+export class ESSearchAdapter implements SearchAdapter {
+  @Inject()
+  private config: EggAppConfig;
+
+  @Inject()
+  private readonly elasticsearch: ElasticsearchClient; // 由 elasticsearch 插件引入
+
+  async search<T>(query: any): Promise<estypes.SearchHitsMetadata<T>> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const result = await this.elasticsearch.search<T>({
+      index,
+      ...query,
+    });
+    return result.hits;
+  }
+
+  async upsert<T>(id: string, document: T): Promise<string> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const res = await this.elasticsearch.index({
+      id,
+      index,
+      document,
+    });
+    return res._id;
+  }
+
+  async delete(id: string): Promise<string> {
+    const { cnpmcore: { elasticsearchIndex: index } } = this.config;
+    const res = await this.elasticsearch.delete({
+      index,
+      id,
+    });
+    return res._id;
+  }
+}

app/port/UserRoleManager.ts

@@ -111,6 +111,9 @@ export class UserRoleManager {
     if (!authorizedUserAndToken) {
       return null;
     }
+
+    // check token expired & set lastUsedAt
+    await this.tokenService.checkTokenStatus(authorizedUserAndToken.token);
     this.currentAuthorizedToken = authorizedUserAndToken.token;
     this.currentAuthorizedUser = authorizedUserAndToken.user;
     ctx.userId = authorizedUserAndToken.user.userId;

app/port/config.ts

@@ -145,4 +145,21 @@ export type CnpmcoreConfig = {
    * enable unpkg features, https://github.com/cnpm/cnpmcore/issues/452
    */
   enableUnpkg: boolean,
+  /**
+   * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag.
+   * in most cases, you should set to false to keep the same behavior as source registry.
+   */
+  strictSyncSpecivicVersion: boolean,
+  /**
+  * enable elasticsearch
+  */
+  enableElasticsearch: boolean,
+  /**
+  * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc.
+  */
+  elasticsearchIndex: string,
+  /**
+   * strictly enforces/validates manifest and tgz when publish, https://github.com/cnpm/cnpmcore/issues/542
+   */
+  strictValidateTarballPkg?: boolean,
 };

app/port/controller/AbstractController.ts

@@ -146,25 +146,25 @@ export abstract class AbstractController extends MiddlewareController {
     return new UnavailableForLegalReasonsError(`${message}, reason: ${reason}`);
   }
 
-  protected async getPackageEntityByFullname(fullname: string): Promise<PackageEntity> {
+  protected async getPackageEntityByFullname(fullname: string, allowSync?: boolean): Promise<PackageEntity> {
     const [ scope, name ] = getScopeAndName(fullname);
-    return await this.getPackageEntity(scope, name);
+    return await this.getPackageEntity(scope, name, allowSync);
   }
 
   // try to get package entity, throw NotFoundError when package not exists
-  protected async getPackageEntity(scope: string, name: string): Promise<PackageEntity> {
+  protected async getPackageEntity(scope: string, name: string, allowSync?:boolean): Promise<PackageEntity> {
     const packageEntity = await this.packageRepository.findPackage(scope, name);
     if (!packageEntity) {
       const fullname = getFullname(scope, name);
-      throw this.createPackageNotFoundErrorWithRedirect(fullname);
+      throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
     }
     return packageEntity;
   }
 
-  protected async getPackageVersionEntity(pkg: PackageEntity, version: string): Promise<PackageVersionEntity> {
+  protected async getPackageVersionEntity(pkg: PackageEntity, version: string, allowSync?: boolean): Promise<PackageVersionEntity> {
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
     if (!packageVersion) {
-      throw new NotFoundError(`${pkg.fullname}@${version} not found`);
+      throw this.createPackageNotFoundErrorWithRedirect(pkg.fullname, version, allowSync);
     }
     return packageVersion;
   }

app/port/controller/DownloadController.ts

@@ -28,15 +28,15 @@ export class DownloadController extends AbstractController {
     const pkg = await this.packageRepository.findPackage(scope, name);
     if (!pkg) throw new NotFoundError(`${fullname} not found`);
     const entities = await this.packageVersionDownloadRepository.query(pkg.packageId, startDate.toDate(), endDate.toDate());
-    const days = {};
-    const versions = {};
+    const days: Record<string, number> = {};
+    const versions: Record<string, { day: string, downloads: number }[]> = {};
     for (const entity of entities) {
       const yearMonth = String(entity.yearMonth);
       const prefix = yearMonth.substring(0, 4) + '-' + yearMonth.substring(4, 6);
       for (let i = 1; i <= 31; i++) {
         const day = String(i).padStart(2, '0');
-        const field = `d${day}`;
-        const counter = entity[field];
+        const field = `d${day}` as keyof typeof entity;
+        const counter = entity[field] as number;
         if (!counter) continue;
         const date = `${prefix}-${day}`;
         days[date] = (days[date] || 0) + counter;
@@ -66,14 +66,14 @@ export class DownloadController extends AbstractController {
   async showTotalDownloads(@HTTPParam() scope: string, @HTTPParam() range: string) {
     const [ startDate, endDate ] = this.checkAndGetRange(range);
     const entities = await this.packageVersionDownloadRepository.query(scope, startDate.toDate(), endDate.toDate());
-    const days = {};
+    const days: Record<string, number> = {};
     for (const entity of entities) {
       const yearMonth = String(entity.yearMonth);
       const prefix = yearMonth.substring(0, 4) + '-' + yearMonth.substring(4, 6);
       for (let i = 1; i <= 31; i++) {
         const day = String(i).padStart(2, '0');
-        const field = `d${day}`;
-        const counter = entity[field];
+        const field = `d${day}` as keyof typeof entity;
+        const counter = entity[field] as number;
         if (!counter) continue;
         const date = `${prefix}-${day}`;
         days[date] = (days[date] || 0) + counter;
@@ -115,4 +115,3 @@ export class DownloadController extends AbstractController {
     return [ startDate, endDate ];
   }
 }
-

app/port/controller/HomeController.ts

@@ -9,6 +9,7 @@ import {
 } from '@eggjs/tegg';
 import { AbstractController } from './AbstractController';
 import { CacheService, DownloadInfo, UpstreamRegistryInfo } from '../../core/service/CacheService';
+import { HomeService } from '../../core/service/HomeService';
 
 const startTime = new Date();
 
@@ -51,6 +52,9 @@ export class HomeController extends AbstractController {
   @Inject()
   private readonly cacheService: CacheService;
 
+  @Inject()
+  private readonly homeService: HomeService;
+
   @HTTPMethod({
     // GET /
     // https://github.com/cnpm/cnpmjs.org/blob/master/docs/registry-api.md#schema
@@ -97,4 +101,23 @@ export class HomeController extends AbstractController {
       use: performance.now() - ctx.performanceStarttime!,
     };
   }
+
+  @HTTPMethod({
+    path: '/*',
+    method: HTTPMethodEnum.POST,
+    priority: -Infinity,
+  })
+  async miscPost(@Context() ctx: EggContext) {
+    await this.homeService.misc(ctx.path);
+  }
+
+  @HTTPMethod({
+    path: '/*',
+    method: HTTPMethodEnum.GET,
+    priority: -Infinity,
+  })
+  async miscGet(@Context() ctx: EggContext) {
+    await this.homeService.misc(ctx.path);
+  }
+
 }

app/port/controller/PackageSyncController.ts

@@ -67,13 +67,13 @@ export class PackageSyncController extends AbstractController {
 
     const params = {
       fullname,
-      remoteAuthToken: data.remoteAuthToken,
       tips,
       skipDependencies: !!data.skipDependencies,
       syncDownloadData: !!data.syncDownloadData,
       force: !!data.force,
       // only admin allow to sync history version
       forceSyncHistory: !!data.forceSyncHistory && isAdmin,
+      specificVersions: data.specificVersions,
     };
     ctx.tValidate(SyncPackageTaskRule, params);
     const [ scope, name ] = getScopeAndName(params.fullname);
@@ -96,12 +96,12 @@ export class PackageSyncController extends AbstractController {
     const task = await this.packageSyncerService.createTask(params.fullname, {
       authorIp: ctx.ip,
       authorId: authorized?.user.userId,
-      remoteAuthToken: params.remoteAuthToken,
       tips: params.tips,
       skipDependencies: params.skipDependencies,
       syncDownloadData: params.syncDownloadData,
       forceSyncHistory: params.forceSyncHistory,
       registryId: registry?.registryId,
+      specificVersions: params.specificVersions && JSON.parse(params.specificVersions),
     });
     ctx.logger.info('[PackageSyncController.createSyncTask:success] taskId: %s, fullname: %s',
       task.taskId, fullname);

app/port/controller/PackageTagController.ts

@@ -29,7 +29,7 @@ export class PackageTagController extends AbstractController {
   async showTags(@HTTPParam() fullname: string) {
     const packageEntity = await this.getPackageEntityByFullname(fullname);
     const tagEntities = await this.packageRepository.listPackageTags(packageEntity.packageId);
-    const tags = {};
+    const tags: Record<string, string> = {};
     for (const entity of tagEntities) {
       tags[entity.tag] = entity.version;
     }

app/port/controller/PackageVersionFileController.ts

@@ -19,6 +19,7 @@ import { PackageManagerService } from '../../core/service/PackageManagerService'
 import { PackageVersionFile } from '../../core/entity/PackageVersionFile';
 import { PackageVersion } from '../../core/entity/PackageVersion';
 import { DistRepository } from '../../repository/DistRepository';
+import { Spec } from '../typebox';
 
 type FileItem = {
   path: string,
@@ -65,49 +66,51 @@ export class PackageVersionFileController extends AbstractController {
   }
 
   @HTTPMethod({
-    // PUT /:fullname/:versionOrTag/files
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files`,
+    // PUT /:fullname/:versionSpec/files
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`,
     method: HTTPMethodEnum.PUT,
   })
   @Middleware(AdminAccess)
-  async sync(@HTTPParam() fullname: string, @HTTPParam() versionOrTag: string) {
+  async sync(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionSpec: string) {
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     this.#requireUnpkgEnable();
     const [ scope, name ] = getScopeAndName(fullname);
     const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
-      scope, name, versionOrTag);
+      scope, name, versionSpec);
     if (!packageVersion) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+      throw new NotFoundError(`${fullname}@${versionSpec} not found`);
     }
     const files = await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
     return files.map(file => formatFileItem(file));
   }
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag/files => /:fullname/:versionOrTag/files/${pkg.main}
-    // GET /:fullname/:versionOrTag/files?meta
-    // GET /:fullname/:versionOrTag/files/
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files`,
+    // GET /:fullname/:versionSpec/files => /:fullname/:versionSpec/files/${pkg.main}
+    // GET /:fullname/:versionSpec/files?meta
+    // GET /:fullname/:versionSpec/files/
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files`,
     method: HTTPMethodEnum.GET,
   })
   async listFiles(@Context() ctx: EggContext,
       @HTTPParam() fullname: string,
-      @HTTPParam() versionOrTag: string,
+      @HTTPParam() versionSpec: string,
       @HTTPQuery() meta: string) {
     this.#requireUnpkgEnable();
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     ctx.vary(this.config.cnpmcore.cdnVaryHeader);
     const [ scope, name ] = getScopeAndName(fullname);
-    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionOrTag);
+    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionSpec);
     ctx.set('cache-control', META_CACHE_CONTROL);
     const hasMeta = typeof meta === 'string' || ctx.path.endsWith('/files/');
     // meta request
     if (hasMeta) {
       const files = await this.#listFilesByDirectory(packageVersion, '/');
       if (!files) {
-        throw new NotFoundError(`${fullname}@${versionOrTag}/files not found`);
+        throw new NotFoundError(`${fullname}@${versionSpec}/files not found`);
       }
       return files;
     }
-    const { manifest } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionOrTag);
+    const { manifest } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, false, true);
     // GET /foo/1.0.0/files => /foo/1.0.0/files/{main}
     // ignore empty entry exp: @types/node@20.2.5/
     const indexFile = manifest?.main || 'index.js';
@@ -115,26 +118,27 @@ export class PackageVersionFileController extends AbstractController {
   }
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag/files/:path
-    // GET /:fullname/:versionOrTag/files/:path?meta
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag/files/:path(.+)`,
+    // GET /:fullname/:versionSpec/files/:path
+    // GET /:fullname/:versionSpec/files/:path?meta
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec/files/:path(.+)`,
     method: HTTPMethodEnum.GET,
   })
   async raw(@Context() ctx: EggContext,
       @HTTPParam() fullname: string,
-      @HTTPParam() versionOrTag: string,
+      @HTTPParam() versionSpec: string,
       @HTTPParam() path: string,
       @HTTPQuery() meta: string) {
     this.#requireUnpkgEnable();
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     ctx.vary(this.config.cnpmcore.cdnVaryHeader);
     const [ scope, name ] = getScopeAndName(fullname);
     path = `/${path}`;
-    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionOrTag);
+    const packageVersion = await this.#getPackageVersion(ctx, fullname, scope, name, versionSpec);
     if (path.endsWith('/')) {
       const directory = path.substring(0, path.length - 1);
       const files = await this.#listFilesByDirectory(packageVersion, directory);
       if (!files) {
-        throw new NotFoundError(`${fullname}@${versionOrTag}/files${directory} not found`);
+        throw new NotFoundError(`${fullname}@${versionSpec}/files${directory} not found`);
       }
       ctx.set('cache-control', META_CACHE_CONTROL);
       return files;
@@ -142,7 +146,7 @@ export class PackageVersionFileController extends AbstractController {
 
     const file = await this.packageVersionFileService.showPackageVersionFile(packageVersion, path);
     if (!file) {
-      throw new NotFoundError(`File ${fullname}@${versionOrTag}${path} not found`);
+      throw new NotFoundError(`File ${fullname}@${versionSpec}${path} not found`);
     }
     const hasMeta = typeof meta === 'string';
     if (hasMeta) {
@@ -157,19 +161,20 @@ export class PackageVersionFileController extends AbstractController {
     return await this.distRepository.getDistStream(file.dist);
   }
 
-  async #getPackageVersion(ctx: EggContext, fullname: string, scope: string, name: string, versionOrTag: string) {
+  async #getPackageVersion(ctx: EggContext, fullname: string, scope: string, name: string, versionSpec: string) {
     const { blockReason, packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
-      scope, name, versionOrTag);
+      scope, name, versionSpec);
     if (blockReason) {
       this.setCDNHeaders(ctx);
-      throw this.createPackageBlockError(blockReason, fullname, versionOrTag);
+      throw this.createPackageBlockError(blockReason, fullname, versionSpec);
     }
     if (!packageVersion) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+      throw new NotFoundError(`${fullname}@${versionSpec} not found`);
     }
-    if (packageVersion.version !== versionOrTag) {
+    if (packageVersion.version !== versionSpec) {
       ctx.set('cache-control', META_CACHE_CONTROL);
-      const location = ctx.url.replace(`/${fullname}/${versionOrTag}/files`, `/${fullname}/${packageVersion.version}/files`);
+      let location = ctx.url.replace(`/${fullname}/${versionSpec}/files`, `/${fullname}/${packageVersion.version}/files`);
+      location = location.replace(`/${fullname}/${encodeURIComponent(versionSpec)}/files`, `/${fullname}/${packageVersion.version}/files`);
       throw this.createControllerRedirectError(location);
     }
     return packageVersion;

app/port/controller/RegistryController.ts

@@ -13,10 +13,10 @@ import {
 import { NotFoundError } from 'egg-errors';
 import { AbstractController } from './AbstractController';
 import { Static } from 'egg-typebox-validate/typebox';
-import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { RegistryManagerService, UpdateRegistryCmd } from '../../core/service/RegistryManagerService';
 import { AdminAccess } from '../middleware/AdminAccess';
 import { ScopeManagerService } from '../../core/service/ScopeManagerService';
-import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions } from '../typebox';
+import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions, RegistryUpdateOptions } from '../typebox';
 
 @HTTPController()
 export class RegistryController extends AbstractController {
@@ -67,7 +67,7 @@ export class RegistryController extends AbstractController {
   async createRegistry(@Context() ctx: EggContext, @HTTPBody() registryOptions: Static<typeof RegistryCreateOptions>) {
     ctx.tValidate(RegistryCreateOptions, registryOptions);
     const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
-    const { name, changeStream, host, userPrefix = '', type } = registryOptions;
+    const { name, changeStream, host, userPrefix = '', type, authToken } = registryOptions;
     await this.registryManagerService.createRegistry({
       name,
       changeStream,
@@ -75,6 +75,7 @@ export class RegistryController extends AbstractController {
       userPrefix,
       operatorId: authorizedUser.userId,
       type,
+      authToken,
     });
     return { ok: true };
   }
@@ -106,4 +107,29 @@ export class RegistryController extends AbstractController {
     await this.registryManagerService.remove({ registryId: id, operatorId: authorizedUser.userId });
     return { ok: true };
   }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.PATCH,
+  })
+  @Middleware(AdminAccess)
+  async updateRegistry(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() updateRegistryOptions: Partial<UpdateRegistryCmd>) {
+    ctx.tValidate(RegistryUpdateOptions, updateRegistryOptions);
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    } else {
+      const { name, changeStream, host, type, authToken } = registry;
+      const _updateRegistryOptions = {
+        name,
+        changeStream,
+        host,
+        type,
+        authToken,
+        ...updateRegistryOptions,
+      };
+      await this.registryManagerService.updateRegistry(registry.registryId, _updateRegistryOptions);
+    }
+    return { ok: true };
+  }
 }

app/port/controller/TokenController.ts

@@ -13,8 +13,6 @@ import {
 import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from './AbstractController';
 import { TokenType, isGranularToken } from '../../core/entity/Token';
-import { TokenService } from '../../../app/core/service/TokenService';
-import { getFullname } from '../../../app/common/PackageUtil';
 
 // Creating and viewing access tokens
 // https://docs.npmjs.com/creating-and-viewing-access-tokens#viewing-access-tokens
@@ -44,8 +42,6 @@ type GranularTokenOptions = Static<typeof GranularTokenOptionsRule>;
 export class TokenController extends AbstractController {
   @Inject()
   private readonly authAdapter: AuthAdapter;
-  @Inject()
-  private readonly tokenService: TokenService;
   // https://github.com/npm/npm-profile/blob/main/lib/index.js#L233
   @HTTPMethod({
     path: '/-/npm/v1/tokens',
@@ -127,6 +123,7 @@ export class TokenController extends AbstractController {
           readonly: token.isReadonly,
           automation: token.isAutomation,
           created: token.createdAt,
+          lastUsedAt: token.lastUsedAt,
           updated: token.updatedAt,
         };
       });
@@ -134,15 +131,12 @@ export class TokenController extends AbstractController {
     return { objects, total: objects.length, urls: {} };
   }
 
-  private async ensureWebUser() {
+  private async ensureWebUser(ip = '') {
     const userRes = await this.authAdapter.ensureCurrentUser();
     if (!userRes?.name || !userRes?.email) {
       throw new ForbiddenError('need login first');
     }
-    const user = await this.userService.findUserByName(userRes.name);
-    if (!user?.userId) {
-      throw new ForbiddenError('invalid user info');
-    }
+    const user = await this.userService.findOrCreateUser({ name: userRes.name, email: userRes.email, ip });
     return user;
   }
 
@@ -158,7 +152,7 @@ export class TokenController extends AbstractController {
   // 3. Need to implement ensureCurrentUser method in AuthAdapter, or pass in this.user
   async createGranularToken(@Context() ctx: EggContext, @HTTPBody() tokenOptions: GranularTokenOptions) {
     ctx.tValidate(GranularTokenOptionsRule, tokenOptions);
-    const user = await this.ensureWebUser();
+    const user = await this.ensureWebUser(ctx.ip);
 
     // 生成 Token
     const { name, description, allowedPackages, allowedScopes, cidr_whitelist, automation, readonly, expires } = tokenOptions;
@@ -197,19 +191,14 @@ export class TokenController extends AbstractController {
     const tokens = await this.userRepository.listTokens(user.userId);
     const granularTokens = tokens.filter(token => isGranularToken(token));
 
-    for (const token of granularTokens) {
-      const packages = await this.tokenService.listTokenPackages(token);
-      if (Array.isArray(packages)) {
-        token.allowedPackages = packages.map(p => getFullname(p.scope, p.name));
-      }
-    }
     const objects = granularTokens.map(token => {
-      const { name, description, expiredAt, allowedPackages, allowedScopes } = token;
+      const { name, description, expiredAt, allowedPackages, allowedScopes, lastUsedAt, type } = token;
       return {
         name,
         description,
         allowedPackages,
         allowedScopes,
+        lastUsedAt,
         expiredAt,
         token: token.tokenMark,
         key: token.tokenKey,
@@ -217,6 +206,7 @@ export class TokenController extends AbstractController {
         readonly: token.isReadonly,
         created: token.createdAt,
         updated: token.updatedAt,
+        type,
       };
     });
     return { objects, total: granularTokens.length, urls: {} };

app/port/controller/UserController.ts

@@ -17,6 +17,7 @@ import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from './AbstractController';
 import { LoginResultCode } from '../../common/enum/User';
 import { sha512 } from '../../common/UserUtil';
+import { isGranularToken } from '../../core/entity/Token';
 
 // body: {
 //   _id: 'org.couchdb.user:dddd',
@@ -89,7 +90,7 @@ export class UserController extends AbstractController {
       ctx.status = 201;
       return {
         ok: true,
-        id: `org.couchdb.user:${result.user?.name}`,
+        id: `org.couchdb.user:${result.user?.displayName}`,
         rev: result.user?.userId,
         token: result.token?.token,
       };
@@ -112,7 +113,7 @@ export class UserController extends AbstractController {
     ctx.status = 201;
     return {
       ok: true,
-      id: `org.couchdb.user:${userEntity.name}`,
+      id: `org.couchdb.user:${userEntity.displayName}`,
       rev: userEntity.userId,
       token: token.token,
     };
@@ -139,14 +140,14 @@ export class UserController extends AbstractController {
     method: HTTPMethodEnum.GET,
   })
   async showUser(@Context() ctx: EggContext, @HTTPParam() username: string) {
-    const user = await this.userRepository.findUserByName(username);
+    const user = await this.userService.findUserByNameOrDisplayName(username);
     if (!user) {
       throw new NotFoundError(`User "${username}" not found`);
     }
     const authorized = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
     return {
-      _id: `org.couchdb.user:${user.name}`,
-      name: user.name,
+      _id: `org.couchdb.user:${user.displayName}`,
+      name: user.displayName,
       email: authorized ? user.email : undefined,
     };
   }
@@ -157,10 +158,34 @@ export class UserController extends AbstractController {
     method: HTTPMethodEnum.GET,
   })
   async whoami(@Context() ctx: EggContext) {
-    const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    await this.userRoleManager.requiredAuthorizedUser(ctx, 'read');
+    const authorizedRes = await this.userRoleManager.getAuthorizedUserAndToken(ctx);
+    const { token, user } = authorizedRes!;
+
+    if (isGranularToken(token)) {
+      const { name, description, expiredAt, allowedPackages, allowedScopes, lastUsedAt, type } = token;
+      return {
+        username: user.displayName,
+        name,
+        description,
+        allowedPackages,
+        allowedScopes,
+        lastUsedAt,
+        expiredAt,
+        // do not return token value
+        // token: token.token,
+        key: token.tokenKey,
+        cidr_whitelist: token.cidrWhitelist,
+        readonly: token.isReadonly,
+        created: token.createdAt,
+        updated: token.updatedAt,
+        type,
+      };
+    }
     return {
-      username: authorizedUser.displayName,
+      username: user.displayName,
     };
+
   }
 
   // https://github.com/cnpm/cnpmcore/issues/64
@@ -184,7 +209,7 @@ export class UserController extends AbstractController {
       //   "pending": false,
       //   "mode": "auth-only"
       // },
-      name: authorizedUser.name,
+      name: authorizedUser.displayName,
       email: authorizedUser.email,
       email_verified: false,
       created: authorizedUser.createdAt,

app/port/controller/admin/PaddingVersionController.ts

@@ -0,0 +1,43 @@
+import {
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  Inject,
+  HTTPQuery, Context, EggContext,
+} from '@eggjs/tegg';
+import { AbstractController } from '../AbstractController';
+import { FixNoPaddingVersionService } from '../../../core/service/FixNoPaddingVersionService';
+
+@HTTPController()
+export class PaddingVersionController extends AbstractController {
+  @Inject()
+  private readonly fixNoPaddingVersionService: FixNoPaddingVersionService;
+
+  @HTTPMethod({
+    method: HTTPMethodEnum.PUT,
+    path: '/-/admin/npm/fixPaddingVersion',
+  })
+  async fixNoPaddingVersion(@Context() ctx: EggContext, @HTTPQuery() id: string) {
+    const isAdmin = await this.userRoleManager.isAdmin(ctx);
+    if (!isAdmin) {
+      return {
+        ok: false,
+        error: 'only admin can do this',
+      };
+    }
+    let idNum: number | undefined;
+    if (id) {
+      idNum = parseInt(id);
+      if (Number.isNaN(idNum)) {
+        return {
+          ok: false,
+          error: `id is not a number ${id}`,
+        };
+      }
+    }
+    await this.fixNoPaddingVersionService.fixPaddingVersion(idNum);
+    return {
+      ok: true,
+    };
+  }
+}

app/port/controller/package/DownloadPackageVersionTar.ts

@@ -14,6 +14,7 @@ import { AbstractController } from '../AbstractController';
 import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil';
 import { NFSAdapter } from '../../../common/adapter/NFSAdapter';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { SyncMode } from '../../../common/constants';
 
 @HTTPController()
 export class DownloadPackageVersionTarController extends AbstractController {
@@ -22,26 +23,47 @@ export class DownloadPackageVersionTarController extends AbstractController {
   @Inject()
   private nfsAdapter: NFSAdapter;
 
+  // Support OPTIONS Request on tgz download
+  @HTTPMethod({
+    // GET /:fullname/-/:filenameWithVersion.tgz
+    path: `/:fullname(${FULLNAME_REG_STRING})/-/:filenameWithVersion.tgz`,
+    method: HTTPMethodEnum.OPTIONS,
+  })
+  async downloadForOptions(@Context() ctx: EggContext) {
+    ctx.set('access-control-allow-origin', '*');
+    ctx.set('access-control-allow-methods', 'GET,HEAD');
+    ctx.status = 204;
+  }
+
   @HTTPMethod({
     // GET /:fullname/-/:filenameWithVersion.tgz
     path: `/:fullname(${FULLNAME_REG_STRING})/-/:filenameWithVersion.tgz`,
     method: HTTPMethodEnum.GET,
   })
   async download(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() filenameWithVersion: string) {
-    // try nfs url first, avoid db query
     // tgz file storeKey: `/packages/${this.fullname}/${version}/${filename}`
     const version = this.getAndCheckVersionFromFilename(ctx, fullname, filenameWithVersion);
     const storeKey = `/packages/${fullname}/${version}/${filenameWithVersion}.tgz`;
     const downloadUrl = await this.nfsAdapter.getDownloadUrl(storeKey);
-    if (downloadUrl) {
+    if (this.config.cnpmcore.syncMode === SyncMode.all && downloadUrl) {
+      // try nfs url first, avoid db query
       this.packageManagerService.plusPackageVersionCounter(fullname, version);
       ctx.redirect(downloadUrl);
       return;
     }
 
+    // check package version in database
+    const allowSync = this.getAllowSync(ctx);
+    const pkg = await this.getPackageEntityByFullname(fullname, allowSync);
+    const packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync);
+
+    // read by nfs url
+    if (downloadUrl) {
+      this.packageManagerService.plusPackageVersionCounter(fullname, version);
+      ctx.redirect(downloadUrl);
+      return;
+    }
     // read from database
-    const pkg = await this.getPackageEntityByFullname(fullname);
-    const packageVersion = await this.getPackageVersionEntity(pkg, version);
     ctx.logger.info('[PackageController:downloadVersionTar] %s@%s, packageVersionId: %s',
       pkg.fullname, version, packageVersion.packageVersionId);
     const urlOrStream = await this.packageManagerService.downloadPackageVersionTar(packageVersion);

app/port/controller/package/SavePackageVersionController.ts

@@ -1,7 +1,9 @@
 import { PackageJson, Simplify } from 'type-fest';
+import { isEqual } from 'lodash';
 import {
   UnprocessableEntityError,
   ForbiddenError,
+  ConflictError,
 } from 'egg-errors';
 import {
   HTTPController,
@@ -17,8 +19,9 @@ import * as ssri from 'ssri';
 import validateNpmPackageName from 'validate-npm-package-name';
 import { Static, Type } from '@sinclair/typebox';
 import { AbstractController } from '../AbstractController';
-import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil';
+import { getScopeAndName, FULLNAME_REG_STRING, extractPackageJSON } from '../../../common/PackageUtil';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { PackageVersion as PackageVersionEntity } from '../../../core/entity/PackageVersion';
 import {
   VersionRule,
   TagWithVersionRule,
@@ -27,6 +30,9 @@ import {
 } from '../../typebox';
 import { RegistryManagerService } from '../../../core/service/RegistryManagerService';
 import { PackageJSONType } from '../../../repository/PackageRepository';
+import { CacheAdapter } from '../../../common/adapter/CacheAdapter';
+
+const STRICT_CHECK_TARBALL_FIELDS: (keyof PackageJson)[] = [ 'name', 'version', 'scripts', 'dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies', 'license', 'licenses', 'bin' ];
 
 type PackageVersion = Simplify<PackageJson.PackageJsonStandard & {
   name: 'string';
@@ -71,6 +77,9 @@ export class SavePackageVersionController extends AbstractController {
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
 
+  @Inject()
+  private readonly cacheAdapter: CacheAdapter;
+
   // https://github.com/cnpm/cnpmjs.org/blob/master/docs/registry-api.md#publish-a-new-package
   // https://github.com/npm/libnpmpublish/blob/main/publish.js#L43
   @HTTPMethod({
@@ -87,11 +96,17 @@ export class SavePackageVersionController extends AbstractController {
     if (fullname !== pkg.name) {
       throw new UnprocessableEntityError(`fullname(${fullname}) not match package.name(${pkg.name})`);
     }
+
     // Using https://github.com/npm/validate-npm-package-name to validate package name
     const validateResult = validateNpmPackageName(pkg.name);
     if (!validateResult.validForNewPackages) {
-      const errors = (validateResult.errors || validateResult.warnings).join(', ');
-      throw new UnprocessableEntityError(`package.name invalid, errors: ${errors}`);
+      // if pkg already exists, still allow to publish
+      const [ scope, name ] = getScopeAndName(fullname);
+      const pkg = await this.packageRepository.findPackage(scope, name);
+      if (!pkg) {
+        const errors = (validateResult.errors || validateResult.warnings || []).join(', ');
+        throw new UnprocessableEntityError(`package.name invalid, errors: ${errors}`);
+      }
     }
     const versions = Object.values(pkg.versions);
     if (versions.length === 0) {
@@ -123,11 +138,25 @@ export class SavePackageVersionController extends AbstractController {
 
     const attachment = attachments[attachmentFilename];
     const distTags = pkg['dist-tags'] ?? {};
-    const tagName = Object.keys(distTags)[0];
-    if (!tagName) {
+    let tagNames = Object.keys(distTags);
+    if (tagNames.length === 0) {
       throw new UnprocessableEntityError('dist-tags is empty');
     }
-    const tagWithVersion = { tag: tagName, version: distTags[tagName] };
+
+    const [ scope, name ] = getScopeAndName(fullname);
+    // see @https://github.com/cnpm/cnpmcore/issues/574
+    // add default latest tag
+    if (!pkg['dist-tags']!.latest) {
+      const existsPkg = await this.packageRepository.findPackage(scope, name);
+      const existsLatestTag = existsPkg && await this.packageRepository.findPackageTag(existsPkg?.packageId, 'latest');
+      if (!existsPkg || !existsLatestTag) {
+        this.logger.warn('[package:version:add] add default latest tag');
+        pkg['dist-tags']!.latest = pkg['dist-tags']![tagNames[0]];
+        tagNames = [ ...tagNames, 'latest' ];
+      }
+    }
+
+    const tagWithVersion = { tag: tagNames[0], version: distTags[tagNames[0]] };
     ctx.tValidate(TagWithVersionRule, tagWithVersion);
     if (tagWithVersion.version !== packageVersion.version) {
       throw new UnprocessableEntityError(`dist-tags version "${tagWithVersion.version}" not match package version "${packageVersion.version}"`);
@@ -165,7 +194,20 @@ export class SavePackageVersionController extends AbstractController {
       }
     }
 
-    const [ scope, name ] = getScopeAndName(fullname);
+    // https://github.com/cnpm/cnpmcore/issues/542
+    // check tgz & manifests
+    if (this.config.cnpmcore.strictValidateTarballPkg) {
+      const tarballPkg = await extractPackageJSON(tarballBytes);
+      const versionManifest = pkg.versions[tarballPkg.version];
+      const diffKeys = STRICT_CHECK_TARBALL_FIELDS.filter(key => {
+        const targetKey = key as unknown as keyof typeof versionManifest;
+        return !isEqual(tarballPkg[key], versionManifest[targetKey]);
+      });
+      if (diffKeys.length > 0) {
+        throw new UnprocessableEntityError(`${diffKeys} mismatch between tarball and manifest`);
+      }
+    }
+
 
     // make sure readme is string
     const readme = typeof packageVersion.readme === 'string' ? packageVersion.readme : '';
@@ -177,28 +219,38 @@ export class SavePackageVersionController extends AbstractController {
     }
 
     const registry = await this.registryManagerService.ensureSelfRegistry();
-    const packageVersionEntity = await this.packageManagerService.publish({
-      scope,
-      name,
-      version: packageVersion.version,
-      description: packageVersion.description,
-      packageJson: packageVersion as PackageJSONType,
-      readme,
-      dist: {
-        content: tarballBytes,
-      },
-      tag: tagWithVersion.tag,
-      registryId: registry.registryId,
-      isPrivate: true,
-    }, user);
+
+    let packageVersionEntity: PackageVersionEntity | undefined;
+    const lockRes = await this.cacheAdapter.usingLock(`${pkg.name}:publish`, 60, async () => {
+      packageVersionEntity = await this.packageManagerService.publish({
+        scope,
+        name,
+        version: packageVersion.version,
+        description: packageVersion.description as string,
+        packageJson: packageVersion as PackageJSONType,
+        readme,
+        dist: {
+          content: tarballBytes,
+        },
+        tags: tagNames,
+        registryId: registry.registryId,
+        isPrivate: true,
+      }, user);
+    });
+
+    // lock fail
+    if (!lockRes) {
+      this.logger.warn('[package:version:add] check lock fail');
+      throw new ConflictError('Unable to create the publication lock, please try again later.');
+    }
 
     this.logger.info('[package:version:add] %s@%s, packageVersionId: %s, tag: %s, userId: %s',
-      packageVersion.name, packageVersion.version, packageVersionEntity.packageVersionId,
-      tagWithVersion.tag, user.userId);
+      packageVersion.name, packageVersion.version, packageVersionEntity?.packageVersionId,
+      tagWithVersion.tag, user?.userId);
     ctx.status = 201;
     return {
       ok: true,
-      rev: `${packageVersionEntity.id}-${packageVersionEntity.packageVersionId}`,
+      rev: `${packageVersionEntity?.id}-${packageVersionEntity?.packageVersionId}`,
     };
   }
 

app/port/controller/package/SearchPackageController.ts

@@ -0,0 +1,69 @@
+import {
+  HTTPController,
+  HTTPMethod,
+  HTTPMethodEnum,
+  HTTPParam,
+  HTTPQuery,
+  Inject,
+  Middleware,
+  Context,
+  EggContext,
+} from '@eggjs/tegg';
+import { Static } from 'egg-typebox-validate/typebox';
+import { E451 } from 'egg-errors';
+
+import { AbstractController } from '../AbstractController';
+import { SearchQueryOptions } from '../../typebox';
+import { PackageSearchService } from '../../../core/service/PackageSearchService';
+import { FULLNAME_REG_STRING } from '../../../common/PackageUtil';
+import { AdminAccess } from '../../middleware/AdminAccess';
+
+@HTTPController()
+export class SearchPackageController extends AbstractController {
+  @Inject()
+  private readonly packageSearchService: PackageSearchService;
+
+  @HTTPMethod({
+    // GET /-/v1/search?text=react&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
+    path: '/-/v1/search',
+    method: HTTPMethodEnum.GET,
+  })
+  async search(
+    @Context() ctx: EggContext,
+    @HTTPQuery() text: Static<typeof SearchQueryOptions>['text'],
+    @HTTPQuery() from: Static<typeof SearchQueryOptions>['from'],
+    @HTTPQuery() size: Static<typeof SearchQueryOptions>['size'],
+  ) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const data = await this.packageSearchService.searchPackage(text, from, size);
+    this.setCDNHeaders(ctx);
+    return data;
+  }
+
+  @HTTPMethod({
+    path: `/-/v1/search/sync/:fullname(${FULLNAME_REG_STRING})`,
+    method: HTTPMethodEnum.PUT,
+  })
+  async sync(@HTTPParam() fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const name = await this.packageSearchService.syncPackage(fullname, true);
+    return { package: name };
+  }
+
+  @HTTPMethod({
+    path: `/-/v1/search/sync/:fullname(${FULLNAME_REG_STRING})`,
+    method: HTTPMethodEnum.DELETE,
+  })
+  @Middleware(AdminAccess)
+  async delete(@HTTPParam() fullname: string) {
+    if (!this.config.cnpmcore.enableElasticsearch) {
+      throw new E451('search feature not enabled in `config.cnpmcore.enableElasticsearch`');
+    }
+    const name = await this.packageSearchService.removePackage(fullname);
+    return { package: name };
+  }
+}

app/port/controller/package/ShowPackageVersionController.ts

@@ -12,6 +12,7 @@ import { AbstractController } from '../AbstractController';
 import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil';
 import { isSyncWorkerRequest } from '../../../common/SyncUtil';
 import { PackageManagerService } from '../../../core/service/PackageManagerService';
+import { Spec } from '../../../port/typebox';
 
 @HTTPController()
 export class ShowPackageVersionController extends AbstractController {
@@ -19,25 +20,29 @@ export class ShowPackageVersionController extends AbstractController {
   private packageManagerService: PackageManagerService;
 
   @HTTPMethod({
-    // GET /:fullname/:versionOrTag
-    path: `/:fullname(${FULLNAME_REG_STRING})/:versionOrTag`,
+    // GET /:fullname/:versionSpec
+    path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec`,
     method: HTTPMethodEnum.GET,
   })
-  async show(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionOrTag: string) {
+  async show(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionSpec: string) {
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format
+    ctx.tValidate(Spec, `${fullname}@${versionSpec}`);
     const [ scope, name ] = getScopeAndName(fullname);
     const isSync = isSyncWorkerRequest(ctx);
-    const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionOrTag, isSync);
+    const abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
+    const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType;
+
+    const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests);
     if (!pkg) {
       const allowSync = this.getAllowSync(ctx);
       throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
     }
     if (blockReason) {
       this.setCDNHeaders(ctx);
-      throw this.createPackageBlockError(blockReason, fullname, versionOrTag);
+      throw this.createPackageBlockError(blockReason, fullname, versionSpec);
     }
     if (!manifest) {
-      throw new NotFoundError(`${fullname}@${versionOrTag} not found`);
+      throw new NotFoundError(`${fullname}@${versionSpec} not found`);
     }
     this.setCDNHeaders(ctx);
     return manifest;

app/port/controller/package/UpdatePackageController.ts

@@ -50,17 +50,21 @@ export class UpdatePackageController extends AbstractController {
     ctx.tValidate(MaintainerDataRule, data);
     const ensureRes = await this.ensurePublishAccess(ctx, fullname, true);
     const pkg = ensureRes.pkg!;
+    const registry = await this.packageManagerService.getSourceRegistry(pkg);
     // make sure all maintainers exists
     const users: UserEntity[] = [];
     for (const maintainer of data.maintainers) {
-      // TODO check userPrefix
+      if (registry?.userPrefix && !maintainer.name.startsWith(registry.userPrefix)) {
+        maintainer.name = `${registry?.userPrefix}${maintainer.name}`;
+      }
       const user = await this.userRepository.findUserByName(maintainer.name);
       if (!user) {
         throw new UnprocessableEntityError(`Maintainer "${maintainer.name}" not exists`);
       }
       users.push(user);
     }
-    await this.packageManagerService.replacePackageMaintainers(pkg, users);
+
+    await this.packageManagerService.replacePackageMaintainersAndDist(pkg, users);
     return { ok: true };
   }
 

app/port/middleware/ErrorHandler.ts

@@ -32,7 +32,8 @@ export async function ErrorHandler(ctx: EggContext, next: Next) {
 
     // http status, default is DEFAULT_SERVER_ERROR_STATUS
     ctx.status = err.status || DEFAULT_SERVER_ERROR_STATUS;
-    if (ctx.status >= DEFAULT_SERVER_ERROR_STATUS) {
+    // don't log NotImplementedError
+    if (ctx.status >= DEFAULT_SERVER_ERROR_STATUS && err.name !== 'NotImplementedError') {
       ctx.logger.error(err);
     }
     let message = err.message;

app/port/schedule/SyncBinaryWorker.ts

@@ -29,7 +29,20 @@ export class SyncBinaryWorker {
     this.logger.info('[SyncBinaryWorker:executeTask:start] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms',
       task.taskId, task.targetName, task.attempts, task.data, task.updatedAt,
       startTime - task.updatedAt.getTime());
-    await this.binarySyncerService.executeTask(task);
+    try {
+      await this.binarySyncerService.executeTask(task);
+    } catch (err) {
+      const use = Date.now() - startTime;
+      this.logger.warn('[SyncBinaryWorker:executeTask:error] taskId: %s, targetName: %s, use %sms, error: %s',
+        task.taskId, task.targetName, use, err.message);
+      if (err.name === 'ConnectTimeoutError'
+        || err.name === 'HttpClientRequestTimeoutError') {
+        this.logger.warn(err);
+      } else {
+        this.logger.error(err);
+      }
+      return;
+    }
     const use = Date.now() - startTime;
     this.logger.info('[SyncBinaryWorker:executeTask:success] taskId: %s, targetName: %s, use %sms',
       task.taskId, task.targetName, use);

app/port/schedule/UpdateTotalData.ts

@@ -79,8 +79,8 @@ export class UpdateTotalData {
       for (const row of rows) {
         for (let i = 1; i <= 31; i++) {
           const day = String(i).padStart(2, '0');
-          const field = `d${day}`;
-          const counter = row[field];
+          const field = `d${day}` as keyof typeof row;
+          const counter = row[field] as number;
           if (!counter) continue;
           const dayInt = row.yearMonth * 100 + i;
           if (dayInt === todayInt) download.today += counter;

app/port/typebox.ts

@@ -1,8 +1,9 @@
 import { Type, Static } from '@sinclair/typebox';
 import { RegistryType } from '../common/enum/Registry';
 import semver from 'semver';
+import npa from 'npm-package-arg';
 import { HookType } from '../common/enum/Hook';
-import binaryConfig from '../../config/binaries';
+import binaryConfig, { BinaryName } from '../../config/binaries';
 
 export const Name = Type.String({
   transform: [ 'trim' ],
@@ -53,6 +54,16 @@ export const Version = Type.String({
   maxLength: 256,
 });
 
+export const VersionStringArray = Type.String({
+  format: 'semver-version-array',
+  transform: [ 'trim' ],
+});
+
+export const Spec = Type.String({
+  format: 'semver-spec',
+  minLength: 1,
+});
+
 export const Description = Type.String({ maxLength: 10240, transform: [ 'trim' ] });
 
 export const TagRule = Type.Object({
@@ -68,17 +79,12 @@ export const TagWithVersionRule = Type.Object({
 
 export const SyncPackageTaskRule = Type.Object({
   fullname: Name,
-  remoteAuthToken: Type.Optional(
-    Type.String({
-      transform: [ 'trim' ],
-      maxLength: 200,
-    }),
-  ),
   tips: Type.String({
     transform: [ 'trim' ],
     maxLength: 1024,
   }),
   skipDependencies: Type.Boolean(),
+  specificVersions: Type.Optional(VersionStringArray),
   syncDownloadData: Type.Boolean(),
   // force sync immediately, only allow by admin
   force: Type.Boolean(),
@@ -125,10 +131,37 @@ export function patchAjv(ajv: any) {
       return !semver.validRange(tag);
     },
   });
+  ajv.addFormat('semver-spec', {
+    type: 'string',
+    validate: (spec: string) => {
+      try {
+        // do not support alias
+        // exp: https://unpkg.com/good@npm:cnpmcore@3.17.1/dist/app.js
+        return [ 'tag', 'version', 'range' ].includes(npa(spec).type);
+      } catch (e) {
+        return false;
+      }
+    },
+  });
   ajv.addFormat('binary-name', {
     type: 'string',
-    validate: (binaryName: string) => {
-      return !!binaryConfig[binaryName];
+    validate: (binaryName: BinaryName) => {
+      return binaryConfig[binaryName];
+    },
+  });
+  ajv.addFormat('semver-version-array', {
+    type: 'string',
+    validate: (versionStringList: string) => {
+      let versionList;
+      try {
+        versionList = JSON.parse(versionStringList);
+      } catch (error) {
+        return false;
+      }
+      if (versionList instanceof Array) {
+        return versionList.every(version => !!semver.valid(version));
+      }
+      return false;
     },
   });
 }
@@ -171,35 +204,44 @@ export const RegistryCreateOptions = Type.Object({
     maxLength: 256,
   })),
   type: Type.Enum(RegistryType),
+  authToken: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      maxLength: 256,
+    }),
+  ),
 });
 
 export const RegistryUpdateOptions = Type.Object({
-  name: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  }),
-  host: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 4096,
-  }),
-  changeStream: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 4096,
-  }),
-  userPrefix: Type.Optional(Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  })),
-  type: Type.Enum(RegistryType),
-  registryId: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  }),
+  name: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 256,
+    }),
+  ),
+  host: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 4096,
+    }),
+  ),
+  changeStream: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 4096,
+    }),
+  ),
+  type: Type.Optional(Type.Enum(RegistryType)),
+  authToken: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 256,
+    }),
+  ),
 });
 
 export const ScopeCreateOptions = Type.Object({
@@ -232,3 +274,21 @@ export const ScopeUpdateOptions = Type.Object({
     maxLength: 256,
   }),
 });
+
+export const SearchQueryOptions = Type.Object({
+  from: Type.Number({
+    transform: [ 'trim' ],
+    minimum: 0,
+    default: 0,
+  }),
+  size: Type.Number({
+    transform: [ 'trim' ],
+    minimum: 1,
+    default: 20,
+  }),
+  text: Type.String({
+    transform: [ 'trim' ],
+    minLength: 1,
+    maxLength: 256,
+  }),
+});

app/port/webauth/WebauthController.ts

@@ -22,6 +22,8 @@ import {
   verifyRegistrationResponse,
   generateAuthenticationOptions,
   verifyAuthenticationResponse,
+  VerifyRegistrationResponseOpts,
+  VerifyAuthenticationResponseOpts,
 } from '@simplewebauthn/server';
 import type { PublicKeyCredentialCreationOptionsJSON, PublicKeyCredentialRequestOptionsJSON } from '@simplewebauthn/typescript-types';
 import { LoginResultCode, WanStatusCode } from '../../common/enum/User';
@@ -44,6 +46,17 @@ type LoginPrepareResult = {
   wanCredentialAuthOption?: PublicKeyCredentialRequestOptionsJSON;
 };
 
+type LoginImplementRequest = {
+  accData: {
+    username: string;
+    password: string;
+  };
+  wanCredentialRegiData: unknown;
+  wanCredentialAuthData: unknown;
+  needUnbindWan: boolean;
+
+};
+
 const UserRule = Type.Object({
   name: Type.String({ minLength: 1, maxLength: 100 }),
   password: Type.String({ minLength: 8, maxLength: 100 }),
@@ -102,7 +115,7 @@ export class WebauthController extends MiddlewareController {
     path: '/-/v1/login/request/session/:sessionId',
     method: HTTPMethodEnum.POST,
   })
-  async loginImplement(@Context() ctx: EggContext, @HTTPParam() sessionId: string, @HTTPBody() loginImplementRequest) {
+  async loginImplement(@Context() ctx: EggContext, @HTTPParam() sessionId: string, @HTTPBody() loginImplementRequest: LoginImplementRequest) {
     ctx.tValidate(SessionRule, { sessionId });
     const sessionToken = await this.cacheAdapter.get(sessionId);
     if (typeof sessionToken !== 'string') {
@@ -123,7 +136,7 @@ export class WebauthController extends MiddlewareController {
       }
     }
 
-    const browserType = getBrowserTypeForWebauthn(ctx.headers['user-agent']);
+    const browserType = getBrowserTypeForWebauthn(ctx.headers['user-agent']) || undefined;
     const expectedChallenge = (await this.cacheAdapter.get(`${sessionId}_challenge`)) || '';
     const expectedOrigin = this.config.cnpmcore.registry;
     const expectedRPID = new URL(expectedOrigin).hostname;
@@ -139,7 +152,7 @@ export class WebauthController extends MiddlewareController {
       }
       try {
         const verification = await verifyAuthenticationResponse({
-          response: wanCredentialAuthData,
+          response: wanCredentialAuthData as VerifyAuthenticationResponseOpts['response'],
           expectedChallenge,
           expectedOrigin,
           expectedRPID,
@@ -193,7 +206,7 @@ export class WebauthController extends MiddlewareController {
       user = result.user;
       // need unbind webauthn credential
       if (needUnbindWan) {
-        await this.userService.removeWebauthnCredential(user.userId, browserType);
+        await this.userService.removeWebauthnCredential(user?.userId, browserType);
       }
     } else {
       // others: LoginResultCode.UserNotFound
@@ -215,7 +228,7 @@ export class WebauthController extends MiddlewareController {
     if (enableWebAuthn && isSupportWebAuthn && wanCredentialRegiData) {
       try {
         const verification = await verifyRegistrationResponse({
-          response: wanCredentialRegiData,
+          response: wanCredentialRegiData as VerifyRegistrationResponseOpts['response'],
           expectedChallenge,
           expectedOrigin,
           expectedRPID,
@@ -225,7 +238,7 @@ export class WebauthController extends MiddlewareController {
           const { credentialPublicKey, credentialID } = registrationInfo;
           const base64CredentialPublicKey = base64url.encode(Buffer.from(new Uint8Array(credentialPublicKey)));
           const base64CredentialID = base64url.encode(Buffer.from(new Uint8Array(credentialID)));
-          this.userService.createWebauthnCredential(user.userId, {
+          this.userService.createWebauthnCredential(user?.userId, {
             credentialId: base64CredentialID,
             publicKey: base64CredentialPublicKey,
             browserType,

app/repository/BinaryRepository.ts

@@ -15,9 +15,9 @@ export class BinaryRepository extends AbstractRepository {
     if (binary.id) {
       const model = await this.Binary.findOne({ id: binary.id });
       if (!model) return;
-      await ModelConvertor.saveEntityToModel(binary, model);
+      await ModelConvertor.saveEntityToModel<BinaryModel>(binary as unknown as Record<string, unknown>, model);
     } else {
-      const model = await ModelConvertor.convertEntityToModel(binary, this.Binary);
+      const model = await ModelConvertor.convertEntityToModel(binary as unknown as Record<string, unknown>, this.Binary);
       this.logger.info('[BinaryRepository:saveBinary:new] id: %s, binaryId: %s', model.id, model.binaryId);
     }
   }

app/repository/DistRepository.ts

@@ -13,7 +13,7 @@ export class DistRepository {
   @Inject()
   private readonly nfsAdapter: NFSAdapter;
 
-  async findPackageVersionManifest(packageId: string, version: string) {
+  async findPackageVersionManifest(packageId: string, version: string): Promise<PackageJSONType | undefined> {
     const packageVersion = await this.packageRepository.findPackageVersion(packageId, version);
     if (packageVersion) {
       const [ packageVersionJson, readme ] = await Promise.all([
@@ -27,7 +27,7 @@ export class DistRepository {
     }
   }
 
-  async findPackageAbbreviatedManifest(packageId: string, version: string) {
+  async findPackageAbbreviatedManifest(packageId: string, version: string): Promise<PackageJSONType | undefined> {
     const packageVersion = await this.packageRepository.findPackageVersion(packageId, version);
     if (packageVersion) {
       return await this.readDistBytesToJSON(packageVersion.abbreviatedDist);

app/repository/PackageRepository.ts

@@ -17,6 +17,7 @@ import type { Maintainer as MaintainerModel } from './model/Maintainer';
 import type { User as UserModel } from './model/User';
 import { User as UserEntity } from '../core/entity/User';
 import { AbstractRepository } from './AbstractRepository';
+import { BugVersionPackages } from '../core/entity/BugVersion';
 
 export type PackageManifestType = Pick<PackageJSONType, PackageJSONPickKey> & {
   _id: string;
@@ -50,7 +51,7 @@ export type PackageJSONType = CnpmcorePatchInfo & {
     url?: string;
     email?: string;
   };
-  license?: string;
+  license?: LicenseType | string;
   author?: AuthorType | string;
   contributors?: ContributorType[] | string[];
   maintainers?: ContributorType[] | string[];
@@ -63,7 +64,9 @@ export type PackageJSONType = CnpmcorePatchInfo & {
   directories?: DirectoriesType;
   repository?: RepositoryType;
   scripts?: Record<string, string>;
-  config?: Record<string, unknown>;
+  config?: {
+    'bug-versions'?: BugVersionPackages;
+  };
   dependencies?: DepInfo;
   devDependencies?: DepInfo;
   peerDependencies?: DepInfo;
@@ -95,12 +98,16 @@ export type PackageJSONType = CnpmcorePatchInfo & {
   hasInstallScript?: boolean;
   dist?: DistType;
   workspace?: string[];
+  _npmUser?: {
+    name: string;
+    email: string;
+  };
   [key: string]: unknown;
 };
 
-type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | 'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions';
+type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | 'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions' | 'contributors';
 
-type CnpmcorePatchInfo = {
+export type CnpmcorePatchInfo = {
   _cnpmcore_publish_time?: Date;
   publish_time?: number;
   _source_registry_name?: string;
@@ -117,12 +124,18 @@ type DistType = {
   [key: string]: unknown,
 };
 
-type AuthorType = {
+export type AuthorType = {
   name: string;
+  username?: string;
   email?: string;
   url?: string;
 };
 
+type LicenseType = {
+  type: string;
+  url: string;
+};
+
 type ContributorType = {
   name?: string;
   email?: string;

app/repository/PackageVersionDownloadRepository.ts

@@ -65,7 +65,7 @@ export class PackageVersionDownloadRepository extends AbstractRepository {
     }
     for (const [ date, counter ] of counters) {
       const field = `d${date}`;
-      model[field] = counter;
+      (model as unknown as Record<string, number>)[field] = counter;
     }
     await model.save();
   }

app/repository/PackageVersionRepository.ts

@@ -0,0 +1,83 @@
+import { AccessLevel, Inject, SingletonProto } from '@eggjs/tegg';
+import { PaddingSemVer } from '../core/entity/PaddingSemVer';
+import type { Package as PackageModel } from './model/Package';
+import { PackageVersion } from '../core/entity/PackageVersion';
+import type { PackageTag } from './model/PackageTag';
+import { ModelConvertor } from './util/ModelConvertor';
+import type { PackageVersion as PackageVersionModel } from './model/PackageVersion';
+import { SqlRange } from '../core/entity/SqlRange';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class PackageVersionRepository {
+  @Inject()
+  private readonly Package: typeof PackageModel;
+
+  @Inject()
+  private readonly PackageVersion: typeof PackageVersionModel;
+
+  @Inject()
+  private readonly PackageTag: typeof PackageTag;
+
+
+  async findHaveNotPaddingVersion(id?: number): Promise<PackageVersion[]> {
+    if (!id) {
+      id = await this.PackageVersion.minimum('id')
+        .where('paddingVersion is null') as number;
+    }
+    if (!id) return [];
+    const versions = await this.PackageVersion.find({ id: { $gte: id } } as object)
+      .limit(1000);
+    const versionModels = versions.map(t => ModelConvertor.convertModelToEntity(t, PackageVersion));
+    return (versionModels as any).toObject();
+  }
+
+  async fixPaddingVersion(pkgVersionId: string, paddingSemver: PaddingSemVer): Promise<void> {
+    await this.PackageVersion.update({ packageVersionId: pkgVersionId }, {
+      paddingVersion: paddingSemver.paddingVersion,
+      isPreRelease: paddingSemver.isPreRelease,
+    });
+  }
+
+  async findVersionByTag(scope: string, name: string, tag: string): Promise<string | undefined> {
+    const tags = await this.PackageTag.select('version')
+      .join(this.Package as any, 'packageTags.packageId = packages.packageId')
+      .where({
+        scope,
+        name,
+        tag,
+      } as object) as { version: string }[];
+    const tagModel = tags && tags[0];
+    return tagModel?.version;
+  }
+
+  /**
+   * if sql version not contains prerelease, find the max version
+   */
+  async findMaxSatisfyVersion(scope: string, name: string, sqlRange: SqlRange): Promise<string | undefined> {
+    const versions = await this.PackageVersion
+      .select('packageVersions.version')
+      .join(this.Package as any, 'packageVersions.packageId = packages.packageId')
+      .where({
+        'packages.scope': scope,
+        'packages.name': name,
+        ...sqlRange.condition,
+      } as object)
+      .order('packageVersions.paddingVersion', 'desc') as { version: string }[];
+    return versions?.[0]?.version;
+  }
+
+  async findSatisfyVersionsWithPrerelease(scope: string, name: string, sqlRange: SqlRange): Promise<Array<string>> {
+    const versions = await this.PackageVersion
+      .select('version')
+      .join(this.Package as any, 'packageVersions.packageId = packages.packageId')
+      .where({
+        scope,
+        name,
+        ...sqlRange.condition,
+      } as object);
+    return (versions as any).toObject()
+      .map((t: { version: string }) => t.version);
+  }
+}

app/repository/RegistryRepository.ts

@@ -38,6 +38,14 @@ export class RegistryRepository extends AbstractRepository {
     return null;
   }
 
+  async findRegistryByRegistryHost(host: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ host });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
   async saveRegistry(registry: Registry) {
     if (registry.id) {
       const model = await this.Registry.findOne({ id: registry.id });

app/repository/SearchRepository.ts

@@ -0,0 +1,48 @@
+import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
+import { SearchAdapter } from '../common/typing';
+import { AuthorType, CnpmcorePatchInfo, PackageManifestType } from './PackageRepository';
+
+export type SearchJSONPickKey = '_rev' | 'name' | 'description' | 'keywords' | 'license' | 'maintainers' | 'dist-tags' | '_source_registry_name';
+
+export type SearchMappingType = Pick<PackageManifestType, SearchJSONPickKey> & CnpmcorePatchInfo & {
+  scope: string;
+  version: string;
+  versions: string[];
+  date: Date;
+  created: Date;
+  modified: Date;
+  author?: AuthorType | undefined;
+  _npmUser?: {
+    name: string;
+    email: string;
+  }
+};
+
+
+export type SearchManifestType = {
+  package: SearchMappingType;
+  downloads: {
+    all: number;
+  };
+};
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class SearchRepository {
+  @Inject()
+  private readonly searchAdapter: SearchAdapter;
+
+
+  async searchPackage(query) {
+    return await this.searchAdapter.search<SearchManifestType>(query);
+  }
+
+  async upsertPackage(document: SearchManifestType) {
+    return await this.searchAdapter.upsert(document.package.name, document);
+  }
+
+  async removePackage(fullname: string) {
+    return await this.searchAdapter.delete(fullname);
+  }
+}

app/repository/TaskRepository.ts

@@ -67,6 +67,21 @@ export class TaskRepository extends AbstractRepository {
     await model.remove();
   }
 
+  async updateSpecificVersionsOfWaitingTask(task: TaskEntity, specificVersions?: Array<string>): Promise<void> {
+    const model = await this.Task.findOne({ id: task.id });
+    if (!model || !model.data.specificVersions) return;
+    if (specificVersions) {
+      const data = model.data;
+      const combinedVersions = Array.from(new Set(data.specificVersions.concat(specificVersions)));
+      data.specificVersions = combinedVersions;
+      await model.update({ data });
+    } else {
+      const data = model.data;
+      Reflect.deleteProperty(data, 'specificVersions');
+      await model.update({ data });
+    }
+  }
+
   async findTask(taskId: string) {
     const task = await this.Task.findOne({ taskId });
     if (task) {

app/repository/UserRepository.ts

@@ -1,6 +1,7 @@
 import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
 import type { User as UserModel } from './model/User';
+import type { Package as PackageModel } from './model/Package';
 import type { Token as TokenModel } from './model/Token';
 import type { WebauthnCredential as WebauthnCredentialModel } from './model/WebauthnCredential';
 import { User as UserEntity } from '../core/entity/User';
@@ -8,7 +9,7 @@ import { Token as TokenEntity, isGranularToken } from '../core/entity/Token';
 import { WebauthnCredential as WebauthnCredentialEntity } from '../core/entity/WebauthnCredential';
 import { AbstractRepository } from './AbstractRepository';
 import { TokenPackage as TokenPackageModel } from './model/TokenPackage';
-import { getScopeAndName } from '../common/PackageUtil';
+import { getFullname, getScopeAndName } from '../common/PackageUtil';
 import { PackageRepository } from './PackageRepository';
 
 @SingletonProto({
@@ -24,6 +25,9 @@ export class UserRepository extends AbstractRepository {
   @Inject()
   private readonly TokenPackage: typeof TokenPackageModel;
 
+  @Inject()
+  private readonly Package: typeof PackageModel;
+
   @Inject()
   private readonly packageRepository: PackageRepository;
 
@@ -58,6 +62,7 @@ export class UserRepository extends AbstractRepository {
     if (!token) return null;
     const userModel = await this.User.findOne({ userId: token.userId });
     if (!userModel) return null;
+
     return {
       token,
       user: ModelConvertor.convertModelToEntity(userModel, UserEntity),
@@ -67,7 +72,19 @@ export class UserRepository extends AbstractRepository {
   async findTokenByTokenKey(tokenKey: string) {
     const model = await this.Token.findOne({ tokenKey });
     if (!model) return null;
-    return ModelConvertor.convertModelToEntity(model, TokenEntity);
+    const token = ModelConvertor.convertModelToEntity(model, TokenEntity);
+    await this._injectTokenPackages(token);
+    return token;
+  }
+
+  private async _injectTokenPackages(token: TokenEntity) {
+    if (isGranularToken(token)) {
+      const models = await this.TokenPackage.find({ tokenId: token.tokenId });
+      const packages = await this.Package.find({ packageId: models.map(m => m.packageId) });
+      if (Array.isArray(packages)) {
+        token.allowedPackages = packages.map(p => getFullname(p.scope, p.name));
+      }
+    }
   }
 
   async saveToken(token: TokenEntity): Promise<void> {
@@ -111,7 +128,11 @@ export class UserRepository extends AbstractRepository {
 
   async listTokens(userId: string): Promise<TokenEntity[]> {
     const models = await this.Token.find({ userId });
-    return models.map(model => ModelConvertor.convertModelToEntity(model, TokenEntity));
+    const tokens = models.map(model => ModelConvertor.convertModelToEntity(model, TokenEntity));
+    for (const token of tokens) {
+      await this._injectTokenPackages(token);
+    }
+    return tokens;
   }
 
   async saveCredential(credential: WebauthnCredentialEntity): Promise<void> {
@@ -125,7 +146,7 @@ export class UserRepository extends AbstractRepository {
     }
   }
 
-  async findCredentialByUserIdAndBrowserType(userId: string, browserType: string | null) {
+  async findCredentialByUserIdAndBrowserType(userId: string | undefined, browserType: string | null) {
     const model = await this.WebauthnCredential.findOne({
       userId,
       browserType,

app/repository/model/PackageVersion.ts

@@ -1,6 +1,7 @@
 import { Attribute, Model } from '@eggjs/tegg/orm';
 import { DataTypes, Bone } from 'leoric';
 import { EntityProperty } from '../util/EntityProperty';
+import { PaddingSemVer } from '../../core/entity/PaddingSemVer';
 
 @Model()
 export class PackageVersion extends Bone {
@@ -47,4 +48,18 @@ export class PackageVersion extends Bone {
 
   @Attribute(DataTypes.DATE)
   publishTime: Date;
+
+  @Attribute(DataTypes.STRING)
+  paddingVersion: string;
+
+  @Attribute(DataTypes.BOOLEAN)
+  isPreRelease: boolean;
+
+  static beforeCreate(instance: { version: string; paddingVersion: string; isPreRelease: boolean }) {
+    if (!instance.paddingVersion) {
+      const paddingSemVer = new PaddingSemVer(instance.version);
+      instance.paddingVersion = paddingSemVer.paddingVersion;
+      instance.isPreRelease = paddingSemVer.isPreRelease;
+    }
+  }
 }

app/repository/model/Registry.ts

@@ -36,4 +36,7 @@ export class Registry extends Bone {
   @Attribute(DataTypes.STRING(256))
   type: RegistryType;
 
+  @Attribute(DataTypes.STRING(256), { name: 'auth_token' })
+  authToken?: string;
+
 }

app/repository/model/Token.ts

@@ -54,4 +54,7 @@ export class Token extends Bone {
 
   @Attribute(DataTypes.DATE)
   expiredAt: Date;
+
+  @Attribute(DataTypes.DATE)
+  lastUsedAt: Date;
 }

app/repository/package.json

@@ -2,5 +2,8 @@
   "name": "cnpmcore-repository",
   "eggModule": {
     "name": "cnpmcoreRepository"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.14.196"
   }
 }

app/repository/util/ModelConvertor.ts

@@ -8,13 +8,16 @@ const CREATED_AT = 'createdAt';
 const UPDATED_AT = 'updatedAt';
 const ID = 'id';
 
+type BonePatchInfo = { id?: bigint, updatedAt?: Date, createdAt?: Date };
+type PatchedBone = Bone & BonePatchInfo;
+
 export class ModelConvertor {
-  static async convertEntityToModel<T extends Bone>(entity: object, ModelClazz: EggProtoImplClass<T>, options?): Promise<T> {
+  static async convertEntityToModel<T extends(PatchedBone)>(entity: object, ModelClazz: EggProtoImplClass<T>, options?: object): Promise<T> {
     const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
     }
-    const attributes = {};
+    const attributes: Record<string, unknown> = {};
     for (const attributeMeta of metadata.attributes) {
       const modelPropertyName = attributeMeta.propertyName;
       const entityPropertyName = ModelConvertorUtil.getEntityPropertyName(ModelClazz, modelPropertyName);
@@ -22,17 +25,17 @@ export class ModelConvertor {
       const attributeValue = _.get(entity, entityPropertyName);
       attributes[modelPropertyName] = attributeValue;
     }
-    const model = await (ModelClazz as unknown as typeof Bone).create(attributes, options);
+    const model = await (ModelClazz as unknown as typeof Bone).create(attributes, options) as PatchedBone;
     // auto set entity id to model id
-    entity[ID] = model[ID];
+    (entity as Record<string, unknown>)[ID] = model[ID];
     // use model dates
-    entity[UPDATED_AT] = model[UPDATED_AT];
-    entity[CREATED_AT] = model[CREATED_AT];
+    (entity as Record<string, unknown>)[UPDATED_AT] = model[UPDATED_AT];
+    (entity as Record<string, unknown>)[CREATED_AT] = model[CREATED_AT];
     return model as T;
   }
 
   static convertEntityToChanges<T extends Bone>(entity: object, ModelClazz: EggProtoImplClass<T>) {
-    const changes = {};
+    const changes: Record<string, unknown> = {};
     const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
       throw new Error(`Model ${ModelClazz.name} has no metadata`);
@@ -45,13 +48,13 @@ export class ModelConvertor {
       changes[modelPropertyName] = attributeValue;
     }
     changes[UPDATED_AT] = new Date();
-    entity[UPDATED_AT] = changes[UPDATED_AT];
+    (entity as Record<string, unknown>)[UPDATED_AT] = changes[UPDATED_AT];
     return changes;
   }
 
   // TODO: options is QueryOptions, should let leoric export it to use
   // Find out which attributes changed and set `updatedAt` to now
-  static async saveEntityToModel<T extends Bone>(entity: object, model: T, options?): Promise<boolean> {
+  static async saveEntityToModel<T extends Bone>(entity: object, model: T & PatchedBone, options?: object): Promise<boolean> {
     const ModelClazz = model.constructor as EggProtoImplClass<T>;
     const metadata = ModelMetadataUtil.getModelMetadata(ModelClazz);
     if (!metadata) {
@@ -64,14 +67,14 @@ export class ModelConvertor {
       // Restricted updates to the primary key
       if (entityPropertyName === ID && model[ID]) continue;
       const attributeValue = _.get(entity, entityPropertyName);
-      model[modelPropertyName] = attributeValue;
+      (model as unknown as Record<string, unknown>)[modelPropertyName] = attributeValue;
     }
 
     // Restricted updates to the UPDATED_AT
     // Leoric will set by default
     model[UPDATED_AT] = undefined;
     await model.save(options);
-    entity[UPDATED_AT] = model[UPDATED_AT];
+    (entity as Record<string, unknown>)[UPDATED_AT] = model[UPDATED_AT];
     return true;
   }
 
@@ -85,7 +88,7 @@ export class ModelConvertor {
     for (const attributeMeta of metadata.attributes) {
       const modelPropertyName = attributeMeta.propertyName;
       const entityPropertyName = ModelConvertorUtil.getEntityPropertyName(ModelClazz as EggProtoImplClass, modelPropertyName);
-      const attributeValue = bone[attributeMeta.propertyName];
+      const attributeValue = bone[attributeMeta.propertyName as keyof Bone];
       _.set(data, entityPropertyName, attributeValue);
     }
     const model = Reflect.construct(entityClazz, [ data ]);

config/binaries.ts

@@ -871,6 +871,23 @@ const binaries = {
       },
     },
   },
+  'libpg-query-node': {
+    category: 'libpg-query-node',
+    description: 'libpg-query is a real PostgreSQL query parser',
+    type: BinaryType.NodePreGyp,
+    repo: 'pyramation/libpg-query-node',
+    distUrl: 'https://supabase-public-artifacts-bucket.s3.amazonaws.com',
+    options: {
+      npmPackageName: 'libpg-query',
+    },
+  },
+  'fuse-t': {
+    category: 'fuse-t',
+    description: 'FUSE-T is a kext-less implementation of FUSE for macOS that uses NFS v4 local server instead of a kernel extension.',
+    type: BinaryType.GitHub,
+    repo: 'macos-fuse-t/fuse-t',
+    distUrl: 'https://github.com/macos-fuse-t/fuse-t/releases',
+  },
 } as const;
 
 export type BinaryName = keyof typeof binaries;

config/config.default.ts

@@ -1,9 +1,10 @@
 import assert from 'assert';
+import { randomUUID } from 'crypto';
 import { join } from 'path';
 import { EggAppConfig, PowerPartial } from 'egg';
 import OSSClient from 'oss-cnpm';
 import { patchAjv } from '../app/port/typebox';
-import { ChangesStreamMode, SyncDeleteMode, SyncMode } from '../app/common/constants';
+import { ChangesStreamMode, NOT_IMPLEMENTED_PATH, SyncDeleteMode, SyncMode } from '../app/common/constants';
 import { CnpmcoreConfig } from '../app/port/config';
 
 export const cnpmcoreConfig: CnpmcoreConfig = {
@@ -31,7 +32,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
   checkChangesStreamInterval: 500,
   changesStreamRegistry: 'https://replicate.npmjs.com',
   changesStreamRegistryMode: ChangesStreamMode.streaming,
-  registry: 'http://localhost:7001',
+  registry: process.env.CNPMCORE_CONFIG_REGISTRY || 'http://localhost:7001',
   alwaysAuth: false,
   allowScopes: [
     '@cnpm',
@@ -43,7 +44,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
   admins: {
     cnpmcore_admin: 'admin@cnpmjs.org',
   },
-  enableWebAuthn: false,
+  enableWebAuthn: !!process.env.CNPMCORE_CONFIG_ENABLE_WEB_AUTHN,
   enableCDN: false,
   cdnCacheControlHeader: 'public, max-age=300',
   cdnVaryHeader: 'Accept, Accept-Encoding',
@@ -52,33 +53,38 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
   syncNotFound: false,
   redirectNotFound: true,
   enableUnpkg: true,
+  strictSyncSpecivicVersion: false,
+  enableElasticsearch: !!process.env.CNPMCORE_CONFIG_ENABLE_ES,
+  elasticsearchIndex: 'cnpmcore_packages',
+  strictValidateTarballPkg: false,
 };
 
 export default (appInfo: EggAppConfig) => {
   const config = {} as PowerPartial<EggAppConfig>;
 
+  config.keys = process.env.CNPMCORE_EGG_KEYS || randomUUID();
   config.cnpmcore = cnpmcoreConfig;
 
   // override config from framework / plugin
-  config.dataDir = join(appInfo.root, '.cnpmcore');
+  config.dataDir = process.env.CNPMCORE_DATA_DIR || join(appInfo.root, '.cnpmcore');
 
   config.orm = {
     client: 'mysql',
-    database: process.env.MYSQL_DATABASE || 'cnpmcore',
-    host: process.env.MYSQL_HOST || '127.0.0.1',
-    port: process.env.MYSQL_PORT || 3306,
-    user: process.env.MYSQL_USER || 'root',
-    password: process.env.MYSQL_PASSWORD,
+    database: process.env.CNPMCORE_MYSQL_DATABASE || process.env.MYSQL_DATABASE || 'cnpmcore',
+    host: process.env.CNPMCORE_MYSQL_HOST || process.env.MYSQL_HOST || '127.0.0.1',
+    port: process.env.CNPMCORE_MYSQL_PORT || process.env.MYSQL_PORT || 3306,
+    user: process.env.CNPMCORE_MYSQL_USER || process.env.MYSQL_USER || 'root',
+    password: process.env.CNPMCORE_MYSQL_PASSWORD || process.env.MYSQL_PASSWORD,
     charset: 'utf8mb4',
     logger: {},
   };
 
   config.redis = {
     client: {
-      port: 6379,
-      host: '127.0.0.1',
-      password: '',
-      db: 0,
+      port: Number(process.env.CNPMCORE_REDIS_PORT || 6379),
+      host: process.env.CNPMCORE_REDIS_HOST || '127.0.0.1',
+      password: process.env.CNPMCORE_REDIS_PASSWORD || '',
+      db: Number(process.env.CNPMCORE_REDIS_DB || 0),
     },
   };
 
@@ -94,11 +100,13 @@ export default (appInfo: EggAppConfig) => {
       return ctx.get('Origin');
     },
     credentials: true,
+    // https://github.com/koajs/cors/blob/master/index.js#L10C57-L10C64
+    allowMethods: 'GET,HEAD,PUT,POST,DELETE,PATCH,OPTIONS',
   };
 
   config.nfs = {
     client: null,
-    dir: join(config.dataDir, 'nfs'),
+    dir: process.env.CNPMCORE_NFS_DIR || join(config.dataDir, 'nfs'),
   };
   /* c8 ignore next 17 */
   // enable oss nfs store by env values
@@ -117,12 +125,41 @@ export default (appInfo: EggAppConfig) => {
         'Cache-Control': 'max-age=0, s-maxage=60',
       },
     });
+  } else if (process.env.CNPMCORE_NFS_TYPE === 's3') {
+    assert(process.env.CNPMCORE_NFS_S3_CLIENT_ENDPOINT, 'require env CNPMCORE_NFS_S3_CLIENT_ENDPOINT');
+    assert(process.env.CNPMCORE_NFS_S3_CLIENT_ID, 'require env CNPMCORE_NFS_S3_CLIENT_ID');
+    assert(process.env.CNPMCORE_NFS_S3_CLIENT_SECRET, 'require env CNPMCORE_NFS_S3_CLIENT_SECRET');
+    assert(process.env.CNPMCORE_NFS_S3_CLIENT_BUCKET, 'require env CNPMCORE_NFS_S3_CLIENT_BUCKET');
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const S3Client = require('s3-cnpmcore');
+    config.nfs.client = new S3Client({
+      region: process.env.CNPMCORE_NFS_S3_CLIENT_REGION || 'default',
+      endpoint: process.env.CNPMCORE_NFS_S3_CLIENT_ENDPOINT,
+      credentials: {
+        accessKeyId: process.env.CNPMCORE_NFS_S3_CLIENT_ID,
+        secretAccessKey: process.env.CNPMCORE_NFS_S3_CLIENT_SECRET,
+      },
+      bucket: process.env.CNPMCORE_NFS_S3_CLIENT_BUCKET,
+      forcePathStyle: !!process.env.CNPMCORE_NFS_S3_CLIENT_FORCE_PATH_STYLE,
+      disableURL: !!process.env.CNPMCORE_NFS_S3_CLIENT_DISABLE_URL,
+    });
   }
 
   config.logger = {
     enablePerformanceTimer: true,
     enableFastContextLogger: true,
+    appLogName: process.env.CNPMCORE_APP_LOG_NAME || `${appInfo.name}-web.log`,
+    coreLogName: process.env.CNPMCORE_CORE_LOG_NAME || 'egg-web.log',
+    agentLogName: process.env.CNPMCORE_AGENT_LOG_NAME || 'egg-agent.log',
+    errorLogName: process.env.CNPMCORE_ERROR_LOG_NAME || 'common-error.log',
+    outputJSON: Boolean(process.env.CNPMCORE_LOG_JSON_OUTPUT || false),
   };
+  if (process.env.CNPMCORE_LOG_DIR) {
+    config.logger.dir = process.env.CNPMCORE_LOG_DIR;
+  }
+  if (process.env.CNPMCORE_LOG_JSON_OUTPUT) {
+    config.logger.outputJSON = Boolean(process.env.CNPMCORE_LOG_JSON_OUTPUT);
+  }
 
   config.logrotator = {
     // only keep 1 days log files
@@ -134,6 +171,8 @@ export default (appInfo: EggAppConfig) => {
     strict: false,
     // set default limit to 10mb, see https://github.com/npm/npm/issues/12750
     jsonLimit: '10mb',
+    // https://github.com/cnpm/cnpmcore/issues/551
+    ignore: NOT_IMPLEMENTED_PATH,
   };
 
   // https://github.com/xiekw2010/egg-typebox-validate#%E5%A6%82%E4%BD%95%E5%86%99%E8%87%AA%E5%AE%9A%E4%B9%89%E6%A0%A1%E9%AA%8C%E8%A7%84%E5%88%99
@@ -154,5 +193,18 @@ export default (appInfo: EggAppConfig) => {
     },
   };
 
+  // more options: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html
+  if (config.cnpmcore.enableElasticsearch) {
+    config.elasticsearch = {
+      client: {
+        node: process.env.CNPMCORE_CONFIG_ES_CLIENT_NODE,
+        auth: {
+          username: process.env.CNPMCORE_CONFIG_ES_CLIENT_AUTH_USERNAME as string,
+          password: process.env.CNPMCORE_CONFIG_ES_CLIENT_AUTH_PASSWORD as string,
+        },
+      },
+    };
+  }
+
   return config;
 };

config/config.unittest.ts

@@ -1,5 +1,8 @@
 import { join } from 'path';
 import { EggAppConfig, PowerPartial } from 'egg';
+import Mock from '@elastic/elasticsearch-mock';
+
+export const mockES = new Mock();
 
 export default (appInfo: EggAppConfig) => {
   const config = {} as PowerPartial<EggAppConfig>;
@@ -16,5 +19,13 @@ export default (appInfo: EggAppConfig) => {
   config.cnpmcore = {
     checkChangesStreamInterval: 10,
   };
+
+  config.elasticsearch = {
+    client: {
+      node: 'http://localhost:9200',
+      Connection: mockES.getConnection(),
+    },
+  };
+
   return config;
 };

config/plugin.ts

@@ -56,6 +56,10 @@ const plugin: EggPlugin = {
     enable: true,
     package: 'egg-status',
   },
+  elasticsearch: {
+    enable: true,
+    package: 'eggjs-elasticsearch',
+  },
 };
 
 export default plugin;

docker-compose.yml

@@ -1,5 +1,30 @@
-version: '3.6'
+version: '3.8'
 services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - ../..:/workspaces:cached
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: sleep infinity    
+    depends_on:
+      - mysql
+      - redis
+    networks:
+      - cnpm
+
+    # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
+    # networks:
+    #   - cnpm
+
+    # Uncomment the next line to use a non-root user for all processes.
+    # user: node
+
+    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
   redis:
     image: redis:6-alpine
     # command: redis-server --appendonly yes --requirepass cnpm
@@ -7,7 +32,7 @@ services:
     volumes:
       - cnpm-redis:/data
     ports:
-      - 6379:6379
+      - 6379
     networks:
       - cnpm
 
@@ -16,7 +41,7 @@ services:
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
     restart: always
     environment:
-      MYSQL_ROOT_PASSWORD:
+      MYSQL_ROOT_PASSWORD: root
       MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
       # MYSQL_DATABASE: 'cnpmcore_unittest'
       MYSQL_USER: user
@@ -25,33 +50,30 @@ services:
       - cnpm-mysql:/var/lib/mysql
       # - ./conf.d/mysql/:/etc/mysql/conf.d
       # - ./init.d/mysql/:/docker-entrypoint-initdb.d
+      - ./sql:/docker-entrypoint-initdb.d/sql
+      - ./prepare-database.sh:/docker-entrypoint-initdb.d/init.sh
     ports:
-      - 3306:3306
+      - 3306
     networks:
       - cnpm
 
   # database explorer
-  phpmyadmin:
-    image: phpmyadmin
+  adminer:
+    image: adminer
     restart: always
     environment:
-      MYSQL_ROOT_PASSWORD:
-      MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
-      MYSQL_USER: user
-      MYSQL_PASSWORD: pass
-      PMA_HOST: 'mysql'
-    ports:
-      - 8080:80
-    networks:
-      - cnpm
+      ADMINER_DEFAULT_DB_HOST: 'mysql'  
     depends_on:
       - mysql
+    ports:
+      - 8080
+    networks:
+      - cnpm
 
 volumes:
   cnpm-redis:
   cnpm-mysql:
 
-
 networks:
   cnpm:
     name: cnpm

docs/deploy-in-docker.md

@@ -0,0 +1,175 @@
+# 通过 Docker 部署 cnpmcore
+
+## 构建镜像
+
+```bash
+docker build -t cnpmcore .
+```
+
+## 通过环境变量配置参数
+
+需要在 docker 容器中配置数据存储参数，否则启动会失败，cnpmcore 镜像要求数据存储与计算分离。
+
+### MySQL
+
+```bash
+CNPMCORE_MYSQL_DATABASE=cnpmcore
+CNPMCORE_MYSQL_HOST=127.0.0.1
+CNPMCORE_MYSQL_PORT=3306
+CNPMCORE_MYSQL_USER=your-db-user-name
+CNPMCORE_MYSQL_PASSWORD=your-db-user-password
+```
+
+### Redis
+
+```bash
+CNPMCORE_REDIS_HOST=127.0.0.1
+CNPMCORE_REDIS_PORT=6379
+CNPMCORE_REDIS_PASSWORD=your-redis-password
+CNPMCORE_REDIS_DB=1
+```
+
+### 文件存储
+
+目前支持的文件存储服务有阿里云 OSS、AWS S3，以及兼容 S3 的 minio。
+
+#### OSS
+
+```bash
+CNPMCORE_NFS_TYPE=oss
+CNPMCORE_NFS_OSS_ENDPOINT==https://your-oss-endpoint
+CNPMCORE_NFS_OSS_BUCKET=your-bucket-name
+CNPMCORE_NFS_OSS_ID=oss-ak
+CNPMCORE_NFS_OSS_SECRET=oss-sk
+```
+
+#### S3 / minio
+
+```bash
+CNPMCORE_NFS_TYPE=s3
+CNPMCORE_NFS_S3_CLIENT_ENDPOINT=https://your-s3-endpoint
+CNPMCORE_NFS_S3_CLIENT_BUCKET=your-bucket-name
+CNPMCORE_NFS_S3_CLIENT_ID=s3-ak
+CNPMCORE_NFS_S3_CLIENT_SECRET=s3-sk
+CNPMCORE_NFS_S3_CLIENT_DISABLE_URL=true
+```
+
+如果使用的是 minio，请务必设置 `CNPMCORE_NFS_S3_CLIENT_FORCE_PATH_STYLE=true`
+
+```bash
+CNPMCORE_NFS_S3_CLIENT_FORCE_PATH_STYLE=true
+```
+
+### 日志
+
+```bash
+CNPMCORE_LOG_DIR=/var/log/cnpmcore
+```
+
+### registry 域名
+
+```bash
+CNPMCORE_CONFIG_REGISTRY=https://your-registry.com
+```
+
+### 使用 `config.prod.js` 覆盖
+
+直接覆盖 `/usr/src/app/config/config.prod.js` 文件也可以实现生产配置自定义。
+
+```js
+module.exports = {
+  cnpmcore: {
+    registry: 'https://your-registry.com',
+    enableWebAuthn: true,
+  },
+  orm: {
+    database: 'cnpmcore',
+    host: '127.0.0.1',
+    port: 3306,
+    user: 'your-db-user-name',
+    password: 'your-db-user-password',
+  },
+  redis: {
+    client: {
+      port: 6379,
+      host: '127.0.0.1',
+      password: 'your-redis-password',
+      db: 1,
+    },
+  },
+  nfs: {
+    client: new (require('s3-cnpmcore'))({
+      region: 'default',
+      endpoint: 'https://your-s3-endpoint',
+      credentials: {
+        accessKeyId: 's3-ak',
+        secretAccessKey: 's3-sk',
+      },
+      bucket: 'your-bucket-name',
+      forcePathStyle: true,
+      disableURL: true,
+    }),
+  },
+  logger: {
+    dir: '/var/log/cnpmcore',
+  },
+};
+```
+
+通过 docker volumes 设置配置文件
+
+```bash
+docker run -p 7001:7001 -it --rm \
+  -v /path-to/config.prod.js:/usr/src/app/config/config.prod.js \
+  --name cnpmcore-prod cnpmcore
+```
+
+## 运行容器
+
+```bash
+docker run -p 7001:7001 -it --rm \
+  -e CNPMCORE_CONFIG_REGISTRY=https://your-registry.com \
+  -e CNPMCORE_MYSQL_DATABASE=cnpmcore \
+  -e CNPMCORE_MYSQL_HOST=127.0.0.1 \
+  -e CNPMCORE_MYSQL_PORT=3306 \
+  -e CNPMCORE_MYSQL_USER=your-db-user-name \
+  -e CNPMCORE_MYSQL_PASSWORD=your-db-user-password \
+  -e CNPMCORE_NFS_TYPE=s3 \
+  -e CNPMCORE_NFS_S3_CLIENT_ENDPOINT=https://your-s3-endpoint \
+  -e CNPMCORE_NFS_S3_CLIENT_BUCKET=your-bucket-name \
+  -e CNPMCORE_NFS_S3_CLIENT_ID=s3-ak \
+  -e CNPMCORE_NFS_S3_CLIENT_SECRET=s3-sk \
+  -e CNPMCORE_NFS_S3_CLIENT_FORCE_PATH_STYLE=true \
+  -e CNPMCORE_NFS_S3_CLIENT_DISABLE_URL=true \
+  -e CNPMCORE_REDIS_HOST=127.0.0.1 \
+  -e CNPMCORE_REDIS_PORT=6379 \
+  -e CNPMCORE_REDIS_PASSWORD=your-redis-password \
+  -e CNPMCORE_REDIS_DB=1 \
+  --name cnpmcore-prod cnpmcore
+```
+
+## 演示地址
+
+https://registry-demo.fengmk2.com:9443
+
+管理员账号：cnpmcore_admin/12345678
+
+通过 npm login 可以登录
+
+```bash
+npm login --registry=https://registry-demo.fengmk2.com:9443
+```
+
+查看当前登录用户
+
+```bash
+npm whoami --registry=https://registry-demo.fengmk2.com:9443
+```
+
+## fengmk2/cnpmcore 镜像
+
+https://hub.docker.com/r/fengmk2/cnpmcore
+
+```bash
+docker pull fengmk2/cnpmcore
+```

docs/elasticsearch-setup.md

@@ -0,0 +1,997 @@
+# 本地搭建 ES 搜索环境
+
+## 单机搭建
+### 下载安装 ES
+
+首先我们进入 [ES 下载的官方网站](https://www.elastic.co/cn/downloads/elasticsearch) ，选择合适的操作系统版本并下载。下载完成后再适当位置解压并运行
+
+```bash
+cd ~/your_path/elaticsearch-8.6.1./bin/elasticsearch
+```
+
+ES 默认的 http.port 端口为 `9200`，此时我们访问 `localhost:9200` 时会可能会抛出证书的异常。
+
+这所因为 ES 默认的自签名证书不被系统所信任。我们可以在当前命令的目录下找到其配置文件 `config.elaticsearch.yml`，在开发阶段先将其关闭。
+
+```yaml
+# Enable security features
+xpack.security.enabled: false
+```
+
+此外，为了更方便查看 ES 的数据和日志的目录，我们也将其修改为当前目录
+
+```yaml
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: ./data
+#
+# Path to log files:
+#
+path.logs: ./logs
+```
+
+再次重启 ES
+
+```bash
+./bin/elasticsearch
+```
+
+此时我们访问 [localhost:9200](http://localhost:9200)，可以看到当前 ES 集群的详细信息。
+
+### 下载安装 Kibana
+
+为了更方便的使用 ES，我们还需要再安装其可视化的数据操作和分析工具 Kibana。 ES 有对应版本的 Kibana 下载地址，这里同理进入 Kibana 的 [官方下载地址](https://www.elastic.co/cn/downloads/kibana) ，当前版本为 8.6.1。
+
+下载完成后进入 kibana-8.6.1 的文件目录并启动它
+
+```bash
+./bin/kibana
+```
+
+此时，访问 http://localhost:5601 ，即可看到 Kibana 引导页面。
+
+
+我们仅仅将其作为一个可视化的操作 API 的可视化工具，可以跳过其引导，访问 `/app/dev_tools#/console` 进入 devtool 页面。
+
+## docker compose
+
+新建文件 `docker-compose.yaml`， 复制如下的 `docker-compose.yaml`
+
+```yaml
+version: "3.8"
+
+volumes:
+  certs:
+    driver: local
+  esdata01:
+    driver: local
+  kibanadata:
+    driver: local
+
+services:
+  setup:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
+    command: >
+      bash -c '
+        if [ x${ELASTIC_PASSWORD} == x ]; then
+          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+          exit 1;
+        elif [ x${KIBANA_PASSWORD} == x ]; then
+          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+          exit 1;
+        fi;
+        if [ ! -f config/certs/ca.zip ]; then
+          echo "Creating CA";
+          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+          unzip config/certs/ca.zip -d config/certs;
+        fi;
+        if [ ! -f config/certs/certs.zip ]; then
+          echo "Creating certs";
+          echo -ne \
+          "instances:\n"\
+          "  - name: es01\n"\
+          "    dns:\n"\
+          "      - es01\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: kibana\n"\
+          "    dns:\n"\
+          "      - kibana\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          > config/certs/instances.yml;
+          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+          unzip config/certs/certs.zip -d config/certs;
+        fi;
+        echo "Setting file permissions"
+        chown -R root:root config/certs;
+        find . -type d -exec chmod 750 \{\} \;;
+        find . -type f -exec chmod 640 \{\} \;;
+        echo "Waiting for Elasticsearch availability";
+        until curl -s --cacert config/certs/ca/ca.crt http://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+        echo "Setting kibana_system password";
+        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+        echo "All done!";
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+
+  es01:
+    depends_on:
+      setup:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=false
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${ES_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt http://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  kibana:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    labels:
+      co.elastic.logs/module: kibana
+    volumes:
+      - certs:/usr/share/kibana/config/certs
+      - kibanadata:/usr/share/kibana/data
+    ports:
+      - ${KIBANA_PORT}:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=http://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+    mem_limit: ${KB_MEM_LIMIT}
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+```
+
+### 新建 .env 文件
+
+复制如下的 `.env` 文件
+
+```bash
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD="abcdef"
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD="abcdef"
+
+# Version of Elastic products
+STACK_VERSION=8.7.1
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+#ES_PORT=127.0.0.1:9200
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+#KIBANA_PORT=80
+
+# Increase or decrease based on the available host memory (in bytes)
+ES_MEM_LIMIT=1073741824
+KB_MEM_LIMIT=1073741824
+LS_MEM_LIMIT=1073741824
+
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject
+
+# SAMPLE Predefined Key only to be used in POC environments
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
+```
+
+### 启动服务
+
+执行如下命令，启动服务
+
+```bash
+$ docker compose up
+```
+
+### 访问 Elastic
+
+浏览器打开 http://localhost:5601/app/dev_tools#/console，默认账号为 `elastic` 密码为 .env 文件中定义的 `abcdef`
+
+
+## 创建索引
+
+ES 可以通过 Kibana devtool 进行数据的写入和查询操作。下面创建一个索引，`cnpmcore_packages` 为索引名称。
+
+```json
+PUT cnpmcore_packages
+{
+  "settings": ${settings} // copy 下方 settings
+  "mappings": ${mappings} // copy 下方 settings
+}
+```
+
+### settings
+
+```json
+{
+  "index": {
+    "analysis": {
+      "analyzer": {
+        "package": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_autocomplete": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "autocomplete",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_autocomplete_highlight": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "autocomplete"
+        },
+        "package_autocomplete_keyword": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "autocomplete",
+            "trim",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "keyword"
+        },
+        "package_autocomplete_keyword_search": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "keyword"
+        },
+        "package_edge_ngram": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "edge_ngram",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_english": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "kstem",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_english_aggressive": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "porter_stem",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "raw": {
+          "filter": [
+            "asciifolding",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "keyword"
+        }
+      },
+      "filter": {
+        "autocomplete": {
+          "max_gram": "15",
+          "min_gram": "1",
+          "type": "edge_ngram"
+        },
+        "edge_ngram": {
+          "max_gram": "15",
+          "min_gram": "4",
+          "type": "edge_ngram"
+        },
+        "non_alfanum_to_space": {
+          "pattern": "(?i)[^a-z0-9]+",
+          "replacement": " ",
+          "type": "pattern_replace"
+        },
+        "split_word": {
+          "catenate_all": "false",
+          "catenate_numbers": "false",
+          "catenate_words": "false",
+          "generate_number_parts": "true",
+          "generate_word_parts": "true",
+          "preserve_original": "true",
+          "split_on_case_change": "true",
+          "split_on_numerics": "true",
+          "stem_english_possessive": "true",
+          "type": "word_delimiter"
+        },
+        "unique_on_same_position": {
+          "only_on_same_position": "false",
+          "type": "unique"
+        }
+      },
+      "normalizer": {
+        "raw": {
+          "filter": [
+            "asciifolding",
+            "lowercase",
+            "trim"
+          ],
+          "type": "custom"
+        }
+      },
+      "tokenizer": {
+        "autocomplete": {
+          "max_gram": "15",
+          "min_gram": "1",
+          "token_chars": [
+            "letter",
+            "digit"
+          ],
+          "type": "edge_ngram"
+        }
+      }
+    }
+  }
+}
+```
+
+### mappings
+
+```json
+{
+  "dynamic": true,
+  "properties": {
+    "downloads": {
+      "properties": {
+        "all": {
+          "type": "long"
+        }
+      }
+    },
+    "package": {
+      "properties": {
+        "_rev": {
+          "index": false,
+          "type": "text"
+        },
+        "author": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "url": {
+              "index": false,
+              "type": "text"
+            },
+            "username": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "created": {
+          "type": "date"
+        },
+        "description": {
+          "fields": {
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "dist-tags": {
+          "dynamic": "true",
+          "enabled": false,
+          "type": "object"
+        },
+        "keywords": {
+          "fields": {
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "raw": {
+              "analyzer": "raw",
+              "type": "text"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "license": {
+          "type": "keyword"
+        },
+        "maintainers": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "username": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "modified": {
+          "type": "date"
+        },
+        "_source_registry_name": {
+          "type": "text"
+        },
+        "_npmUser": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "publish_time": {
+          "type": "long"
+        },
+        "name": {
+          "fields": {
+            "autocomplete": {
+              "analyzer": "package_autocomplete",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "autocomplete_highlight": {
+              "analyzer": "package_autocomplete_highlight",
+              "index_options": "offsets",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "autocomplete_keyword": {
+              "analyzer": "package_autocomplete_keyword",
+              "search_analyzer": "package_autocomplete_keyword_search",
+              "type": "text"
+            },
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "raw": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "scope": {
+          "normalizer": "raw",
+          "type": "keyword"
+        },
+        "versions": {
+          "index": false,
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+```
+
+### 在 kibana 操作
+
+```json
+PUT /cnpmcore_packages
+{
+  "settings": {
+  "index": {
+    "analysis": {
+      "analyzer": {
+        "package": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_autocomplete": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "autocomplete",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_autocomplete_highlight": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "autocomplete"
+        },
+        "package_autocomplete_keyword": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "autocomplete",
+            "trim",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "keyword"
+        },
+        "package_autocomplete_keyword_search": {
+          "filter": [
+            "asciifolding",
+            "non_alfanum_to_space",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "keyword"
+        },
+        "package_edge_ngram": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "edge_ngram",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_english": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "kstem",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "package_english_aggressive": {
+          "filter": [
+            "asciifolding",
+            "split_word",
+            "lowercase",
+            "porter_stem",
+            "unique_on_same_position"
+          ],
+          "tokenizer": "standard"
+        },
+        "raw": {
+          "filter": [
+            "asciifolding",
+            "lowercase",
+            "trim"
+          ],
+          "tokenizer": "keyword"
+        }
+      },
+      "filter": {
+        "autocomplete": {
+          "max_gram": "15",
+          "min_gram": "1",
+          "type": "edge_ngram"
+        },
+        "edge_ngram": {
+          "max_gram": "15",
+          "min_gram": "4",
+          "type": "edge_ngram"
+        },
+        "non_alfanum_to_space": {
+          "pattern": "(?i)[^a-z0-9]+",
+          "replacement": " ",
+          "type": "pattern_replace"
+        },
+        "split_word": {
+          "catenate_all": "false",
+          "catenate_numbers": "false",
+          "catenate_words": "false",
+          "generate_number_parts": "true",
+          "generate_word_parts": "true",
+          "preserve_original": "true",
+          "split_on_case_change": "true",
+          "split_on_numerics": "true",
+          "stem_english_possessive": "true",
+          "type": "word_delimiter"
+        },
+        "unique_on_same_position": {
+          "only_on_same_position": "false",
+          "type": "unique"
+        }
+      },
+      "normalizer": {
+        "raw": {
+          "filter": [
+            "asciifolding",
+            "lowercase",
+            "trim"
+          ],
+          "type": "custom"
+        }
+      },
+      "tokenizer": {
+        "autocomplete": {
+          "max_gram": "15",
+          "min_gram": "1",
+          "token_chars": [
+            "letter",
+            "digit"
+          ],
+          "type": "edge_ngram"
+        }
+      }
+    }
+  }
+},
+  "mappings": {
+  "dynamic": true,
+  "properties": {
+    "downloads": {
+      "properties": {
+        "all": {
+          "type": "long"
+        }
+      }
+    },
+    "package": {
+      "properties": {
+        "_rev": {
+          "index": false,
+          "type": "text"
+        },
+        "author": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "url": {
+              "index": false,
+              "type": "text"
+            },
+            "username": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "created": {
+          "type": "date"
+        },
+        "description": {
+          "fields": {
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "dist-tags": {
+          "dynamic": "true",
+          "enabled": false,
+          "type": "object"
+        },
+        "keywords": {
+          "fields": {
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "raw": {
+              "analyzer": "raw",
+              "type": "text"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "license": {
+          "type": "keyword"
+        },
+        "maintainers": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "modified": {
+          "type": "date"
+        },
+        "_source_registry_name": {
+          "type": "text"
+        },
+        "_npmUser": {
+          "properties": {
+            "email": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "name": {
+              "normalizer": "raw",
+              "type": "keyword"
+            }
+          }
+        },
+        "publish_time": {
+          "type": "long"
+        },
+        "name": {
+          "fields": {
+            "autocomplete": {
+              "analyzer": "package_autocomplete",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "autocomplete_highlight": {
+              "analyzer": "package_autocomplete_highlight",
+              "index_options": "offsets",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "autocomplete_keyword": {
+              "analyzer": "package_autocomplete_keyword",
+              "search_analyzer": "package_autocomplete_keyword_search",
+              "type": "text"
+            },
+            "edge_ngram": {
+              "analyzer": "package_edge_ngram",
+              "search_analyzer": "package",
+              "type": "text"
+            },
+            "english": {
+              "analyzer": "package_english",
+              "type": "text"
+            },
+            "english_aggressive": {
+              "analyzer": "package_english_aggressive",
+              "type": "text"
+            },
+            "raw": {
+              "normalizer": "raw",
+              "type": "keyword"
+            },
+            "standard": {
+              "analyzer": "standard",
+              "type": "text"
+            }
+          },
+          "type": "text"
+        },
+        "scope": {
+          "normalizer": "raw",
+          "type": "keyword"
+        },
+        "versions": {
+          "index": false,
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+}
+```
+
+## 开启 cnpmcore 中的 ES 服务
+
+```ts
+// config.default.ts
+config: {
+  cnpmcore: {
+    enableElasticsearch: true,
+    // 写入索引，与上述创建索引一致
+    elasticsearchIndex: 'cnpmcore_packages',
+  },
+  // elasticsearch 插件的 config，参考官方文档
+  // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/configuration.html
+  elasticsearch: {
+    client: {
+      node: 'http://localhost:9200',
+      auth: {
+        username: 'elastic',
+        password: 'abcdef',
+      },
+    },
+  };
+}
+```
+
+### 同步一条数据
+
+```bash
+$ curl -X PUT https://r.cnpmjs.org/-/v1/search/sync/${pkgName}
+```
+
+### 删除一条数据
+
+注意需要添加管理员 token，管理员在本地进行登录后，可通过查询 `~/.npmrc` 查看
+```bash
+$ curl -X DELETE -H 'Authorization: Bearer ${token}' http://localhost:7001/-/v1/search/${pkgName}
+```
+
+### 修改数据
+
+创建同步任务即可，会自动进行覆盖式同步
+
+### 查询
+
+```bash
+$ npm search colors --registry=http://localhost:7001
+```

index.d.ts

@@ -0,0 +1,113 @@
+declare module 'fs-cnpm' {
+  export default class FSClient extends NFSClient {
+    constructor(options: {
+      dir: string;
+    });
+  }
+}
+
+declare module 'ssri' {
+  export interface Integrity {
+    algorithm: string;
+    digest: string;
+    options?: string[];
+  }
+
+  export interface HashLike {
+    digest: string;
+    algorithm: string;
+    options?: string[];
+    sha1: {
+      hexDigest(): string;
+    }[];
+    sha512: { toString(): string }[];
+  }
+
+  export interface HashOptions {
+    algorithms?: string[];
+    options?: string[];
+  }
+
+  export interface IntegrityOptions {
+    algorithms?: string[];
+    options?: string[];
+    single?: boolean;
+  }
+
+  export interface CreateRes {
+    update(v: string): { digest: () => { toString() }; };
+  }
+
+  export function fromHex(hexDigest: string, algorithm: string, options?: string[]): Integrity;
+
+  export function fromData(data: Buffer | string | Uint8Array, options?: HashOptions): HashLike;
+
+  export function fromStream(stream: NodeJS.ReadableStream, options?: HashOptions): Promise<HashLike>;
+
+  export function checkData(data: Buffer | string, sri: string | Integrity, options?: IntegrityOptions): boolean;
+
+  export function checkStream(stream: NodeJS.ReadableStream, sri: string | Integrity, options?: IntegrityOptions): Promise<boolean>;
+
+  export function parse(sri: string): Integrity;
+
+  export function create(): CreateRes;
+
+  export function stringify(integrity: Integrity, options?: { strict?: boolean }): string;
+}
+
+declare module 'oss-cnpm' {
+  import { Readable } from 'stream';
+
+  export interface AppendResult {
+    name: string;
+    url: string;
+    etag: string;
+    size: number;
+  }
+
+  export interface UploadOptions {
+    key: string;
+    content: Readable;
+    size: number;
+  }
+
+  export interface UploadResult {
+    name: string;
+    url: string;
+    etag: string;
+    size: number;
+  }
+
+  export interface DownloadOptions {
+    key: string;
+  }
+
+  export default class OSSClient {
+    constructor(options: {
+      cdnBaseUrl?: string;
+      accessKeyId: string;
+      accessKeySecret: string;
+      bucket: string;
+      internal?: boolean;
+      secure?: boolean;
+      timeout?: number;
+      cname?: boolean;
+      endpoint?: string;
+      defaultHeaders?: Record<string, string>;
+    });
+
+    append(options: UploadOptions): Promise<AppendResult>;
+
+    upload(options: UploadOptions): Promise<UploadResult>;
+
+    download(options: DownloadOptions): Promise<Readable>;
+
+    delete(key: string): Promise<void>;
+
+    exists(key: string): Promise<boolean>;
+
+    stat(key: string): Promise<{ size: number }>;
+
+    url(key: string): string;
+  }
+}

module.d.ts

@@ -4,4 +4,4 @@ declare module "egg" {
   export interface EggContextModule {
     cnpmcoreCore: ContextCnpmcore;
   }
-}
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmcore",
-  "version": "3.23.2",
+  "version": "3.48.3",
   "description": "npm core",
   "files": [
     "dist/**/*"
@@ -52,6 +52,7 @@
     "tsc:prod": "npm run clean && tsc -p ./tsconfig.prod.json",
     "prepublishOnly": "npm run tsc:prod",
     "start": "eggctl start --daemon && touch egg.status",
+    "start:foreground": "eggctl start",
     "stop": "rm -f egg.status && sleep 15 && eggctl stop"
   },
   "repository": {
@@ -67,15 +68,16 @@
     "registry"
   ],
   "dependencies": {
-    "@eggjs/tegg": "^3.2.1",
-    "@eggjs/tegg-aop-plugin": "^3.2.1",
-    "@eggjs/tegg-config": "^3.1.0",
-    "@eggjs/tegg-controller-plugin": "^3.2.1",
-    "@eggjs/tegg-eventbus-plugin": "^3.2.1",
-    "@eggjs/tegg-orm-plugin": "^3.2.1",
-    "@eggjs/tegg-plugin": "^3.2.1",
-    "@eggjs/tegg-schedule-plugin": "^3.2.1",
+    "@eggjs/tegg": "^3.12.0",
+    "@eggjs/tegg-aop-plugin": "^3.12.0",
+    "@eggjs/tegg-config": "^3.12.0",
+    "@eggjs/tegg-controller-plugin": "^3.12.0",
+    "@eggjs/tegg-eventbus-plugin": "^3.12.0",
+    "@eggjs/tegg-orm-plugin": "^3.12.0",
+    "@eggjs/tegg-plugin": "^3.12.0",
+    "@eggjs/tegg-schedule-plugin": "^3.12.0",
     "@eggjs/tsconfig": "^1.0.0",
+    "@elastic/elasticsearch": "^8.8.1",
     "@node-rs/crc32": "^1.2.2",
     "@simplewebauthn/server": "^7.0.1",
     "@sinclair/typebox": "^0.23.0",
@@ -92,15 +94,18 @@
     "egg-tracer": "^1.1.0",
     "egg-typebox-validate": "^2.0.0",
     "egg-view-nunjucks": "^2.3.0",
+    "eggjs-elasticsearch": "^0.0.6",
     "fs-cnpm": "^2.4.0",
     "ioredis": "^5.3.1",
-    "leoric": "^2.6.2",
+    "leoric": "^2.11.5",
     "lodash": "^4.17.21",
     "mime-types": "^2.1.35",
     "mysql": "^2.18.1",
     "mysql2": "^2.3.0",
-    "oss-cnpm": "^4.0.0",
+    "npm-package-arg": "^10.1.0",
+    "oss-cnpm": "^5.0.1",
     "p-map": "^4.0.0",
+    "s3-cnpmcore": "^1.1.2",
     "semver": "^7.3.5",
     "ssri": "^8.0.1",
     "tar": "^6.1.13",
@@ -108,20 +113,29 @@
     "ua-parser-js": "^1.0.34",
     "validate-npm-package-name": "^3.0.0"
   },
+  "optionalDependencies": {
+    "s3-cnpmcore": "^1.1.2"
+  },
   "devDependencies": {
     "@cnpmjs/npm-cli-login": "^1.1.0",
+    "@elastic/elasticsearch-mock": "^2.0.0",
     "@simplewebauthn/typescript-types": "^7.0.0",
+    "@types/lodash": "^4.14.196",
+    "@types/mime-types": "^2.1.1",
     "@types/mocha": "^10.0.1",
     "@types/mysql": "^2.15.21",
+    "@types/npm-package-arg": "^6.1.1",
     "@types/semver": "^7.3.12",
     "@types/tar": "^6.1.4",
+    "@types/ua-parser-js": "^0.7.36",
+    "@types/validate-npm-package-name": "^4.0.0",
     "coffee": "^5.4.0",
     "egg-bin": "^6.0.0",
     "egg-mock": "^5.10.4",
     "eslint": "^8.29.0",
-    "eslint-config-egg": "^12.1.0",
+    "eslint-config-egg": "^13.0.0",
     "git-contributor": "2",
-    "typescript": "^5.0.4"
+    "typescript": "^5.2.2"
   },
   "author": "killagu",
   "license": "MIT",

sql/1.16.0.sql

@@ -0,0 +1,4 @@
+ALTER TABLE
+  `tokens`
+ADD
+  COLUMN `last_used_at` datetime(3) DEFAULT NULL COMMENT 'token last used time';

sql/3.28.0.sql

@@ -0,0 +1,8 @@
+ALTER TABLE
+  `package_versions`
+  ADD
+    COLUMN padding_version varchar(255) DEFAULT NULL COMMENT 'token name',
+  ADD
+    COLUMN `is_pre_release` tinyint(4) DEFAULT NULL COMMENT 'pre release version or not',
+  ADD
+    KEY `idx_pkg_id_is_pre_release_padding_version` (`package_id`, `padding_version`, `is_pre_release`, `version`);

sql/3.46.0.sql

@@ -0,0 +1 @@
+ALTER TABLE `registries` ADD COLUMN `auth_token` varchar(256) DEFAULT NULL COMMENT 'registry auth token';

test/TestUtil.ts

@@ -7,7 +7,7 @@ import { Readable } from 'stream';
 import mysql from 'mysql';
 import path from 'path';
 import crypto from 'crypto';
-import { getScopeAndName } from '../app/common/PackageUtil';
+import { cleanUserPrefix, getScopeAndName } from '../app/common/PackageUtil';
 import semver from 'semver';
 import { PackageJSONType } from '../app/repository/PackageRepository';
 
@@ -49,7 +49,7 @@ export class TestUtil {
       host: process.env.MYSQL_HOST || '127.0.0.1',
       port: process.env.MYSQL_PORT || 3306,
       user: process.env.MYSQL_USER || 'root',
-      password: process.env.MYSQL_PASSWORD,
+      password: process.env.MYSQL_PASSWORD || '',
       multipleStatements: true,
     };
   }
@@ -242,7 +242,7 @@ export class TestUtil {
       user.name = `testuser-${crypto.randomBytes(20).toString('hex')}`;
     }
     const password = user.password ?? 'password-is-here';
-    const email = user.email ?? `${user.name}@example.com`;
+    const email = cleanUserPrefix(user.email ?? `${user.name}@example.com`);
     let res = await this.app.httpRequest()
       .put(`/-/user/org.couchdb.user:${user.name}`)
       .send({
@@ -266,6 +266,7 @@ export class TestUtil {
     }
     return {
       name: user.name,
+      displayName: cleanUserPrefix(user.name),
       token,
       authorization: `Bearer ${token}`,
       password,