Release 3.0.0-rc.66

> 包名超长时，更新下载记录接口会无限重试
* 更新下载记录前，先判断当前包名是否存在

-------------

> When the package name is too long, the update download record
interface will retry indefinitely.

* Check pkg exists, before updating the download record.

.github/workflows/nodejs.yml

@@ -0,0 +1,52 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: true
+          MYSQL_DATABASE: cnpmjs_test
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [14, 16]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout Git Source
+      uses: actions/checkout@v2
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Dependencies
+      run: npm i -g npminstall && npminstall
+
+    - name: Continuous Integration
+      run: npm run ci
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.travis.yml

@@ -1,10 +0,0 @@
-language: node_js
-node_js:
-  - '10'
-  - '12'
-services:
-  - mysql
-  - postgresql
-script: 'make test-travis-all'
-after_script:
-  - 'npm i codecov && codecov'

Dockerfile

@@ -12,7 +12,7 @@ WORKDIR ${CNPM_DIR}
 
 COPY package.json ${CNPM_DIR}
 
-RUN npm set registry https://registry.npm.taobao.org
+RUN npm set registry https://registry.npmmirror.com
 
 RUN npm install --production
 

History.md

@@ -1,3 +1,82 @@
+
+3.0.0-rc.66 / 2023-03-25
+==================
+
+**features**
+  * [[`a289d7d`](http://github.com/cnpm/cnpmjs.org/commit/a289d7dd5ec4881e86c6f500c72cab134388782b)] - feat: ignore sync private pkg (#1747) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`af9f5f7`](http://github.com/cnpm/cnpmjs.org/commit/af9f5f749979d1f389d61e98eab7dfbf639c995b)] - feat: add changes delay (#1739) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+**fixes**
+  * [[`6769a46`](http://github.com/cnpm/cnpmjs.org/commit/6769a46d71508a24f4e2d4dc14aa4aebd826bf38)] - fix: download row create error (#1749) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+3.0.0-rc.63 / 2022-08-31
+==================
+
+**features**
+  * [[`1964d66`](http://github.com/cnpm/cnpmjs.org/commit/1964d66abf3a78bc97022db2bfd1b88fa2c1fa37)] - feat: remove block version changes (#1737) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+
+3.0.0-rc.62 / 2022-07-28
+==================
+
+**features**
+  * [[`d8b5c9f`](http://github.com/cnpm/cnpmjs.org/commit/d8b5c9f7a9a4648460415936e492ee0bb9cba9b0)] - feat: add changes api (#1734) (elrrrrrrr <<elrrrrrrr@gmail.com>>)
+  * [[`f52e9c3`](http://github.com/cnpm/cnpmjs.org/commit/f52e9c3382ef3583a9cac9be5510e11a3ab17b8c)] - feat: support sync private package from define registry (#1701) (XXBeii <<36651530+XXBeii@users.noreply.github.com>>)
+  * [[`97d501c`](http://github.com/cnpm/cnpmjs.org/commit/97d501c088aee43af30e31aef279b3036a7cc5f7)] - feat: support bug-versions on server (#1684) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`3c5bc9d`](http://github.com/cnpm/cnpmjs.org/commit/3c5bc9dc5ef46c567e718705d18edc1f57009b5e)] - feat: support package version block list (#1683) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`abc1723`](http://github.com/cnpm/cnpmjs.org/commit/abc1723bef5eb12168d084cc16916998288e879b)] - feat: support dist.integrity (#1677) (fengmk2 <<fengmk2@gmail.com>>)
+
+**fixes**
+  * [[`4d534ca`](http://github.com/cnpm/cnpmjs.org/commit/4d534cad024f08923b7d4df57d87d14fa96a975a)] - fix: listModulesByUser map error (#1731) (Ke Wu <<gemwuu@163.com>>)
+  * [[`b7bbf84`](http://github.com/cnpm/cnpmjs.org/commit/b7bbf84ea31b5c15a48edd596bbb87a72f7de595)] - fix: add missing config property for bug-versions (#1685) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`74d408d`](http://github.com/cnpm/cnpmjs.org/commit/74d408d006b1c7a08283b178e842ef98f850fcf2)] - fix: database dialect config detect error (#1503) (alsotang <<alsotang@gmail.com>>)
+  * [[`37ba629`](http://github.com/cnpm/cnpmjs.org/commit/37ba6290284f260230e6d4461311bb8739088191)] - fix: get count missing name (#1674) (Solais <<924615994@qq.com>>)
+
+**others**
+  * [[`f8bcca1`](http://github.com/cnpm/cnpmjs.org/commit/f8bcca1ea0c0ad952fffd0539b625de69b9099f0)] - 🐛 FIX: path url maybe encode (#1729) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`8663e21`](http://github.com/cnpm/cnpmjs.org/commit/8663e215f038b812b74948e28a05b704d5d7780c)] - 🐛 FIX: Should show sync button on scoped package not exists (#1728) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`8573c4a`](http://github.com/cnpm/cnpmjs.org/commit/8573c4a600077585681ac1eddcebc4b2abff4f8e)] - 📖 DOC: DEPRECATED, please use https://github.com/cnpm/cnpmcore instead (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`83497f2`](http://github.com/cnpm/cnpmjs.org/commit/83497f20521dd5016c119500aff0ccb64cf5a929)] - 👌 IMPROVE: Return libc field on abbreviated manifests (#1721) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`1e0f7e0`](http://github.com/cnpm/cnpmjs.org/commit/1e0f7e06bb7df40f1dca309b7e8e533130293ab6)] - 📖 DOC: Modify comments (#1710) (AN <<455454007@qq.com>>)
+  * [[`e682e7a`](http://github.com/cnpm/cnpmjs.org/commit/e682e7a0a39f750c61d7a2a46de862d118e61e73)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`19e5c3d`](http://github.com/cnpm/cnpmjs.org/commit/19e5c3def257f4ceab3256a1c9d615d002c5fe49)] - chore: Add license scan report and status (#1708) (fossabot <<badges@fossa.io>>)
+  * [[`9f8dca4`](http://github.com/cnpm/cnpmjs.org/commit/9f8dca4ac0292ba87c59149972f0f635bd17ec37)] - refactor: Sync exists package from cnpmcore changes stream (#1707) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`f2c4b9f`](http://github.com/cnpm/cnpmjs.org/commit/f2c4b9f1c8d0b97b0b1890ac3aec058a99f0b224)] - 📖 DOC: Use git-contributor instead (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`ac07b21`](http://github.com/cnpm/cnpmjs.org/commit/ac07b215a73f80aa11e9631d3e6c9be4c4db87a2)] - 🐛 FIX: Support new bson logId (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`559d5ba`](http://github.com/cnpm/cnpmjs.org/commit/559d5baccfc3f5fa582461856f7048927aac3f23)] - 📦 NEW: sync web support webDataRemoteRegistry (#1697) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`e5c5179`](http://github.com/cnpm/cnpmjs.org/commit/e5c5179e9e12be054aa733a5b0befbb064f32a44)] - 📦 NEW: Support set registry to new cnpmcore registry (#1696) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`ad622d5`](http://github.com/cnpm/cnpmjs.org/commit/ad622d55e384743b48e79bb6aec574a7f354ee9f)] - chore: update contributors (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`c6973a9`](http://github.com/cnpm/cnpmjs.org/commit/c6973a98cef8207b56219e1b813575f7efe13653)] - fix(db.sql): use utf8mb4 on description column (#1681) (hellojukay <<hellojukay@163.com>>)
+  * [[`f22a3e7`](http://github.com/cnpm/cnpmjs.org/commit/f22a3e7370792eef1c7b613188411af2354e59a6)] - refactor: dist tarbar url don't contains querystring (#1682) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`a49deec`](http://github.com/cnpm/cnpmjs.org/commit/a49deec3b2e1e8483801dea757c727054df34b34)] - chore: remove unused config (#1495) (alsotang <<alsotang@gmail.com>>)
+  * [[`7f0aa2a`](http://github.com/cnpm/cnpmjs.org/commit/7f0aa2ad95dedef7ec50fbebeb6df92839b42621)] - chore: update contributors (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`39cf77a`](http://github.com/cnpm/cnpmjs.org/commit/39cf77ae0f676584d93ada77c6ea61c3d044b267)] - refactor: use remote abbreviated version data (#1675) (fengmk2 <<fengmk2@gmail.com>>)
+
+3.0.0-rc.50 / 2021-11-04
+==================
+
+**features**
+  * [[`e4c3cd1`](http://github.com/cnpm/cnpmjs.org/commit/e4c3cd125ebe771ecc31b4a207ab926c13817f16)] - feat: package view add download total count (#1673) (Solais <<924615994@qq.com>>)
+  * [[`283f149`](http://github.com/cnpm/cnpmjs.org/commit/283f14976dae8ce9144d387533d1637a79c6cf12)] - feat: add missing abbreviated meta attibutes (#1668) (fengmk2 <<fengmk2@gmail.com>>)
+
+**fixes**
+  * [[`62fdbd4`](http://github.com/cnpm/cnpmjs.org/commit/62fdbd400a2259983c6793173ac8d816a732452c)] - fix: should update etag after abbreviated meta change (#1670) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`2e7354c`](http://github.com/cnpm/cnpmjs.org/commit/2e7354c647d78a41f033f2066c6eb60e270f2a6b)] - fix: allow download without scope filename (#1665) (killa <<killa123@126.com>>)
+
+3.0.0-rc.46 / 2021-09-10
+==================
+
+**fixes**
+  * [[`596bca9`](http://github.com/cnpm/cnpmjs.org/commit/596bca908ee87504dfeb3dbf463b4d9a551d9f27)] - fix: fs-cnpm on dockerize config (#1656) (wxhuang <<wxhuang1985@gmail.com>>)
+  * [[`bbae9d3`](http://github.com/cnpm/cnpmjs.org/commit/bbae9d3bc3a565a59c397a63c4301272ac4509e9)] - fix: new publish with token should add user to maintainers (#1662) (killa <<killa123@126.com>>)
+
+3.0.0-rc.45 / 2021-07-28
+==================
+
+**features**
+  * [[`d28af7f`](http://github.com/cnpm/cnpmjs.org/commit/d28af7fd7e1bf530ad2e5a73fa5a54d2caf5cf9b)] - feat: support custom tokenService (#1658) (killa <<killa123@126.com>>)
+
+**others**
+  * [[`e258241`](http://github.com/cnpm/cnpmjs.org/commit/e2582417fa06a82c2fbb4d08eaf764465683a00b)] - remove maintainers logic (#1654) (Solais <<924615994@qq.com>>)
 3.0.0-rc.44 / 2021-06-11
 ==================
 

Makefile

@@ -11,8 +11,8 @@ init-database:
 	@NODE_ENV=test node test/init_db.js
 
 init-mysql:
-	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
-	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
+	@mysql -uroot -h 127.0.0.1 --port 3306 -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@mysql -uroot -h 127.0.0.1 --port 3306 -e 'CREATE DATABASE cnpmjs_test;'
 
 init-pg:
 	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;'

README.md

@@ -1,16 +1,21 @@
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+
+------
+
 cnpmjs.org
 =======
 
 [![npm version][npm-image]][npm-url]
-[![build status][travis-image]][travis-url]
+[![Node.js CI](https://github.com/cnpm/cnpmjs.org/actions/workflows/nodejs.yml/badge.svg)](https://github.com/cnpm/cnpmjs.org/actions/workflows/nodejs.yml)
 [![Test coverage][codecov-image]][codecov-url]
 [![Known Vulnerabilities][snyk-image]][snyk-url]
 [![npm download][download-image]][download-url]
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org?ref=badge_shield)
 
 [npm-image]: http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square
 [npm-url]: http://cnpmjs.org/package/cnpmjs.org
-[travis-image]: https://img.shields.io/travis/cnpm/cnpmjs.org.svg?style=flat-square
-[travis-url]: https://travis-ci.org/cnpm/cnpmjs.org
 [codecov-image]: https://codecov.io/gh/cnpm/cnpmjs.org/branch/master/graph/badge.svg
 [codecov-url]: https://codecov.io/gh/cnpm/cnpmjs.org
 [snyk-image]: https://snyk.io/test/npm/cnpmjs.org/badge.svg?style=flat-square
@@ -30,7 +35,7 @@ Our goal is to provide a low cost maintenance, easy to use, and easy to scale so
 ## What can you do with `cnpmjs.org`?
 
 * Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
-* Build a npm mirror. (we use it to build a mirror in China: [https://npm.taobao.org/](https://npm.taobao.org/))
+* Build a npm mirror. (we use it to build a mirror in China: [https://npmmirror.com/](https://npmmirror.com/))
 * Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
 
 ## Features
@@ -66,7 +71,7 @@ you only need to change the registry in client config.
 
 * [node](http://nodejs.org) >= 8.0.0
 * Databases: only required one type
-  * [sqlite3](https://npm.taobao.org/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
+  * [sqlite3](https://npmmirror.com/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
   * [MySQL](http://dev.mysql.com/downloads/) >= 5.6.16, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
   * MariaDB
   * PostgreSQL
@@ -157,19 +162,6 @@ $docker volume inspect cnpmjsorg_cnpm-db-volume
 
 Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
 
-## Top contributors
-
-
-[![0](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/0)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/0)
-[![1](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/1)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/1)
-[![2](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/2)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/2)
-[![3](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/3)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/3)
-[![4](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/4)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/4)
-[![5](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/5)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/5)
-[![6](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/6)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/6)
-[![7](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/images/7)](https://sourcerer.io/fame/fengmk2/cnpm/cnpmjs.org/links/7)
-
-
 ## Sponsors
 
 - [![阿里云](https://static.aliyun.com/images/www-summerwind/logo.gif)](http://click.aliyun.com/m/4288/) [![阿里云云效](https://img.alicdn.com/tfs/TB116yt3fb2gK0jSZK9XXaEgFXa-106-20.png)](https://devops.aliyun.com/?channel=pd_cnpm_github) (2016.2 - now)
@@ -177,3 +169,20 @@ Tips: make sure your code is following the [node-style-guide](https://github.com
 ## License
 
 [MIT](LICENSE.txt)
+
+<!-- GITCONTRIBUTOR_START -->
+
+## Contributors
+
+|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/543405?v=4" width="100px;"/><br/><sub><b>ibigbug</b></sub>](https://github.com/ibigbug)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|
+| :---: | :---: | :---: | :---: | :---: | :---: |
+|[<img src="https://avatars.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars.githubusercontent.com/u/327019?v=4" width="100px;"/><br/><sub><b>JacksonTian</b></sub>](https://github.com/JacksonTian)<br/>|[<img src="https://avatars.githubusercontent.com/u/11251401?v=4" width="100px;"/><br/><sub><b>Solais</b></sub>](https://github.com/Solais)<br/>|[<img src="https://avatars.githubusercontent.com/u/1134761?v=4" width="100px;"/><br/><sub><b>4simple</b></sub>](https://github.com/4simple)<br/>|[<img src="https://avatars.githubusercontent.com/u/6622122?v=4" width="100px;"/><br/><sub><b>21paradox</b></sub>](https://github.com/21paradox)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1294440?v=4" width="100px;"/><br/><sub><b>albertZhang2013</b></sub>](https://github.com/albertZhang2013)<br/>|[<img src="https://avatars.githubusercontent.com/u/6111524?v=4" width="100px;"/><br/><sub><b>ankurk91</b></sub>](https://github.com/ankurk91)<br/>|[<img src="https://avatars.githubusercontent.com/u/1935436?v=4" width="100px;"/><br/><sub><b>huangbowen521</b></sub>](https://github.com/huangbowen521)<br/>|[<img src="https://avatars.githubusercontent.com/u/5365267?v=4" width="100px;"/><br/><sub><b>gwenaelp</b></sub>](https://github.com/gwenaelp)<br/>|[<img src="https://avatars.githubusercontent.com/u/324440?v=4" width="100px;"/><br/><sub><b>KidkArolis</b></sub>](https://github.com/KidkArolis)<br/>|[<img src="https://avatars.githubusercontent.com/u/922240?v=4" width="100px;"/><br/><sub><b>tq0fqeu</b></sub>](https://github.com/tq0fqeu)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1587797?v=4" width="100px;"/><br/><sub><b>limianwang</b></sub>](https://github.com/limianwang)<br/>|[<img src="https://avatars.githubusercontent.com/u/900947?v=4" width="100px;"/><br/><sub><b>LoicMahieu</b></sub>](https://github.com/LoicMahieu)<br/>|[<img src="https://avatars.githubusercontent.com/u/1614482?v=4" width="100px;"/><br/><sub><b>paulvi</b></sub>](https://github.com/paulvi)<br/>|[<img src="https://avatars.githubusercontent.com/u/36651530?v=4" width="100px;"/><br/><sub><b>XXBeii</b></sub>](https://github.com/XXBeii)<br/>|[<img src="https://avatars.githubusercontent.com/u/1422472?v=4" width="100px;"/><br/><sub><b>jpuncle</b></sub>](https://github.com/jpuncle)<br/>|[<img src="https://avatars.githubusercontent.com/u/20092391?v=4" width="100px;"/><br/><sub><b>vincentmrlau</b></sub>](https://github.com/vincentmrlau)<br/>|
+[<img src="https://avatars.githubusercontent.com/u/4470552?v=4" width="100px;"/><br/><sub><b>stoneChen</b></sub>](https://github.com/stoneChen)<br/>|[<img src="https://avatars.githubusercontent.com/u/2196373?v=4" width="100px;"/><br/><sub><b>dickeylth</b></sub>](https://github.com/dickeylth)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|[<img src="https://avatars.githubusercontent.com/u/1941756?v=4" width="100px;"/><br/><sub><b>gniavaj</b></sub>](https://github.com/gniavaj)<br/>|[<img src="https://avatars.githubusercontent.com/u/10371891?v=4" width="100px;"/><br/><sub><b>hellojukay</b></sub>](https://github.com/hellojukay)<br/>|[<img src="https://avatars.githubusercontent.com/u/5040076?v=4" width="100px;"/><br/><sub><b>liyangready</b></sub>](https://github.com/liyangready)<br/>
+
+This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sun Mar 20 2022 09:50:54 GMT+0800`.
+
+<!-- GITCONTRIBUTOR_END -->
+
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org?ref=badge_large)

common/sequelize.js

@@ -48,7 +48,7 @@ database.syncFirst = false;
 
 // add longtext for mysql
 Sequelize.LONGTEXT = DataTypes.LONGTEXT = DataTypes.TEXT;
-if (config.dialect === 'mysql') {
+if (database.dialect === 'mysql') {
   Sequelize.LONGTEXT = DataTypes.LONGTEXT = 'LONGTEXT';
 }
 

config/index.js

@@ -108,12 +108,9 @@ var config = {
   customReadmeFile: '', // you can use your custom readme file instead the cnpm one
   customFooter: '', // you can add copyright and site total script html here
   npmClientName: 'cnpm', // use `${name} install package`
-  packagePageContributorSearch: true, // package page contributor link to search, default is true
 
   // max handle number of package.json `dependencies` property
   maxDependencies: 200,
-  // backup filepath prefix
-  backupFilePrefix: '/cnpm/backup/',
 
   /**
    * database config
@@ -204,11 +201,20 @@ var config = {
   officialNpmReplicate: 'https://replicate.npmjs.com',
   cnpmRegistry: 'https://r.cnpmjs.com',
 
+  // /-/all/changes
+  // since different changes are aggregated through many tables
+  // prevent changesStream changes collisions
+  changesDelay: 5000,
+
   // sync source, upstream registry
   // If you want to directly sync from official npm's registry
   // please drop them an email first
-  sourceNpmRegistry: 'https://registry.npm.taobao.org',
-  sourceNpmWeb: 'https://npm.taobao.org',
+  sourceNpmRegistry: 'https://registry.npmmirror.com',
+  sourceNpmWeb: 'https://npmmirror.com',
+
+  // set remote registry to show web page data
+  enableWebDataRemoteRegistry: false,
+  webDataRemoteRegistry: '',
 
   // upstream registry is base on cnpm/cnpmjs.org or not
   // if your upstream is official npm registry, please turn it off
@@ -247,6 +253,20 @@ var config = {
   syncDownloadOptions: {
     // formatRedirectUrl: function (url, location)
   },
+
+  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
+  syncScope: false,
+  syncScopeInterval: '12h',
+  // scope package sync config
+  /**
+ * sync scope package from assign registry
+ * @param {Array<scope>} scopes
+ * @param {String} scope.scope scope name
+ * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
+ * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
+ */
+  syncScopeConfig: [],
+
   handleSyncRegistry: 'http://127.0.0.1:7001',
 
   // default badge subject
@@ -290,6 +310,12 @@ var config = {
   // if enable this option, must create module_abbreviated and package_readme table in database
   enableAbbreviatedMetadata: false,
 
+  // enable package or package version block list, must create package_version_blocklist table in database
+  enableBlockPackageVersion: false,
+
+  // enable bug version hotfix by https://github.com/cnpm/bug-versions
+  enableBugVersion: false,
+
   // global hook function: function* (envelope) {}
   // envelope format please see https://github.com/npm/registry/blob/master/docs/hooks/hooks-payload.md#payload
   globalHook: null,
@@ -337,6 +363,9 @@ if (process.env.NODE_ENV === 'test') {
       yield next;
     };
   });
+
+  config.enableBlockPackageVersion = true;
+  config.enableBugVersion = true;
 }
 
 if (process.env.NODE_ENV !== 'test') {

controllers/registry/package/changes.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+var lodash = require('lodash');
+var gather = require('co-gather');
+
+// GET /-/_changes?since={timestamp}&limit={number}&cursorId={number}
+// List changes since the timestamp
+// Similar with https://registry.npmmirror.com/_changes?since=1658974943840
+// Change types:
+// 1. ✅ PACKAGE_VERSION_ADDED
+// 2. ✅ PACKAGE_TAG_ADDED
+// 3. 🆕 PACKAGE_UNPUBLISHED
+// 5. ❎ PACKAGE_MAINTAINER_REMOVED
+// 6. ❎ PACKAGE_MAINTAINER_CHANGED
+// 7. ❎ PACKAGE_TAG_CHANGED
+//
+// Since we don't have the previous data,
+// We can't compute the reliable seqId
+// use gmt_modified cinstead of seqId
+module.exports = function* listSince() {
+  var query = this.query;
+  var since = query.since;
+  var limit = Number(query.limit);
+
+  // ensure limit
+  if (Number.isNaN(limit)) {
+    limit = 1000;
+  }
+  var queryResults = yield gather(
+    [
+      "listVersionSince",
+      "listTagSince",
+      "listUnpublishedModuleSince",
+    ].map(function (method) {
+      return packageService[method](since, limit);
+    })
+  );
+
+  var validResults = queryResults.map(function (result) {
+    if (!result.isError) {
+      return result.value;
+    }
+    return [];
+  });
+
+  var results = lodash.orderBy(
+    lodash.flatten(validResults).filter(Boolean),
+    "gmt_modified",
+    "asc"
+  ).slice(0, limit);
+  this.body = { results };
+};

controllers/registry/package/download.js

@@ -18,6 +18,13 @@ let globalDownloads = new Map();
 module.exports = function* download(next) {
   var name = this.params.name || this.params[0];
   var filename = this.params.filename || this.params[1];
+  // scope pkg and download with out scope
+  if (name.startsWith('@') && !filename.startsWith('@')) {
+    var scope = name.slice(0, name.indexOf('/'));
+    // fix filename with scope
+    filename = `${scope}/${filename}`;
+  }
+
   var version = filename.slice(name.length + 1, -4);
   // can not get dist
   var url = null;
@@ -123,9 +130,12 @@ defer.setInterval(function* () {
         err.message += '; name: ' + name + ', count: ' + count + ', date: ' + date;
         logger.error(err);
       }
-      // save back to globalDownloads, try again next time
-      count = (globalDownloads.get(name) || 0) + count;
-      globalDownloads.set(name, count);
+      var pkgExist = yield packageService.getModuleLastModified(name);
+      if (pkgExist) {
+        // save back to globalDownloads, try again next time
+        count = (globalDownloads.get(name) || 0) + count;
+        globalDownloads.set(name, count);
+      }
     }
   }
   saving = false;

controllers/registry/package/list.js

@@ -3,6 +3,8 @@
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
 var utility = require('utility');
 var packageService = require('../../../services/package');
+var blocklistService = require('../../../services/blocklist');
+var bugVersionService = require('../../../services/bug_version');
 var common = require('../../../lib/common');
 var SyncModuleWorker = require('../../sync_module_worker');
 var config = require('../../../config');
@@ -15,33 +17,33 @@ function etag(objs) {
   return 'W/"' + utility.md5(JSON.stringify(objs)) + '"';
 }
 
+function filterBlockVerions(rows, blocks) {
+  if (!blocks) {
+    return rows;
+  }
+  return rows.filter(row => !blocks[row.version]);
+}
+
 /**
  * list all version of a module
  * GET /:name
  */
 module.exports = function* list() {
   const name = this.params.name || this.params[0];
-  // sync request will contain this query params
-  let noCache = this.query.cache === '0';
-  if (!noCache) {
-    const ua = this.headers['user-agent'] || '';
-    // old sync client will request with these user-agent
-    if (ua.indexOf('npm_service.cnpmjs.org/') !== -1) {
-      noCache = true;
-    }
-  }
+  const isSyncWorkerRequest = common.isSyncWorkerRequest(this);
   const isJSONPRequest = this.query.callback;
-  let cacheKey;
+  let cacheKey = '';
   let needAbbreviatedMeta = false;
   let abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
   if (config.enableAbbreviatedMetadata && this.accepts([ 'json', abbreviatedMetaType ]) === abbreviatedMetaType) {
     needAbbreviatedMeta = true;
-    if (cache && !isJSONPRequest) {
+    // don't cache result on sync request
+    if (cache && !isJSONPRequest && !isSyncWorkerRequest) {
       cacheKey = `list-${name}-v1`;
     }
   }
-
-  if (cacheKey && !noCache) {
+  
+  if (cacheKey) {
     const values = yield cache.hmget(cacheKey, 'etag', 'body');
     if (values && values[0] && values[1]) {
       this.body = values[1];
@@ -63,9 +65,22 @@ module.exports = function* list() {
   var rs = yield [
     packageService.getModuleLastModified(name),
     packageService.listModuleTags(name),
+    blocklistService.findBlockPackageVersions(name),
   ];
   var modifiedTime = rs[0];
   var tags = rs[1];
+  var blocks = rs[2];
+
+  if (blocks && blocks['*']) {
+    this.status = 451;
+    const error = `[block] package was blocked, reason: ${blocks['*'].reason}`;
+    this.jsonp = {
+      name,
+      error,
+      reason: error,
+    };
+    return;
+  }
 
   debug('show %s, last modified: %s, tags: %j', name, modifiedTime, tags);
   if (modifiedTime) {
@@ -89,14 +104,16 @@ module.exports = function* list() {
 
   if (needAbbreviatedMeta) {
     var rows = yield packageService.listModuleAbbreviatedsByName(name);
+    rows = filterBlockVerions(rows, blocks);
     if (rows.length > 0) {
-      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows, cacheKey);
+      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows, cacheKey, isSyncWorkerRequest);
       return;
     }
     var fullRows = yield packageService.listModulesByName(name);
+    fullRows = filterBlockVerions(fullRows, blocks);
     if (fullRows.length > 0) {
       // no abbreviated meta rows, use the full meta convert to abbreviated meta
-      yield handleAbbreviatedMetaRequestWithFullMeta(this, name, modifiedTime, tags, fullRows);
+      yield handleAbbreviatedMetaRequestWithFullMeta(this, name, modifiedTime, tags, fullRows, isSyncWorkerRequest);
       return;
     }
   }
@@ -106,7 +123,7 @@ module.exports = function* list() {
     packageService.listStarUserNames(name),
     packageService.listMaintainers(name),
   ];
-  var rows = r[0];
+  var rows = filterBlockVerions(r[0], blocks);
   var starUsers = r[1];
   var maintainers = r[2];
 
@@ -205,6 +222,9 @@ module.exports = function* list() {
       createdTime = t;
     }
   }
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
+  }
 
   if (modifiedTime && createdTime) {
     var ts = {
@@ -280,8 +300,9 @@ module.exports = function* list() {
   }
 };
 
-function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cacheKey) {
-  debug('show %s got %d rows, %d tags, modifiedTime: %s', name, rows.length, tags.length, modifiedTime);
+function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cacheKey, isSyncWorkerRequest) {
+  debug('show %s got %d rows, %d tags, modifiedTime: %s, cacheKey: %s, isSyncWorkerRequest: %s',
+    name, rows.length, tags.length, modifiedTime, cacheKey, isSyncWorkerRequest);
   const isJSONPRequest = ctx.query.callback;
   var latestMod = null;
   // set tags
@@ -307,6 +328,14 @@ function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cach
     if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
       latestMod = row;
     }
+    // abbreviatedMeta row maybe update by syncer on missing attributes add
+    if (!modifiedTime || row.gmt_modified > modifiedTime) {
+      modifiedTime = row.gmt_modified;
+    }
+  }
+  // don't use bug-versions hotfix on sync request
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
   }
 
   if (!latestMod) {
@@ -361,9 +390,9 @@ function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cach
   }
 }
 
-function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags, rows) {
-  debug('show %s got %d rows, %d tags',
-    name, rows.length, tags.length);
+function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags, rows, isSyncWorkerRequest) {
+  debug('show %s got %d rows, %d tags, isSyncWorkerRequest: %s',
+    name, rows.length, tags.length, isSyncWorkerRequest);
   var latestMod = null;
   // set tags
   var distTags = {};
@@ -382,6 +411,13 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
       continue;
     }
     // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+    var hasInstallScript;
+    if (row.package.scripts) {
+      // https://www.npmjs.com/package/fix-has-install-script
+      if (row.package.scripts.install || row.package.scripts.preinstall || row.package.scripts.postinstall) {
+        hasInstallScript = true;
+      }
+    }
     var pkg = {
       name: row.package.name,
       version: row.package.version,
@@ -391,22 +427,32 @@ function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags
       devDependencies: row.package.devDependencies,
       bundleDependencies: row.package.bundleDependencies,
       peerDependencies: row.package.peerDependencies,
+      peerDependenciesMeta: row.package.peerDependenciesMeta,
       bin: row.package.bin,
+      os: row.package.os,
+      cpu: row.package.cpu,
+      libc: row.package.libc,
       directories: row.package.directories,
       dist: row.package.dist,
       engines: row.package.engines,
+      workspaces: row.package.workspaces,
       _hasShrinkwrap: row.package._hasShrinkwrap,
+      hasInstallScript: hasInstallScript,
       publish_time: row.package.publish_time || row.publish_time,
     };
     common.setDownloadURL(pkg, ctx);
 
     versions[pkg.version] = pkg;
+    row.package = pkg;
     allVersionString += pkg.version + ',';
 
     if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
       latestMod = row;
     }
   }
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
+  }
 
   if (!latestMod) {
     latestMod = rows[0];

controllers/registry/package/save.js

@@ -1,9 +1,10 @@
 'use strict';
 
 var debug = require('debug')('cnpmjs.org:controllers:registry:package:save');
-var crypto = require('crypto');
+var ssri = require('ssri');
 var deprecateVersions = require('./deprecate');
 var packageService = require('../../../services/package');
+var logger = require('../../../common/logger');
 var common = require('../../../lib/common');
 var nfs = require('../../../common/nfs');
 var config = require('../../../config');
@@ -16,14 +17,42 @@ var hook = require('../../../services/hook');
 //
 // new flows: only one request
 // PUT /:name
-// https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
+// old publish: https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
+// new publish: https://github.com/npm/libnpmpublish/blob/main/publish.js#L91
 module.exports = function* save(next) {
-  // 'dist-tags': { latest: '0.0.2' },
-  //  _attachments:
-  // { 'nae-sandbox-0.0.2.tgz':
-  //    { content_type: 'application/octet-stream',
-  //      data: 'H4sIAAAAA
-  //      length: 9883
+  // {
+  //   "_id": "@cnpm/foo",
+  //   "name": "@cnpm/foo",
+  //   "dist-tags": {
+  //     "latest": "1.0.0"
+  //   },
+  //   "versions": {
+  //     "1.0.0": {
+  //       "name": "@cnpm/foo",
+  //       "version": "1.0.0",
+  //       "dependencies": {
+  //         "xprofiler": "^1.2.6"
+  //       },
+  //       "readme": "ERROR: No README data found!",
+  //       "_id": "@cnpm/foo@1.0.0",
+  //       "_nodeVersion": "16.13.0",
+  //       "_npmVersion": "8.1.0",
+  //       "dist": {
+  //         "integrity": "sha512-7nm0vpDEWs7y+tTwlxd7YnGaBc+9Gk5KaPsx2cqQz6H84ndBXlw5nMxGtL4Uy0bCQIknPAZAVe+KNheInmmJrQ==",
+  //         "shasum": "afd05dcfb8759b9b1c7151492a04f2254365c602",
+  //         "tarball": "http://127.0.0.1:7001/@cnpm/foo/-/@cnpm/foo-1.0.0.tgz"
+  //       }
+  //     }
+  //   },
+  //   "access": null,
+  //   "_attachments": {
+  //     "@cnpm/foo-1.0.0.tgz": {
+  //       "content_type": "application/octet-stream",
+  //       "data": "H4sIAAAAAA...",
+  //       "length": 208
+  //     }
+  //   }
+  // }
   var pkg = this.request.body;
   var username = this.user.name;
   var name = this.params.name || this.params[0];
@@ -78,8 +107,8 @@ module.exports = function* save(next) {
   var versionPackage = pkg.versions[version];
   var maintainers = versionPackage.maintainers;
 
+  var authorizeType = common.getAuthorizeType(this);
   if (!maintainers) {
-    var authorizeType = common.getAuthorizeType(this);
     if (authorizeType === common.AuthorizeType.BEARER) {
       // With the token mode, pub lib with no maintainers
       // make the maintainer to be puber
@@ -102,11 +131,22 @@ module.exports = function* save(next) {
   // notice that admins can not publish to all modules
   // (but admins can add self to maintainers first)
 
-  // make sure user in auth is in maintainers
-  // should never happened in normal request
   var m = maintainers.filter(function (maintainer) {
     return maintainer.name === username;
   });
+
+  // package.json has maintainers and publisher in not in the list
+  if (authorizeType === common.AuthorizeType.BEARER && m.length === 0) {
+    var publisher = {
+      name: this.user.name,
+      email: this.user.email,
+    };
+    m = [ publisher ];
+    maintainers.push(publisher);
+  }
+
+  // make sure user in auth is in maintainers
+  // should never happened in normal request
   if (m.length === 0) {
     this.status = 403;
     const error = '[maintainers_error] ' + username + ' does not in maintainer list';
@@ -139,7 +179,6 @@ module.exports = function* save(next) {
     username, name, version, attachment.length, versionPackage.maintainers, distTags);
 
   var exists = yield packageService.getModule(name, version);
-  var shasum;
   if (exists) {
     this.status = 403;
     const error = '[forbidden] cannot modify pre-existing version: ' + version;
@@ -175,21 +214,60 @@ module.exports = function* save(next) {
     }
   }
 
-  shasum = crypto.createHash('sha1');
-  shasum.update(tarballBuffer);
-  shasum = shasum.digest('hex');
+  var originDist = versionPackage.dist || {};
+  var shasum;
+  var integrity = originDist.integrity;
+  // for content security reason
+  // check integrity
+  if (integrity) {
+    var algorithm = ssri.checkData(tarballBuffer, integrity);
+    if (!algorithm) {
+      logger.error('[registry:save:integrity:invalid] %s@%s, dist:%j', name, version, originDist);
+      this.status = 400;
+      const error = '[invalid] dist.integrity invalid';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+    var integrityObj = ssri.fromData(tarballBuffer, {
+      algorithms: ['sha1'],
+    });
+    shasum = integrityObj.sha1[0].hexDigest();
+  } else {
+    var integrityObj = ssri.fromData(tarballBuffer, {
+      algorithms: ['sha512', 'sha1'],
+    });
+    integrity = integrityObj.sha512[0].toString();
+    shasum = integrityObj.sha1[0].hexDigest();
+    if (originDist.shasum && originDist.shasum !== shasum) {
+      // if integrity not exists, check shasum
+      logger.error('[registry:save:shasum:invalid] %s@%s, dist:%j', name, version, originDist);
+      this.status = 400;
+      const error = '[invalid] dist.shasum invalid';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+  }
 
   var options = {
     key: common.getCDNKey(name, filename),
-    shasum: shasum
+    shasum: shasum,
+    integrity: integrity,
   };
   var uploadResult = yield nfs.uploadBuffer(tarballBuffer, options);
-  debug('upload %j', uploadResult);
+  debug('upload %j, options: %j', uploadResult, options);
 
-  var dist = {
+  var dist = Object.assign({}, originDist, {
+    tarball: '',
+    integrity: integrity,
     shasum: shasum,
-    size: attachment.length
-  };
+    size: attachment.length,
+  });
 
   // if nfs upload return a key, record it
   if (uploadResult.url) {

controllers/registry/token/create.js

@@ -44,7 +44,9 @@ module.exports = function* createToken() {
     return;
   }
 
-  var token = yield tokenService.createToken(this.user.name, {
+  var tokenServiceImpl = this.tokenService || tokenService;
+
+  var token = yield tokenServiceImpl.createToken(this.user.name, {
     readonly: !!readonly,
     cidrWhitelist: cidrWhitelist || [],
   });

controllers/sync.js

@@ -2,6 +2,7 @@
 
 var debug = require('debug')('cnpmjs.org:controllers:sync');
 var Log = require('../services/module_log');
+var npmService = require('../services/npm');
 var SyncModuleWorker = require('./sync_module_worker');
 var config = require('../config');
 
@@ -20,7 +21,7 @@ exports.sync = function* () {
   var noDep = this.query.nodeps === 'true';
   var syncUpstreamFirst = this.query.sync_upstream === 'true';
   var syncFromBackupFile = this.query.sync_from_backup === 'true';
-  if (!config.sourceNpmRegistryIsCNpm) {
+  if (!config.enableWebDataRemoteRegistry && !config.sourceNpmRegistryIsCNpm) {
     syncUpstreamFirst = false;
   }
   debug('sync %s with query: %j, syncUpstreamFirst: %s', name, this.query, syncUpstreamFirst);
@@ -52,6 +53,55 @@ exports.sync = function* () {
   };
 };
 
+exports.scopeSync = function* () {
+  var scope = this.params.scope;
+
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    this.status = 404;
+    this.body = {
+      error: 'no_scope',
+      reason: 'only has syncScopeConfig config can use this feature'
+    };
+    return;
+  }
+
+  var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
+  var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
+  var packages = yield npmService.getScopePackagesShort(scope, scopeCnpmWeb)
+
+  debug('scopeSync %s with query: %j', scope, this.query);
+
+  var packageSyncWorkers = []
+
+  for (let i = 0; i < packages.length; i++) {
+    packageSyncWorkers.push(function* () {
+      var name = packages[i]
+      var logId = yield SyncModuleWorker.sync(name, 'admin', {
+        type: 'package',
+        publish: true,
+        noDep: true,
+        syncUpstreamFirst: false,
+        syncPrivatePackage: { [scope]: scopeCnpmRegistry }
+      })
+      return { name: name, logId: logId }
+    })
+  }
+
+  var logIds = yield packageSyncWorkers
+
+  debug('scopeSync %s got log id %j', scope, logIds);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logIds: logIds
+  };
+};
+
 exports.getSyncLog = function* (next) {
   var logId = Number(this.params.id || this.params[1]);
   var offset = Number(this.query.offset) || 0;

controllers/sync_module_worker.js

@@ -49,6 +49,8 @@ function SyncModuleWorker(options) {
   this.names = options.name;
   this.startName = this.names[0];
 
+  this.syncPrivatePackage = options.syncPrivatePackage;
+
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
   this._publish = options.publish === true; // _publish_on_cnpm
@@ -218,18 +220,24 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
 };
 
 SyncModuleWorker.prototype.syncUpstream = function* (name) {
-  if (config.sourceNpmRegistry.indexOf('registry.npmjs.org') >= 0 ||
-      config.sourceNpmRegistry.indexOf('registry.npmjs.com') >= 0 ||
-      config.sourceNpmRegistry.indexOf('replicate.npmjs.com') >= 0) {
+  var sourceNpmRegistry = config.sourceNpmRegistry;
+  if (config.enableWebDataRemoteRegistry) {
+    sourceNpmRegistry = config.webDataRemoteRegistry;
+  }
+
+  if (sourceNpmRegistry.indexOf('registry.npmjs.org') >= 0 ||
+      sourceNpmRegistry.indexOf('registry.npmjs.com') >= 0 ||
+      sourceNpmRegistry.indexOf('replicate.npmjs.com') >= 0) {
     this.log('----------------- upstream is npm registry: %s, ignore it -------------------',
-      config.sourceNpmRegistry);
+      sourceNpmRegistry);
     return;
   }
   var syncname = name;
   if (this.type === 'user') {
     syncname = this.type + ':' + syncname;
   }
-  var url = config.sourceNpmRegistry + '/' + syncname + '/sync?sync_upstream=true';
+
+  var url = sourceNpmRegistry + '/' + syncname + '/sync?sync_upstream=true';
   if (this.noDep) {
     url += '&nodeps=true';
   }
@@ -248,7 +256,7 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
       url, r.status, r.data);
   }
 
-  var logURL = config.sourceNpmRegistry + '/' + name + '/sync/log/' + r.data.logId;
+  var logURL = sourceNpmRegistry + '/' + name + '/sync/log/' + r.data.logId;
   var offset = 0;
   this.log('----------------- Syncing upstream %s -------------------', logURL);
 
@@ -313,14 +321,21 @@ SyncModuleWorker.prototype.next = function* (concurrencyId) {
     return setImmediate(this.finish.bind(this));
   }
 
-  if (config.syncModel === 'none') {
+  const defineRegistry = this.getPrivatePackageDefineRegistry(name)
+
+  if (!defineRegistry && config.syncModel === 'none') {
     this.log('[c#%d] [%s] syncModel is none, ignore',
-      concurrencyId, name);
+     concurrencyId, name);
     return this.finish();
   }
 
-  // try to sync from official replicate when source npm registry is not cnpmjs.org
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  // try to sync from official replicate when no defineRegistry or source npm registry is not cnpmjs.org
+  let registry
+  if (defineRegistry) {
+    registry = defineRegistry
+  } else {
+    registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;
+  }
 
   yield this.syncByName(concurrencyId, name, registry);
 };
@@ -520,8 +535,9 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
 
   this.log('----------------- Syncing %s -------------------', name);
 
+  const isNeedSyncPrivatePackage = this.getPrivatePackageDefineRegistry(name)
   // ignore private scoped package
-  if (common.isPrivateScopedPackage(name)) {
+  if (!isNeedSyncPrivatePackage && common.isPrivateScopedPackage(name)) {
     this.log('[c#%d] [%s] ignore sync private scoped %j package',
       concurrencyId, name, config.scopes);
     yield this._doneOne(concurrencyId, name, true);
@@ -530,7 +546,7 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
 
   let realRegistry = registry;
   // get from npm, don't cache
-  const packageUrl = '/' + name.replace('/', '%2f') + '?sync_timestamp=' + Date.now();
+  const packageUrl = '/' + name.replace('/', '%2f') + '?cache=0&sync_timestamp=' + Date.now();
   try {
     var result = yield npmSerivce.request(packageUrl, { registry: registry });
     pkg = result.data;
@@ -687,6 +703,21 @@ SyncModuleWorker.prototype.syncByName = function* (concurrencyId, name, registry
   return versions;
 };
 
+SyncModuleWorker.prototype.getPrivatePackageDefineRegistry = function (name) {
+  if (typeof name !== 'string') return false
+  return this.syncPrivatePackage && this.syncPrivatePackage[name.split('/')[0]]
+}
+
+SyncModuleWorker.prototype.isLocalModule = function (mods) {
+  var res = common.isLocalModule(mods)
+  if (!this.syncPrivatePackage) return res
+  if (!mods[0] || !mods[0].package || !mods[0].package.name) return res
+
+  if (this.getPrivatePackageDefineRegistry(mods[0].package.name)) return false
+
+  return res
+}
+
 function* _listStarUsers(modName) {
   var users = yield packageService.listStarUserNames(modName);
   var userMap = {};
@@ -727,7 +758,7 @@ SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
   var mods = yield packageService.listModulesByName(name);
   this.log('  [%s] start unpublished %d versions from local cnpm registry',
     name, mods.length);
-  if (common.isLocalModule(mods)) {
+  if (this.isLocalModule(mods)) {
     // publish on cnpm, dont sync this version package
     this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -795,7 +826,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var existsNpmMaintainers = result[3];
   var existsModuleAbbreviateds = result[4];
 
-  if (common.isLocalModule(moduleRows)) {
+  if (this.isLocalModule(moduleRows)) {
     // publish on cnpm, dont sync this version package
     that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
     return [];
@@ -841,6 +872,8 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
 
   // get package AbbreviatedMetadata
   var remoteAbbreviatedMetadatas = {};
+  // store remote abbreviated versions
+  var remoteAbbreviatedVersionsMap = {};
   if (config.enableAbbreviatedMetadata) {
     // use ?cache=0 tell registry dont use cache result
     var packageUrl = '/' + name.replace('/', '%2f') + '?cache=0&sync_timestamp=' + Date.now();
@@ -860,11 +893,32 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
           name, err, result.headers, result.data);
       }
       if (data) {
-        var versions = data && data.versions || {};
-        for (var version in versions) {
-          const item = versions[version];
-          if (item && typeof item._hasShrinkwrap === 'boolean') {
-            remoteAbbreviatedMetadatas[version] = { _hasShrinkwrap: item._hasShrinkwrap };
+        remoteAbbreviatedVersionsMap = data && data.versions || {};
+        for (var version in remoteAbbreviatedVersionsMap) {
+          const item = remoteAbbreviatedVersionsMap[version];
+          if (!item) {
+            continue;
+          }
+          let hasMetaData = false;
+          const metaData = {};
+          // _hasShrinkwrap maybe undefined, dont change it
+          if (typeof item._hasShrinkwrap === 'boolean') {
+            hasMetaData = true;
+            metaData._hasShrinkwrap = item._hasShrinkwrap;
+          }
+
+          // https://github.com/cnpm/cnpmjs.org/issues/1667
+          const metaDataKeys = [
+            'peerDependenciesMeta', 'os', 'cpu', 'libc', 'workspaces', 'hasInstallScript',
+          ];
+          for (const key of metaDataKeys) {
+            if (key in item) {
+              hasMetaData = true;
+              metaData[key] = item[key];
+            }
+          }
+          if (hasMetaData) {
+            remoteAbbreviatedMetadatas[version] = metaData;
           }
         }
       }
@@ -884,7 +938,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var diffNpmMaintainers = [];
 
   // [
-  //   { name, version, _hasShrinkwrap }
+  //   { name, version, _hasShrinkwrap(boolean), peerDependenciesMeta(object), os(array), cpu(array), workspaces(array) }
   // ]
   var missingAbbreviatedMetadatas = [];
   // [
@@ -1056,16 +1110,32 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
           });
           changedVersions[v] = 1;
         }
+
         // find missing abbreviatedMetadata
         if (abbreviatedMetadata) {
           for (var key in abbreviatedMetadata) {
-            if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
-              missingAbbreviatedMetadatas.push(Object.assign({
-                id: exists.id,
-                name: exists.package.name,
-                version: exists.package.version,
-              }, abbreviatedMetadata));
-              break;
+            const value = abbreviatedMetadata[key];
+            // boolean: _hasShrinkwrap, hasInstallScript
+            if ((key === '_hasShrinkwrap' || key === 'hasInstallScript') && typeof value === 'boolean') {
+              if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
+                missingAbbreviatedMetadatas.push(Object.assign({
+                  id: exists.id,
+                  name: exists.package.name,
+                  version: exists.package.version,
+                }, abbreviatedMetadata));
+                break;
+              }
+            } else if (Array.isArray(value) || (typeof value === 'object' && value)) {
+              // array: os, cpu, workspaces, libc
+              // object: peerDependenciesMeta
+              if (existsModuleAbbreviated && !(key in existsModuleAbbreviated.package)) {
+                missingAbbreviatedMetadatas.push(Object.assign({
+                  id: exists.id,
+                  name: exists.package.name,
+                  version: exists.package.version,
+                }, abbreviatedMetadata));
+                break;
+              }
             }
           }
         }
@@ -1137,7 +1207,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     var tries = 3;
     while (true) {
       try {
-        yield that._syncOneVersion(index, syncModule);
+        yield that._syncOneVersion(index, syncModule, remoteAbbreviatedVersionsMap[syncModule.version]);
         syncedVersionNames.push(syncModule.version);
         break;
       } catch (err) {
@@ -1288,7 +1358,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
     that.log('  [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length);
 
     var res = yield gather(missingModuleAbbreviateds.map(function (item) {
-      return packageService.saveModuleAbbreviated(item);
+      return packageService.saveModuleAbbreviated(item, remoteAbbreviatedVersionsMap[item.version]);
     }));
 
     for (var i = 0; i < res.length; i++) {
@@ -1507,7 +1577,7 @@ SyncModuleWorker.prototype._sync = function* (name, pkg) {
   return syncedVersionNames;
 };
 
-SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
+SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage, remoteAbbreviatedVersion) {
   var delay = Date.now() - sourcePackage.publish_time;
   logger.syncInfo('[sync_module_worker] delay: %s ms, publish_time: %s, start sync %s@%s',
     delay, utility.logDate(new Date(sourcePackage.publish_time)),
@@ -1516,6 +1586,12 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
   var username = this.username;
   var downurl = sourcePackage.dist.tarball;
   var urlobj = urlparse(downurl);
+  // let cnpmjs.org registry know this request send from sync worker
+  if (downurl.indexOf('?') > 0) {
+    downurl = `${downurl}&cache=0&sync_timestamp=${Date.now()}`;
+  } else {
+    downurl = `${downurl}?cache=0&sync_timestamp=${Date.now()}`;
+  }
   var filename = path.basename(urlobj.pathname);
   var filepath = common.getTarballFilepath(sourcePackage.name, sourcePackage.version, filename);
   var ws = fs.createWriteStream(filepath);
@@ -1644,7 +1720,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       throw err;
     }
     logger.syncInfo('[sync_module_worker] uploaded, saving %j to database', result);
-    var r = yield afterUpload(result);
+    var r = yield afterUpload(result, remoteAbbreviatedVersion);
     logger.syncInfo('[sync_module_worker] sync %s@%s done!',
       sourcePackage.name, sourcePackage.version);
     return r;
@@ -1653,7 +1729,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     fs.unlink(filepath, utility.noop);
   }
 
-  function *afterUpload(result) {
+  function *afterUpload(result, remoteAbbreviatedVersion) {
     //make sure sync module have the correct author info
     //only if can not get maintainers, use the username
     var author = username;
@@ -1680,11 +1756,13 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
       mod.package._publish_on_cnpm = true;
     }
 
-    var dist = {
+    // keep orgin dist fields, like `integrity` and so on
+    var dist = Object.assign({}, sourcePackage.dist, {
+      tarball: '',
       shasum: shasum,
       size: dataSize,
       noattachment: dataSize === 0,
-    };
+    });
 
     if (result.url) {
       dist.tarball = result.url;
@@ -1697,7 +1775,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     var r = yield packageService.saveModule(mod);
     var moduleAbbreviatedId = null;
     if (config.enableAbbreviatedMetadata) {
-      var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod);
+      var moduleAbbreviatedResult = yield packageService.saveModuleAbbreviated(mod, remoteAbbreviatedVersion);
       moduleAbbreviatedId = moduleAbbreviatedResult.id;
     }
 
@@ -1862,6 +1940,7 @@ SyncModuleWorker.sync = function* (name, username, options) {
     publish: options.publish,
     syncUpstreamFirst: options.syncUpstreamFirst,
     syncFromBackupFile: options.syncFromBackupFile,
+    syncPrivatePackage: options.syncPrivatePackage
   });
   worker.start();
   return result.id;

controllers/utils.js

@@ -69,6 +69,7 @@ exports.getDownloadTotal = function* (name) {
     lastday: 0,
     lastweek: 0,
     lastmonth: 0,
+    total: 0,
   };
 
   for (var i = 0; i < rows.length; i++) {
@@ -93,6 +94,9 @@ exports.getDownloadTotal = function* (name) {
       download.lastmonth += r.count;
     }
   }
+  if (name) {
+    download.total = yield downloadTotalService.getTotalByName(name);
+  }
   return download;
 };
 
@@ -155,3 +159,10 @@ exports.getOssLicenseUrlFromName = function (name) {
   return licenseMap[name.toLowerCase()] ?
     base + licenseMap[name.toLowerCase()] : base + name;
 };
+
+exports.ensureSinceIsDate = function(since) {
+  if (!(since instanceof Date)) {
+    return new Date(Number(since));
+  }
+  return since;
+}

controllers/web/package/show.js

@@ -12,9 +12,14 @@ var utils = require('../../utils');
 var setDownloadURL = require('../../../lib/common').setDownloadURL;
 var renderMarkdown = require('../../../common/markdown').render;
 var packageService = require('../../../services/package');
+var blocklistService = require('../../../services/blocklist');
 var downloadTotalService = require('../../../services/download_total');
+var showWithRemote = require('./showWithRemote');
 
 module.exports = function* show(next) {
+  if (config.enableWebDataRemoteRegistry) {
+    return yield showWithRemote(this, next);
+  }
   var params = this.params;
   // normal: {name: $name, version: $version}
   // scope: [$name, $version]
@@ -71,6 +76,16 @@ module.exports = function* show(next) {
     return yield next;
   }
 
+  var blocks = yield blocklistService.findBlockPackageVersions(name);
+  if (blocks) {
+    var block = blocks['*'] || blocks[pkg.version];
+    if (block) {
+      this.status = 451;
+      this.body = `[block] package@${pkg.version} was blocked, reason: ${block.reason}`;
+      return;
+    }
+  }
+
   var r = yield [
     utils.getDownloadTotal(name),
     packageService.listDependents(name),

controllers/web/package/showWithRemote.js

@@ -0,0 +1,97 @@
+'use strict';
+
+const debug = require('debug')('cnpmjs.org:controllers:web:package:showWithRemote');
+const moment = require('moment');
+const gravatar = require('gravatar');
+const giturl = require('giturl');
+const urllib = require('../../../common/urllib');
+const config = require('../../../config');
+const renderMarkdown = require('../../../common/markdown').render;
+
+module.exports = function* showWithRemote(ctx, next) {
+  const params = ctx.params;
+  const fullname = params.name || params[0];
+  const versionOrTag = params.version || params[1] || 'latest';
+  debug('display %s with %j', fullname, params);
+
+  const url = `${config.webDataRemoteRegistry}/${fullname}`;
+  const result = yield urllib.request(url, {
+    dataType: 'json',
+    timeout: 20000,
+    followRedirect: true,
+    gzip: true,
+  });
+  if (result.status !== 200) {
+    return yield next;
+  }
+  const manifest = result.data;
+
+  const distTags = manifest['dist-tags'] || {};
+  const realVersion = distTags[versionOrTag] || versionOrTag;
+  const versionsMap = manifest.versions || {};
+  const pkg = versionsMap[realVersion];
+  if (!pkg) {
+    return yield next;
+  }
+
+  const maintainers = manifest.maintainers;
+  if (maintainers) {
+    for (const maintainer of maintainers) {
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
+      }
+    }
+    pkg.maintainers = maintainers;
+  }
+
+  const timeMap = manifest.time || {};
+  
+  pkg.readme = manifest.readme || '';
+  if (typeof pkg.readme !== 'string') {
+    pkg.readme = 'readme is not string: ' + JSON.stringify(pkg.readme);
+  } else {
+    pkg.readme = renderMarkdown(pkg.readme);
+  }
+  
+  pkg.fromNow = moment(new Date(timeMap[pkg.version])).fromNow();
+  // [ {tag, version, fromNow} ]
+  const tags = [];
+  for (const tag in distTags) {
+    const version = distTags[tag];
+    const time = timeMap[version];
+    if (time) {
+      const fromNow = moment(new Date(time)).fromNow();
+      tags.push({ tag, version, fromNow });
+    }
+  }
+  pkg.tags = tags;
+  // [ {version, deprecated, fromNow} ]
+  const versions = [];
+  for (const version in versionsMap) {
+    const item = versionsMap[version];
+    versions.push({
+      version,
+      deprecated: item.deprecated,
+      fromNow: moment(new Date(timeMap[version])).fromNow(),
+    });
+  }
+  pkg.versions = versions;
+
+  pkg.registryUrl = '//' + config.registryHost + '/' + pkg.name;
+  pkg.registryPackageUrl = '//' + config.registryHost + '/' + pkg.name + '/' + pkg.version;
+
+  if (pkg.repository === 'undefined') {
+    pkg.repository = null;
+  }
+  if (pkg.repository && pkg.repository.url) {
+    if (!pkg.repository.weburl) {
+      pkg.repository.weburl = /^https?:\/\//.test(pkg.repository.url) ? pkg.repository.url : (giturl.parse(pkg.repository.url) || pkg.repository.url);
+    }
+  }
+
+  yield ctx.render('package', {
+    title: 'Package - ' + manifest.name,
+    package: pkg,
+    download: null,
+  });
+};

controllers/web/show_scope_sync.js

@@ -0,0 +1,23 @@
+'use strict';
+var config = require('../../config');
+var npmService = require('../../services/npm');
+
+module.exports = function* showScopeSync () {
+  var scope = this.params.scope;
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    return this.redirect('/');
+  }
+
+  var packages = yield npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
+
+  yield this.render('scope_sync', {
+    packages: packages,
+    scope: scopeConfig.scope,
+    sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
+    title: 'Sync Scope Packages',
+  });
+};

controllers/web/show_sync.js

@@ -1,19 +1,6 @@
-/**!
- * cnpmjs.org - controllers/web/show_sync.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+var config = require('../../config');
 
 module.exports = function* showSync() {
   var name = this.params.name || this.params[0] || this.query.name;
@@ -26,9 +13,11 @@ module.exports = function* showSync() {
     name = splits[1];
     type = splits[0];
   }
+  var syncTaskUrl = config.enableWebDataRemoteRegistry ? `${config.webDataRemoteRegistry}/${name}/sync` : null;
   yield this.render('sync', {
     type: type,
     name: name,
     title: 'Sync ' + type + ' - ' + name,
+    syncTaskUrl,
   });
 };

dispatch.js

@@ -7,20 +7,21 @@ var cfork = require('cfork');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
 var syncPath = path.join(__dirname, 'sync');
+var scopeSyncPath = path.join(__dirname, 'sync/sync_scope');
 
 console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s',
   config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel);
 
 if (config.enableCluster) {
   forkWorker();
-  if (config.syncModel !== 'none') {
-    forkSyncer();
-  }
+  config.syncModel !== 'none' && forkSyncer();
+  // sync assign private scope package
+  config.syncScope && forkScopeSyncer();
 } else {
   require(workerPath);
-  if (config.syncModel !== 'none') {
-    require(syncPath);
-  }
+  config.syncModel !== 'none' && require(syncPath);
+  // sync assign private scope package
+  config.syncScope && require(scopeSyncPath);
 }
 
 function forkWorker() {
@@ -52,3 +53,15 @@ function forkSyncer() {
     setTimeout(forkSyncer, 1000);
   });
 }
+
+function forkScopeSyncer() {
+  var syncer = childProcess.fork(scopeSyncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkScopeSyncer, 1000);
+  });
+}

docs/db.sql

@@ -82,8 +82,8 @@ CREATE TABLE IF NOT EXISTS `module` (
  `author` varchar(100) NOT NULL COMMENT 'module author',
  `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
- `description` longtext COMMENT 'module description',
- `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
+ `description` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'module description',
+ `package` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'package.json',
  `dist_shasum` varchar(100) DEFAULT NULL COMMENT 'module dist SHASUM',
  `dist_tarball` varchar(2048) DEFAULT NULL COMMENT 'module dist tarball',
  `dist_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'module dist size',
@@ -333,3 +333,14 @@ CREATE TABLE IF NOT EXISTS `token` (
  UNIQUE KEY `uk_token` (`token`),
  KEY `idx_user` (`user_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='token info';
+
+CREATE TABLE IF NOT EXISTS `package_version_blocklist` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime(3) NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time',
+  `name` varchar(214) NOT NULL COMMENT 'package name',
+  `version` varchar(30) NOT NULL COMMENT 'package version, "*" meaning all versions',
+  `reason` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL COMMENT 'block reason',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_version` (`name`, `version`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='package version block list';

docs/dockerize/config.js

@@ -7,7 +7,7 @@ var fs = require('fs');
 var os = require('os');
 
 var version = require('../package.json').version;
-
+var Nfs = require('fs-cnpm');
 var root = path.dirname(__dirname);
 var dataDir = process.env.CNPM_DATA_DIR ;
 
@@ -86,12 +86,9 @@ var config = {
   customReadmeFile: '', // you can use your custom readme file instead the cnpm one
   customFooter: '', // you can add copyright and site total script html here
   npmClientName: 'cnpm', // use `${name} install package`
-  packagePageContributorSearch: true, // package page contributor link to search, default is true
 
   // max handle number of package.json `dependencies` property
   maxDependencies: 200,
-  // backup filepath prefix
-  backupFilePrefix: '/cnpm/backup/',
 
   /**
    * database config
@@ -128,7 +125,7 @@ var config = {
   },
 
   // package tarball store in local filesystem by default
-  nfs: require('fs-cnpm')({
+  nfs: new Nfs({
     dir: path.join(dataDir, 'nfs')
   }),
   // if set true, will 302 redirect to `nfs.url(dist.key)`
@@ -168,7 +165,7 @@ var config = {
   // sync source, upstream registry
   // If you want to directly sync from official npm's registry
   // please drop them an email first
-  sourceNpmRegistry: 'https://registry.npm.taobao.org',
+  sourceNpmRegistry: 'https://registry.npmmirror.com',
 
   // upstream registry is base on cnpm/cnpmjs.org or not
   // if your upstream is official npm registry, please turn it off

docs/network.puml

@@ -1,6 +1,6 @@
 @startuml
 
-title cnpmjs.org, npm.taobao.org Network
+title cnpmjs.org, npmmirror.com Network
 
 node "China User" {
   [cnpm cli]
@@ -11,8 +11,8 @@ node "OSS: aliyuncs.com" {
 }
 
 node "SLB: 114.55.80.225 Hangzhou" {
-  [npm.taobao.org]
-  [registry.npm.taobao.org]
+  [npmmirror.com]
+  [registry.npmmirror.com]
 }
 
 node "npmtaobao3: 121.41.*" {
@@ -38,7 +38,7 @@ node "cnpm6: 47.88.189.193 Singapore" {
 }
 
 node "Aliyun CDN" {
-  [cdn.npm.taobao.org]
+  [oss.npmmirror.com]
 }
 
 node "qiniu CDN" {
@@ -59,17 +59,17 @@ node "npmjs.com" {
   [www.npmjs.com]
 }
 
-[cnpm cli] -down-> [registry.npm.taobao.org]: Read, Write
-[cnpm cli] -down-> [cdn.npm.taobao.org]: Read tgz
-[cnpm cli] -down-> [npm.taobao.org]: "Read /mirrors"
+[cnpm cli] -down-> [registry.npmmirror.com]: Read, Write
+[cnpm cli] -down-> [oss.npmmirror.com]: Read tgz
+[cnpm cli] -down-> [npmmirror.com]: "Read /mirrors"
 
-[registry.npm.taobao.org] -down-> [nginx 5]
+[registry.npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [registry 5]
-[npm.taobao.org] -down-> [nginx 5]
+[npmmirror.com] -down-> [nginx 5]
 [nginx 5] -down-> [web 5]
-[registry.npm.taobao.org] -down-> [nginx 6]
+[registry.npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [registry 6]
-[npm.taobao.org] -down-> [nginx 6]
+[npmmirror.com] -down-> [nginx 6]
 [nginx 6] -down-> [web 6]
 
 [registry 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
@@ -92,7 +92,7 @@ node "npmjs.com" {
 [syncer 7] -down-> [cnpmjs.up.qiniu.com]: Write
 [syncer 7] -down-> [registry.npmjs.com]: Read
 
-[cdn.npm.taobao.org] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
+[oss.npmmirror.com] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
 [dn-cnpm.qbox.me] -down-> [cnpmjs.up.qiniu.com]: Read
 
 @enduml

docs/web/readme.md

@@ -7,7 +7,7 @@ So `cnpm` is meaning: **Company npm**.
 - Our public registry: [r.cnpmjs.org](https://r.cnpmjs.org), syncing from [registry.npmjs.com](https://registry.npmjs.com)
 - [cnpmjs.org](/) version: <span id="app-version"></span>
 - [Node.js](https://nodejs.org) version: <span id="node-version"></span>
-- For developers in China, please visit [the China mirror](https://npm.taobao.org). 中国用户请访问[国内镜像站点](https://npm.taobao.org)。
+- For developers in China, please visit [the China mirror](https://npmmirror.com). 中国用户请访问[国内镜像站点](https://npmmirror.com)。
 - Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
 
 <div class="ant-table">
@@ -81,21 +81,21 @@ Badge URL: `https://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-bad
 use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
 
 ```bash
-$ npm install -g cnpm --registry=https://registry.npm.taobao.org
+$ npm install -g cnpm --registry=https://registry.npmmirror.com
 ```
 
 Or you can alias NPM to use it:
 
 ```bash
-alias cnpm="npm --registry=https://registry.npm.taobao.org \
+alias cnpm="npm --registry=https://registry.npmmirror.com \
 --cache=$HOME/.npm/.cache/cnpm \
---disturl=https://npm.taobao.org/mirrors/node \
+--disturl=https://npmmirror.com/mirrors/node \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npm.taobao.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npmmirror.com \
   --cache=$HOME/.npm/.cache/cnpm \
-  --disturl=https://npm.taobao.org/mirrors/node \
+  --disturl=https://npmmirror.com/mirrors/node \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
@@ -118,7 +118,7 @@ $ cnpm sync connect
 sync package on web: [sync/connect](/sync/connect)
 
 ```bash
-$ open http://registry.npm.taobao.org/sync/connect
+$ open http://registry.npmmirror.com/sync/connect
 ```
 
 ### publish / unpublish
@@ -146,7 +146,7 @@ $ cnpm info cnpm
 
 Release [History](/history).
 
-## npmjs.org, cnpmjs.org and npm.taobao.org relation
+## npmjs.org, cnpmjs.org and npmmirror.com relation
 
 ![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png)
 

lib/common.js

@@ -73,10 +73,6 @@ exports.setDownloadURL = function (pkg, ctx, host) {
     pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
       protocol,
       host, pkg.name, pkg.name, pkg.version);
-    if (ctx.querystring) {
-      var backupUrl = pkg.dist.tarball;
-      pkg.dist.tarball += '?' + ctx.querystring + '&other_urls=' + encodeURIComponent(backupUrl);
-    }
   }
 };
 
@@ -101,6 +97,9 @@ exports.isMaintainer = function (user, maintainers) {
 exports.isLocalModule = function (mods) {
   for (var i = 0; i < mods.length; i++) {
     var r = mods[i];
+    if (config.privatePackages.includes(r.name)) {
+      return true;
+    }
     if (r.package && r.package._publish_on_cnpm) {
       return true;
     }
@@ -132,3 +131,16 @@ exports.getAuthorizeType = function (ctx) {
     return AuthorizeType.BEARER;
   }
 };
+
+exports.isSyncWorkerRequest = function (ctx) {
+  // sync request will contain this query params
+  let isSyncWorkerRequest = ctx.query.cache === '0';
+  if (!isSyncWorkerRequest) {
+    const ua = ctx.headers['user-agent'] || '';
+    // old sync client will request with these user-agent
+    if (ua.indexOf('npm_service.cnpmjs.org/') !== -1) {
+      isSyncWorkerRequest = true;
+    }
+  }
+  return isSyncWorkerRequest;
+};

middleware/sync_by_install.js

@@ -1,17 +1,5 @@
-/**
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var config = require('../config');
 
 /**

middleware/web_not_found.js

@@ -23,7 +23,15 @@ module.exports = function* notFound(next) {
   }
 
   // package not found
-  m = /\/package\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  m = /^\/package\/([\w\-\_\.]+)\/?$/.exec(this.path);
+  if (!m) {
+    // scoped packages
+    m = /^\/package\/(@[\w\-\.]+\/[\w\-\.]+)$/.exec(this.path);
+    // maybe encode url: /package/%40foo%2Fawdawda
+    if (!m && this.path.startsWith('/package/%40')) {
+      m = /^\/package\/(@[\w\-\.]+\/[\w\-\.]+)$/.exec(decodeURIComponent(this.path));
+    }
+  }
   var name = null;
   var title = '404: Page Not Found';
   if (m) {

models/block_package_version.js

@@ -0,0 +1,29 @@
+'use strict';
+
+module.exports = function (sequelize, DataTypes) {
+  return sequelize.define('BlockPackageVersion', {
+    name: {
+      type: DataTypes.STRING(214),
+      allowNull: false,
+      comment: 'package name'
+    },
+    version: {
+      type: DataTypes.STRING(30),
+      allowNull: false,
+      comment: 'package version'
+    },
+    reason: {
+      type: DataTypes.LONGTEXT,
+      comment: 'block reason',
+    },
+  }, {
+    tableName: 'package_version_blocklist',
+    comment: 'package version block list',
+    indexes: [
+      {
+        unique: true,
+        fields: ['name', 'version'],
+      },
+    ],
+  });
+};

models/download_total.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - models/download_total.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 // CREATE TABLE IF NOT EXISTS `downloads` (
 //  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
 //  `gmt_create` datetime NOT NULL COMMENT 'create time',

models/index.js

@@ -10,6 +10,7 @@ function load(name) {
 
 var _ModuleAbbreviated = config.enableAbbreviatedMetadata ? load('module_abbreviated') : null;
 var _PackageReadme = config.enableAbbreviatedMetadata ? load('package_readme') : null;
+var _BlockPackageVersion = config.enableBlockPackageVersion ? load('block_package_version') : null;
 
 module.exports = {
   sequelize: sequelize,
@@ -60,4 +61,14 @@ module.exports = {
     }
     return _PackageReadme;
   },
+
+  get BlockPackageVersion() {
+    if (!config.enableBlockPackageVersion) {
+      return null;
+    }
+    if (!_BlockPackageVersion) {
+      _BlockPackageVersion = load('block_package_version');
+    }
+    return _BlockPackageVersion;
+  }
 };

models/total.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - models/total.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 // CREATE TABLE IF NOT EXISTS `total` (
 //  `name` varchar(214) NOT NULL COMMENT 'total name',
 //  `gmt_modified` datetime NOT NULL COMMENT 'modified time',

package.json

@@ -1,15 +1,17 @@
 {
   "name": "cnpmjs.org",
-  "version": "3.0.0-rc.44",
+  "version": "3.0.0-rc.66",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
+    "contributor": "git-contributor",
     "dev": "DEBUG=cnpm* node dispatch.js",
-    "test": "make jshint && make test",
+    "test": "make test",
     "test-local": "make test",
     "start": "./bin/nodejsctl start && cp History.md docs/web/history.md",
     "status": "./bin/nodejsctl status",
-    "stop": "./bin/nodejsctl stop"
+    "stop": "./bin/nodejsctl stop",
+    "ci": "make test-travis-mysql"
   },
   "bin": {
     "cnpmjs.org": "bin/cli.js"
@@ -52,6 +54,7 @@
     "koa-rewrite": "^1.1.2",
     "koa-rt": "^1.0.0",
     "koa-safe-jsonp": "^0.3.1",
+    "lodash": "^4.17.21",
     "markdown-it": "^8.3.2",
     "mime": "^1.3.6",
     "mini-logger": "^1.1.3",
@@ -64,6 +67,7 @@
     "rimraf": "^2.6.2",
     "semver": "^5.4.1",
     "sequelize": "^3.23.4",
+    "ssri": "^8.0.1",
     "thunkify-wrap": "^1.0.4",
     "treekill": "^1.0.0",
     "tunnel-agent": "^0.6.0",
@@ -75,7 +79,7 @@
   "devDependencies": {
     "autod": "*",
     "chunkstream": "*",
-    "contributors": "*",
+    "git-contributor": "^1.0.10",
     "intelli-espower-loader": "^1.0.1",
     "istanbul": "*",
     "jshint": "*",
@@ -113,9 +117,17 @@
   "engines": {
     "node": ">= 8.0.0"
   },
-  "author": [
-    "fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)",
-    "dead_horse <dead_horse@qq.com> (http://deadhorse.me)"
-  ],
+  "ci": {
+    "type": "github",
+    "os": {
+      "github": "linux, macos"
+    },
+    "npminstall": true,
+    "version": "10, 12",
+    "command": {
+      "github": "ci"
+    },
+    "services": "mysql"
+  },
   "license": "MIT"
 }

routes/registry.js

@@ -14,6 +14,7 @@ var listAll = require('../controllers/registry/package/list_all');
 var listAllPackageVersions = require('../controllers/registry/package/list_versions');
 var listShorts = require('../controllers/registry/package/list_shorts');
 var listSince = require('../controllers/registry/package/list_since');
+var changes = require('../controllers/registry/package/changes');
 var listAllVersions = require('../controllers/registry/package/list');
 var listDependents = require('../controllers/registry/package/list_dependents');
 var getOneVersion = require('../controllers/registry/package/show');
@@ -50,10 +51,10 @@ function routes(app) {
   }
 
   app.get('/', jsonp, showTotal);
-
   // before /:name/:version
   // get all modules, for npm search
   app.get('/-/all', listAll);
+  app.get('/-/all/changes', changes);
   app.get('/-/all/since', listSince);
   // get all module names, for auto completion
   app.get('/-/short', listShorts);
@@ -88,8 +89,10 @@ function routes(app) {
 
   // need limit by ip
   app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/download\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, downloadPackage);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/download\/([\w\-\.]+)$/, limit, downloadPackage);
   app.get('/:name/download/:filename', limit, downloadPackage);
   app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-\/(@[\w\-\.]+\/[\w\-\.]+)$/, limit, downloadPackage);
+  app.get(/^\/(@[\w\-\.]+\/[\w\-\.]+)\/\-\/([\w\-\.]+)$/, limit, downloadPackage);
   app.get('/:name/-/:filename', limit, downloadPackage);
 
   // delete tarball and remove one version

routes/web.js

@@ -5,6 +5,7 @@ var searchPackage = require('../controllers/web/package/search');
 var searchRange = require('../controllers/web/package/search_range');
 var listPrivates = require('../controllers/web/package/list_privates');
 var showSync = require('../controllers/web/show_sync');
+var showScopeSync = require('../controllers/web/show_scope_sync');
 var showUser = require('../controllers/web/user/show');
 var sync = require('../controllers/sync');
 var showTotal = require('../controllers/total');
@@ -35,6 +36,9 @@ function routes(app) {
   app.put(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)$/, sync.sync);
   app.put('/sync/:name', sync.sync);
 
+  app.get('/scopeSync/:scope', showScopeSync);
+  app.put('/scopeSync/:scope', sync.scopeSync);
+
   app.get(/\/sync\/(@[\w\-\.]+\/[\w\-\.]+)\/log\/(\d+)$/, sync.getSyncLog);
   app.get('/sync/:name/log/:id', sync.getSyncLog);
 

services/blocklist.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const BlockPackageVersion = require('../models').BlockPackageVersion;
+
+exports.blockPackageVersion = function* (name, version, reason) {
+  const row = yield BlockPackageVersion.findOne({ where: { name, version } });
+  if (row) {
+    row.reason = reason;
+    yield row.save();
+  } else {
+    yield BlockPackageVersion.create({ name, version, reason });
+  }
+};
+
+exports.findBlockPackageVersions = function* (name) {
+  if (!BlockPackageVersion) {
+    return null;
+  }
+  const rows = yield BlockPackageVersion.findAll({ where: { name } });
+  if (rows.length === 0) {
+    return null;
+  }
+  const blocks = {};
+  for (const row of rows) {
+    blocks[row.version] = row;
+  }
+  return blocks;
+};

services/bug_version.js

@@ -0,0 +1,50 @@
+'use strict';
+
+const config = require('../config');
+const packageService = require('./package');
+
+// replace bug version and set deprecated property for cli tooltip
+exports.hotfix = function* (rows) {
+  if (!config.enableBugVersion) {
+    return;
+  }
+  let row = rows[0];
+  if (!row) {
+    return;
+  }
+  // https://github.com/cnpm/bug-versions/blob/master/package.json#L118
+  // "config": {
+  //   "bug-versions": {
+  //     "gifsicle": {
+  //       "5.3.0": {
+  //         "version": "5.2.1",
+  //         "reason": "https://github.com/imagemin/gifsicle-bin/issues/133"
+  //       }
+  //     },
+  const moduleRow = yield packageService.getLatestModule('bug-versions');
+  if (!moduleRow) {
+    return;
+  }
+  const bugVersions = moduleRow.package.config && moduleRow.package.config['bug-versions'];
+  const bugs = bugVersions && bugVersions[row.package.name];
+  if (!bugs) {
+    return;
+  }
+
+  const existsVerionsMap = {};
+  for (row of rows) {
+    existsVerionsMap[row.package.version] = row.package;
+  }
+
+  for (row of rows) {
+    const bug = bugs[row.package.version];
+    if (bug && bug.version && existsVerionsMap[bug.version]) {
+      const packageJSON = JSON.parse(JSON.stringify(existsVerionsMap[bug.version]));
+      const hotfixDeprecated = `[WARNING] Use ${bug.version} instead of ${row.package.version}, reason: ${bug.reason}`;
+      packageJSON.deprecated = row.package.deprecated ? `${row.package.deprecated} (${hotfixDeprecated})` : hotfixDeprecated;
+      // don't change version
+      packageJSON.version = row.package.version;
+      Object.assign(row.package, packageJSON);
+    }
+  }
+};

services/common.js

@@ -14,3 +14,10 @@ exports.isPrivatePackage = function (name) {
   }
   return false;
 };
+
+exports.CHANGE_TYPE = {
+  PACKAGE_TAG_ADDED: 'PACKAGE_TAG_ADDED',
+  PACKAGE_VERSION_ADDED: 'PACKAGE_VERSION_ADDED',
+  PACKAGE_UNPUBLISHED: 'PACKAGE_UNPUBLISHED',
+  PACKAGE_VERSION_BLOCKED: 'PACKAGE_VERSION_BLOCKED'
+};

services/npm.js

@@ -153,7 +153,7 @@ exports.getAllToday = function* (timeout) {
 };
 
 exports.getShort = function* (timeout) {
-  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://r.cnpmjs.org';
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://registry.npmmirror.com';
   var r = yield request('/-/short', {
     timeout: timeout || 300000,
     // registry.npmjs.org/-/short is 404 now therefore have a fallback
@@ -198,3 +198,53 @@ exports.getPopular = function* (top, timeout) {
     return r[0];
   });
 };
+
+exports.getChangesUpdateSeq = function* () {
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://registry.npmmirror.com';
+  const r = yield request('/', {
+    timeout: 30000,
+    registry: registry,
+  });
+  const data = r.data || {};
+  if (r.status !== 200) {
+    if (data.code && data.message) {
+      const url = registry + '/';
+      const err = new Error(data.message + ', url: ' + url);
+      err.name = data.code;
+      err.url = url;
+      throw err;
+    }
+  }
+  return data.update_seq || 0;
+};
+
+exports.listChanges = function* (updateSeq) {
+  const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://registry.npmmirror.com';
+  const changesUrl = `/_changes?since=${updateSeq}`;
+  const r = yield request(changesUrl, {
+    timeout: 30000,
+    registry: registry,
+  });
+  const data = r.data || {};
+  if (r.status !== 200) {
+    if (data.code && data.message) {
+      const url = registry + changesUrl;
+      const err = new Error(data.message + ', url: ' + url);
+      err.name = data.code;
+      err.url = url;
+      throw err;
+    }
+  }
+  // {"results":[{"seq":1988653,"type":"PACKAGE_VERSION_ADDED","id":"dsr-package-mercy-magot-thorp-sward","changes":[{"version":"1.0.1"}]},
+  return data.results || [];
+};
+
+exports.getScopePackagesShort = function* (scope, registry) {
+  var response = yield request('/browse/keyword/' + scope, {
+    timeout: 3000,
+    registry: registry,
+    dataType: 'text'
+  });
+  var res = response.data.match(/class="package-name">(\S*)<\/a>/g)
+  return res ? res.map(a => a.match(/class="package-name">(\S*)<\/a>/)[1]).filter(name => name.indexOf(`${scope}/`) === 0) : []
+};

services/package.js

@@ -5,6 +5,8 @@ var models = require('../models');
 var common = require('./common');
 var libCommon = require('../lib/common');
 var config = require('../config');
+var { ensureSinceIsDate } = require('../controllers/utils');
+var { BlockPackageVersion } = require('../models');
 var Tag = models.Tag;
 var User = models.User;
 var Module = models.Module;
@@ -15,6 +17,8 @@ var ModuleDependency = models.ModuleDependency;
 var ModuleUnpublished = models.ModuleUnpublished;
 var NpmModuleMaintainer = models.NpmModuleMaintainer;
 
+var CHANGE_TYPE = common.CHANGE_TYPE;
+
 // module
 var _parseRow = function (row) {
   if (row.package.indexOf('%7B%22') === 0) {
@@ -153,27 +157,19 @@ exports.listModulesByUser = function* (username) {
 exports.listModuleNamesByUser = function* (username) {
   var sql = 'SELECT distinct(name) AS name FROM module WHERE author=?;';
   var rows = yield models.query(sql, [username]);
-  var map = {};
   var names = rows.map(function (r) {
     return r.name;
   });
 
   // find from npm module maintainer table
   var moduleNames = yield NpmModuleMaintainer.listModuleNamesByUser(username);
-  moduleNames.forEach(function (name) {
-    if (!map[name]) {
-      names.push(name);
-    }
-  });
-
   // find from private module maintainer table
-  moduleNames = yield PrivateModuleMaintainer.listModuleNamesByUser(username);
-  moduleNames.forEach(function (name) {
-    if (!map[name]) {
-      names.push(name);
-    }
-  });
-  return names;
+  var privateModuleNames = yield PrivateModuleMaintainer.listModuleNamesByUser(username);
+  return Array.from(new Set([
+    ...names,
+    ...moduleNames,
+    ...privateModuleNames,
+  ]));
 };
 
 exports.listPublicModulesByUser = function* (username) {
@@ -206,6 +202,84 @@ exports.listPublicModuleNamesByUser = function* (username) {
   return names;
 };
 
+exports.listModelSince = function(Model, attributes, mapper) {
+
+  return function*(since, limit) {
+
+    var start = ensureSinceIsDate(since);
+    var findCondition = {
+      attributes: attributes,
+      where: {
+        gmt_modified: {
+          gte: start,
+          // 添加延时，防止同一时间多个数据未同步
+          lte: new Date(Date.now() - config.changesDelay || 5000),
+        },
+      },
+      order: [
+        ["gmt_modified", "ASC"],
+        ["id", "ASC"],
+      ],
+    };
+    if (limit) {
+      findCondition.limit = limit;
+    }
+    var rows = yield Model.findAll(findCondition);
+    return rows.map(mapper);
+  }
+}
+
+exports.listTagSince = this.listModelSince(
+  Tag,
+  ['id', 'name', 'tag', 'gmt_modified'],
+  function (row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_TAG_ADDED,
+      id: row.name,
+      changes: [{tag: row.tag}],
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listVersionSince = this.listModelSince(
+  Module,
+  ['id', 'name', 'version', 'gmt_modified'],
+  function (row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_VERSION_ADDED,
+      id: row.name,
+      changes: [{version: row.version}],
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listUnpublishedModuleSince = this.listModelSince(
+  ModuleUnpublished,
+  ['id', 'name', 'gmt_modified'],
+  function(row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_UNPUBLISHED,
+      id: row.name,
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+exports.listBlockVersionSince = this.listModelSince(
+  BlockPackageVersion,
+  ['id', 'name', 'version', 'gmt_modified'],
+  function(row) {
+    return {
+      type: CHANGE_TYPE.PACKAGE_VERSION_BLOCKED,
+      id: row.name,
+      gmt_modified: row.gmt_modified,
+    };
+  }
+);
+
+
 // start must be a date or timestamp
 exports.listPublicModuleNamesSince = function* listPublicModuleNamesSince(start) {
   if (!(start instanceof Date)) {
@@ -355,23 +429,22 @@ exports.findAllModuleAbbreviateds = function* (where, order, limit, offset) {
 };
 
 // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
-exports.saveModuleAbbreviated = function* (mod) {
-  var pkg = JSON.stringify({
-    name: mod.package.name,
-    version: mod.package.version,
-    deprecated: mod.package.deprecated,
-    dependencies: mod.package.dependencies,
-    optionalDependencies: mod.package.optionalDependencies,
-    devDependencies: mod.package.devDependencies,
-    bundleDependencies: mod.package.bundleDependencies,
-    peerDependencies: mod.package.peerDependencies,
-    bin: mod.package.bin,
-    directories: mod.package.directories,
-    dist: mod.package.dist,
-    engines: mod.package.engines,
-    _hasShrinkwrap: mod.package._hasShrinkwrap,
-    _publish_on_cnpm: mod.package._publish_on_cnpm,
-  });
+exports.saveModuleAbbreviated = function* (mod, remoteAbbreviatedVersion) {
+  // try to use remoteAbbreviatedVersion first
+  var pkg;
+  if (remoteAbbreviatedVersion) {
+    pkg = Object.assign({}, remoteAbbreviatedVersion, {
+      // override remote tarball
+      dist: Object.assign({}, remoteAbbreviatedVersion.dist, mod.package.dist, {
+        noattachment: undefined,
+      }),
+    });
+  } else {
+    pkg = Object.assign({}, mod.package, {
+      // ignore readme force
+      readme: undefined,
+    });
+  }
   var publish_time = mod.publish_time || Date.now();
   var item = yield models.ModuleAbbreviated.findByNameAndVersion(mod.name, mod.version);
   if (!item) {
@@ -381,7 +454,7 @@ exports.saveModuleAbbreviated = function* (mod) {
     });
   }
   item.publish_time = publish_time;
-  item.package = pkg;
+  item.package = JSON.stringify(pkg);
 
   if (item.changed()) {
     item = yield item.save();
@@ -417,7 +490,7 @@ exports.updateModulePackageFields = function* (id, fields) {
 };
 
 exports.updateModuleAbbreviatedPackage = function* (item) {
-  // item => { id, name, version, _hasShrinkwrap }
+  // item => { id, name, version, _hasShrinkwrap, os, cpu, peerDependenciesMeta, workspaces }
   var mod = yield models.ModuleAbbreviated.findByNameAndVersion(item.name, item.version);
   if (!mod) {
     return null;

sync/sync_exist.js

@@ -34,35 +34,38 @@ module.exports = function* sync() {
     throw new Error('can not found total info');
   }
 
-  var allPackages;
-  if (!info.last_exist_sync_time) {
-    var pkgs = yield npmService.getShort();
-    debug('First time sync all packages from official registry, got %d packages', pkgs.length);
-    if (info.last_sync_module) {
-      // start from last success
-      var lastIndex = pkgs.indexOf(info.last_sync_module);
-      if (lastIndex > 0) {
-        pkgs = pkgs.slice(lastIndex);
-        debug('recover from %d: %s', lastIndex, info.last_sync_module);
-      }
-    }
-    allPackages = pkgs;
-  } else {
-    debug('sync new module from last exist sync time: %s', info.last_exist_sync_time);
-    var result = yield npmService.fetchUpdatesSince(info.last_exist_sync_time);
-    allPackages = result.names;
-    syncTime = result.lastModified;
+  var lastSeq = info.last_exist_sync_time;
+  if (lastSeq && lastSeq > 132897820073) {
+    // ignore exists timestamp
+    lastSeq = null;
   }
-
-  var packages = intersection(existPackages, allPackages);
-  if (!packages.length) {
+  if (!lastSeq) {
+    lastSeq = yield npmService.getChangesUpdateSeq();
+  }
+  if (!lastSeq) {
     debug('no packages need be sync');
     return {
       successes: [],
       fails: []
     };
   }
+
+  var updatesPackages = [];
+  var changes = yield npmService.listChanges(lastSeq);
+  changes.forEach(change => {
+    updatesPackages.push(change.id);
+    lastSeq = change.seq;
+  });
+  var packages = intersection(existPackages, updatesPackages);
   debug('Total %d packages to sync, top 10: %j', packages.length, packages.slice(0, 10));
+  if (!packages.length) {
+    yield totalService.setLastExistSyncTime(lastSeq);
+    debug('no packages need be sync, lastSeq: %s, changes: %s', lastSeq, changes.length);
+    return {
+      successes: [],
+      fails: []
+    };
+  }
 
   var worker = new SyncModuleWorker({
     username: 'admin',
@@ -75,10 +78,10 @@ module.exports = function* sync() {
   var end = thunkify.event(worker);
   yield end();
 
-  debug('All packages sync done, successes %d, fails %d',
-    worker.successes.length, worker.fails.length);
+  debug('All packages sync done, successes %d, fails %d, lastSeq: %s, changes: %s',
+    worker.successes.length, worker.fails.length, lastSeq, changes.length);
 
-  yield totalService.setLastExistSyncTime(syncTime);
+  yield totalService.setLastExistSyncTime(lastSeq);
   return {
     successes: worker.successes,
     fails: worker.fails

sync/sync_scope.js

@@ -0,0 +1,89 @@
+'use strict';
+
+var thunkify = require('thunkify-wrap');
+const co = require('co');
+const ms = require('humanize-ms');
+var config = require('../config');
+var npmService = require('../services/npm');
+var SyncModuleWorker = require('../controllers/sync_module_worker');
+var logger = require('../common/logger');
+
+
+let syncing = false;
+const syncFn = co.wrap(function*() {
+  if (syncing) { return; }
+  syncing = true;
+  logger.syncInfo('Start syncing scope modules...');
+  let data;
+  let error;
+  try {
+    data = yield sync();
+  } catch (err) {
+    error = err;
+    error.message += ' (sync package error)';
+    logger.syncError(error);
+  }
+
+  if (data) {
+    logger.syncInfo(data);
+  }
+  if (!config.debug) {
+    sendMailToAdmin(error, data, new Date());
+  }
+  syncing = false;
+});
+
+syncFn().catch(onerror);
+setInterval(() => syncFn().catch(onerror), ms(config.syncScopeInterval));
+
+function onerror(err) {
+  logger.error('====================== scope sync error ========================');
+  logger.error(err);
+}
+
+function* getOtherCnpmDefineScopePackages(scopes) {
+  var arr = []
+  for (var i = 0; i < scopes.length; i++) {
+    var packageList = yield npmService.getScopePackagesShort(scopes[i].scope, scopes[i].sourceCnpmWeb)
+    arr = arr.concat(packageList)
+  }
+  return arr
+}
+
+function* sync() {
+  var scopeConfig = config.syncScopeConfig
+  if (!scopeConfig || scopeConfig.length === 0) {
+    process.exit(0);
+  }
+  var packages = yield getOtherCnpmDefineScopePackages(scopeConfig);
+
+  if (!packages.length) {
+    return;
+  }
+  logger.syncInfo('Total %d scope packages to sync: %j', packages.length, packages);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    noDep: true,
+    syncUpstreamFirst: false,
+    publish: true,
+    concurrency: config.syncConcurrency,
+    syncPrivatePackage: scopeConfig.reduce((arr, cur) => {
+      arr[cur.scope] = cur.sourceCnpmRegistry
+      return arr
+    }, {})
+  });
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  logger.syncInfo('scope packages sync done, successes %d, fails %d, updates %d',
+      worker.successes.length, worker.fails.length, worker.updates.length);
+
+  return {
+    successes: worker.successes,
+    fails: worker.fails,
+    updates: worker.updates,
+  };
+};

test/common/urllib.test.js

@@ -9,7 +9,8 @@ describe('test/common/urllib.test.js', () => {
   describe('accelerate request', () => {
     beforeEach(() => {
       mm(config, 'accelerateHostMap', {
-        'www.alipay.com': 'www.antgroup.com'
+        'www.alipay.com': 'www.antgroup.com',
+        'www.google.com': 'www.google.com'
       });
     });
 
@@ -17,6 +18,7 @@ describe('test/common/urllib.test.js', () => {
       it('should work', function* () {
         const res = yield urllib.request('https://www.alipay.com', {
           followRedirect: true,
+          timeout: 30000,
         });
         assert.deepStrictEqual(res.res.requestUrls, [
           'https://www.antgroup.com/',
@@ -24,14 +26,15 @@ describe('test/common/urllib.test.js', () => {
       });
     });
 
-    describe('redirect', () => {
+    describe.skip('redirect', () => {
       it('should work', function* () {
-        const res = yield urllib.request('http://alipay.com', {
+        const res = yield urllib.request('https://google.com', {
           followRedirect: true,
+          timeout: 30000,
         });
         assert.deepStrictEqual(res.res.requestUrls, [
-          'http://alipay.com/',
-          'https://www.antgroup.com/',
+          'https://google.com/',
+          'https://www.google.com/',
         ]);
       });
     });

test/controllers/registry/package/changes.test.js

@@ -0,0 +1,100 @@
+'use strict';
+
+var should = require('should');
+var request = require('supertest');
+var mm = require('mm');
+var config = require('../../../../config');
+var app = require('../../../../servers/registry');
+var utils = require('../../../utils');
+var CHANGE_TYPE = require('../../../../services/common').CHANGE_TYPE;
+
+describe('test/controllers/registry/package/changes.test.js', function () {
+  afterEach(mm.restore);
+
+  var since;
+  before(function (done) {
+    mm(config, 'changesDelay', 0);
+    setTimeout(() => {
+      since = Date.now();
+      var pkg = utils.getPackage('@cnpmtest/test_changes', '0.0.1', utils.admin, 'alpha');
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, function() {
+        setTimeout(function() {
+          pkg = utils.getPackage('@cnpmtest/test_changes_gogo', '0.0.2', utils.admin, 'beta');
+          request(app)
+          .put('/' + pkg.name)
+          .set('authorization', utils.adminAuth)
+          .send(pkg)
+          .expect(201, done);
+        }, 2000);
+      });
+    }, 1000);
+  });
+
+  describe('GET /-/all/changes', function () {
+    it('should 200', function (done) {
+      request(app)
+        .get("/-/all/changes?since=" + since)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(2);
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(2);
+          done();
+        });
+    });
+
+    it('changes delay should work', function(done) {
+      mm(config, 'changesDelay', 10000);
+      request(app)
+        .get("/-/all/changes?since=" + since)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+          res.body.results
+            .filter(function (item) {
+              return item.type === CHANGE_TYPE.PACKAGE_VERSION_ADDED;
+            })
+            .length.should.equal(0);
+          done();
+        });
+    })
+
+    it('since should work', function (done) {
+      var now = Date.now();
+      request(app)
+        .get("/-/all/changes?since=" + now + 5000)
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          res.body.results.should.be.an.Array();
+          res.body.results.length.should.equal(0);
+          done();
+        });
+    });
+
+    it('limit should work', function (done) {
+      mm(config, 'changesDelay', 0);
+      request(app)
+      .get('/-/all/changes?limit=1&since=' + since)
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.body.results.should.be.an.Array();
+        res.body.results.length.should.equal(1);
+        done();
+      });
+    });
+
+  });
+});

test/controllers/registry/package/download.test.js

@@ -25,12 +25,24 @@ describe('test/controllers/registry/package/download.test.js', function () {
       .expect(200, done);
     });
 
+    it('should download a file with 200', function (done) {
+      request(app)
+        .get('/@cnpmtest/download-test-module/download/download-test-module-1.0.0.tgz')
+        .expect(200, done);
+    });
+
     it('should alias /:name/-/:filename to /:name/download/:filename', function (done) {
       request(app)
       .get('/@cnpmtest/download-test-module/-/@cnpmtest/download-test-module-1.0.0.tgz')
       .expect(200, done);
     });
 
+    it('should alias /:name/-/:filename to /:name/download/:filename', function (done) {
+      request(app)
+        .get('/@cnpmtest/download-test-module/-/download-test-module-1.0.0.tgz')
+        .expect(200, done);
+    });
+
     it('should 404 when package not exists', function (done) {
       request(app)
       .get('/@cnpmtest/download-test-module-not-exists/download/@cnpmtest/download-test-module-not-exists-1.0.0.tgz')

test/controllers/registry/package/list.test.js

@@ -6,6 +6,7 @@ var request = require('supertest');
 var mm = require('mm');
 var pedding = require('pedding');
 var packageService = require('../../../../services/package');
+var blocklistService = require('../../../../services/blocklist');
 var app = require('../../../../servers/registry');
 var utils = require('../../../utils');
 var config = require('../../../../config');
@@ -20,7 +21,10 @@ describe('test/controllers/registry/package/list.test.js', () => {
     var pkg = utils.getPackage('@cnpmtest/testmodule-list-1', '0.0.1', utils.otherUser);
     pkg.versions['0.0.1'].dependencies = {
       bytetest: '~0.0.1',
-      mocha: '~1.0.0'
+      mocha: '~1.0.0',
+    };
+    pkg.versions['0.0.1'].scripts = {
+      install: 'node -v',
     };
     request(app)
     .put('/' + pkg.name)
@@ -41,6 +45,337 @@ describe('test/controllers/registry/package/list.test.js', () => {
     });
   });
 
+  describe('config.enableBugVersion = true / false', () => {
+    before(done => {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      utils.sync('base62', done);
+    });
+
+    before(function* () {
+      let pkg = utils.getPackage('bug-versions', '1.0.0', utils.admin);
+      const packageJSON = pkg.versions['1.0.0'];
+      packageJSON.config = {};
+      packageJSON.config['bug-versions'] = {
+        "base62": {
+          "1.2.5": {
+            "version": "1.2.1",
+            "reason": "ignore post-install script https://github.com/andrew/base62.js/commits/master"
+          }
+        },
+        "request": {
+          "2.84.0": {
+            "version": "2.85.0",
+            "reason": "https://github.com/cnpm/bug-versions/issues/3"
+          }
+        },
+        "@cnpmtest/testmodule-list-bugversion": {
+          "4.14.0": {
+            "version": "4.13.2",
+            "reason": "https://github.com/ali-sdk/ali-oss/pull/382"
+          },
+          "4.14.1": {
+            "version": "4.13.2",
+            "reason": "https://github.com/ali-sdk/ali-oss/pull/382"
+          },
+          "4.14.2": {
+            "version": "1.0.0",
+            "reason": "https://github.com/ali-sdk/ali-oss/pull/382"
+          }
+        }
+      };
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.adminAuth)
+        .send(pkg)
+        .expect(201);
+
+      pkg = utils.getPackage('@cnpmtest/testmodule-list-bugversion', '4.14.0', utils.otherUser);
+      pkg.versions['4.14.0'].deprecated = 'mock deprecated exists here';
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+      pkg = utils.getPackage('@cnpmtest/testmodule-list-bugversion', '4.14.1', utils.otherUser);
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+      pkg = utils.getPackage('@cnpmtest/testmodule-list-bugversion', '4.14.2', utils.otherUser);
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+      pkg = utils.getPackage('@cnpmtest/testmodule-list-bugversion', '4.13.2', utils.otherUser);
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+    });
+
+    it('should replace base62\'s bug version 1.2.5 to 1.2.1', function* () {
+      mm(config, 'syncModel', 'all');
+      mm(config, 'enableAbbreviatedMetadata', true);
+      mm(config, 'enableBugVersion', true);
+      mm(config.nfs, 'url', function* (key) {
+        return 'http://foo.test.com' + key;
+      });
+      mm(config, 'downloadRedirectToNFS', true);
+      // https://github.com/cnpm/bug-versions/blob/master/package.json#L201
+      let res = yield request(app)
+        .get('/base62')
+        .expect(200);
+      let data = res.body;
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5'].version === '1.2.5');
+      assert(data.versions['1.2.5'].deprecated === '[WARNING] Use 1.2.1 instead of 1.2.5, reason: ignore post-install script https://github.com/andrew/base62.js/commits/master');
+      assert(data.versions['1.2.5'].dist.tarball.endsWith('/base62-1.2.1.tgz'));
+      // assert(!data.versions['1.2.1'].deprecated);
+
+      res = yield request(app)
+        .get('/base62')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5'].version === '1.2.5');
+      assert(data.versions['1.2.5'].deprecated);
+      assert(data.versions['1.2.5'].dist.tarball.endsWith('/base62-1.2.1.tgz'));
+      // assert(!data.versions['1.2.1'].deprecated);
+
+      // ignore when sync worker request
+      res = yield request(app)
+        .get('/base62?cache=0')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5'].version === '1.2.5');
+      assert(data.versions['1.2.5'].dist.tarball.endsWith('/base62-1.2.5.tgz'));
+      // assert(!data.versions['1.2.5'].deprecated);
+      res = yield request(app)
+        .get('/base62?cache=0')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5'].version === '1.2.5');
+      assert(data.versions['1.2.5'].dist.tarball.endsWith('/base62-1.2.5.tgz'));
+      // assert(!data.versions['1.2.5'].deprecated);
+
+      // dont change download url
+      yield request(app)
+        .get('/base62/download/base62-1.2.5.tgz')
+        .expect('location', 'http://foo.test.com/base62/-/base62-1.2.5.tgz')
+        .expect(302);
+
+      // ignore when enableBugVersion = false
+      mm(config, 'enableBugVersion', false);
+      res = yield request(app)
+        .get('/base62')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['1.2.5']);
+      assert(data.versions['1.2.1']);
+      assert(data.versions['1.2.5'].version === '1.2.5');
+      assert(data.versions['1.2.5'].dist.tarball.endsWith('/base62-1.2.5.tgz'));
+      // assert(!data.versions['1.2.5'].deprecated);
+
+      yield request(app)
+        .get('/base62/download/base62-1.2.5.tgz')
+        .expect('location', 'http://foo.test.com/base62/-/base62-1.2.5.tgz')
+        .expect(302);
+    });
+
+    it('should replace @cnpmtest/testmodule-list-bugversion bug versions', function* () {
+      mm(config, 'enableBugVersion', true);
+      mm(config, 'enableAbbreviatedMetadata', true);
+      mm(config.nfs, 'url', function* (key) {
+        return 'http://foo.test.com' + key;
+      });
+      mm(config, 'downloadRedirectToNFS', true);
+      // https://github.com/cnpm/bug-versions/blob/master/package.json#L201
+      let res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion')
+        .expect(200);
+      let data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here ([WARNING] Use 4.13.2 instead of 4.14.0, reason: https://github.com/ali-sdk/ali-oss/pull/382)');
+      assert(data.versions['4.14.1'].deprecated === '[WARNING] Use 4.13.2 instead of 4.14.1, reason: https://github.com/ali-sdk/ali-oss/pull/382');
+      assert(data.versions['4.14.0'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.13.2.tgz'));
+      assert(data.versions['4.14.1'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.13.2.tgz'));
+      assert(data.versions['4.14.2'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.14.2.tgz'));
+      // 4.14.2 replace bug version 1.0.0 dont exists, dont replace
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+
+      // dont change download url
+      yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion/download/@cnpmtest/testmodule-list-bugversion-4.14.0.tgz')
+        .expect('location', 'http://foo.test.com/@cnpmtest/testmodule-list-bugversion/-/@cnpmtest/testmodule-list-bugversion-4.14.0.tgz')
+        .expect(302);
+
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here ([WARNING] Use 4.13.2 instead of 4.14.0, reason: https://github.com/ali-sdk/ali-oss/pull/382)');
+      assert(data.versions['4.14.1'].deprecated === '[WARNING] Use 4.13.2 instead of 4.14.1, reason: https://github.com/ali-sdk/ali-oss/pull/382');
+      assert(data.versions['4.14.0'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.13.2.tgz'));
+      assert(data.versions['4.14.1'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.13.2.tgz'));
+      assert(data.versions['4.14.2'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.14.2.tgz'));
+      // 4.14.2 replace bug version 1.0.0 dont exists, dont replace
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+
+      // ignore when sync worker request
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion?cache=0')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here');
+      assert(!data.versions['4.14.1'].deprecated);
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion?cache=0')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here');
+      assert(!data.versions['4.14.1'].deprecated);
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+
+      // ignore when enableBugVersion = false
+      mm(config, 'enableBugVersion', false);
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here');
+      assert(!data.versions['4.14.1'].deprecated);
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-bugversion')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(data.versions['4.14.0']);
+      assert(data.versions['4.14.1']);
+      assert(data.versions['4.14.2']);
+      assert(data.versions['4.13.2']);
+      assert(data.versions['4.14.0'].version === '4.14.0');
+      assert(data.versions['4.14.1'].version === '4.14.1');
+      assert(data.versions['4.14.0'].deprecated === 'mock deprecated exists here');
+      assert(data.versions['4.14.0'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.14.0.tgz'));
+      assert(data.versions['4.14.1'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.14.1.tgz'));
+      assert(data.versions['4.14.2'].dist.tarball.endsWith('/@cnpmtest/testmodule-list-bugversion-4.14.2.tgz'));
+      assert(!data.versions['4.14.1'].deprecated);
+      assert(!data.versions['4.14.2'].deprecated);
+      assert(!data.versions['4.13.2'].deprecated);
+    });
+  });
+
+  describe('block versions', () => {
+    before(function* () {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-list-block', '0.0.1', utils.otherUser);
+      pkg.versions['0.0.1'].dependencies = {
+        bytetest: '~0.0.1',
+        mocha: '~1.0.0',
+      };
+      pkg.versions['0.0.1'].scripts = {
+        install: 'node -v',
+      };
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+      
+      pkg = utils.getPackage('@cnpmtest/testmodule-list-block', '1.0.0', utils.otherUser);
+      pkg.versions['1.0.0'].dependencies = {
+        bytetest: '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', utils.otherUserAuth)
+        .send(pkg)
+        .expect(201);
+    });
+
+    it('should block one version and all versions', function* () {
+      yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-list-block', '0.0.1', 'unittest');
+      let res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-block')
+        .expect(200);
+      let data = res.body;
+      assert(Object.keys(data.versions).length === 1);
+      assert(data.versions['1.0.0']);
+      assert(!data.versions['0.0.1']);
+
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-block')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200);
+      data = res.body;
+      assert(Object.keys(data.versions).length === 1);
+      assert(data.versions['1.0.0']);
+      assert(!data.versions['0.0.1']);
+
+      yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-list-block', '*', 'unittest');
+      res = yield request(app)
+        .get('/@cnpmtest/testmodule-list-block')
+        .expect(451);
+      data = res.body;
+      console.log(data);
+      assert(data.error === '[block] package was blocked, reason: unittest');
+    });
+  });
+
   it('should use costomized registry middleware', done => {
     request(app)
     .get('/@cnpmtest/testmodule-list-1')
@@ -351,6 +686,23 @@ describe('test/controllers/registry/package/list.test.js', () => {
         });
     });
 
+    it('should return abbreviated meta on private package and has hasInstallScript field', () => {
+      mm(config, 'enableAbbreviatedMetadata', true);
+      return request(app)
+        .get('/@cnpmtest/testmodule-list-1')
+        .set('Accept', 'application/vnd.npm.install-v1+json')
+        .expect(200)
+        .expect(res => {
+          const data = res.body;
+          // console.log(JSON.stringify(data, null, 2));
+          assert(data.name === '@cnpmtest/testmodule-list-1');
+          assert(data.modified);
+          assert(data['dist-tags'].latest);
+          assert(data.versions['0.0.1'].hasInstallScript === true);
+          assert(data.versions['0.0.1'].scripts === undefined);
+        });
+    });
+
     it('should return abbreviated meta with cache-control', () => {
       mm(config, 'registryCacheControlHeader', 'max-age=0, s-maxage=10, must-revalidate');
       mm(config, 'syncModel', 'all');
@@ -434,7 +786,7 @@ describe('test/controllers/registry/package/list.test.js', () => {
             // assert('_hasShrinkwrap' in pkg);
             assert(pkg.publish_time && typeof pkg.publish_time === 'number');
             assert(pkg._publish_on_cnpm === undefined);
-            assert(pkg.dist.tarball.includes('.tgz?bucket=foo-us1&admin=1&other_urls=http'));
+            assert(pkg.dist.tarball.endsWith('.tgz'));
           }
           assert(/^W\/"\w{32}"$/.test(res.headers.etag));
         });

test/controllers/registry/package/save.test.js

@@ -11,13 +11,13 @@ var app = require('../../../../servers/registry');
 var config = require('../../../../config');
 var utils = require('../../../utils');
 
-describe('test/controllers/registry/package/save.test.js', function () {
+describe('test/controllers/registry/package/save.test.js', () => {
   afterEach(mm.restore);
 
   describe('no @scoped package', function () {
     beforeEach(function () {
       mm(config, 'syncModel', 'all');
-      mm(config, 'privatePackages', ['testmodule-new-1', 'testmodule-new-2', 'testmodule-no-latest']);
+      mm(config, 'privatePackages', ['testmodule-new-1', 'testmodule-new-2', 'testmodule-no-latest', 'testmodule-new-4']);
     });
 
     before(function (done) {
@@ -46,22 +46,112 @@ describe('test/controllers/registry/package/save.test.js', function () {
       });
     });
 
-    it('should publish new version package and save dependencies', function (done) {
+    it('should publish new version package and save dependencies', done => {
       request(app)
       .get('/testmodule-new-1')
       .expect(200, function (err, res) {
-        should.not.exist(err);
+        assert(!err);
         var data = res.body;
-        data.name.should.equal('testmodule-new-1');
-        Object.keys(data.versions).should.eql(['0.0.1']);
-        data.versions['0.0.1'].dependencies.should.eql({
+        assert(data.name === 'testmodule-new-1');
+        assert.deepStrictEqual(Object.keys(data.versions), ['0.0.1']);
+        assert.deepStrictEqual(data.versions['0.0.1'].dependencies, {
           'bytetest-1': '~0.0.1',
           mocha: '~1.0.0'
         });
+        // should has integrity with sha512
+        assert(data.versions['0.0.1'].dist.integrity === 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==');
+        assert(data.versions['0.0.1'].dist.shasum === 'fa475605f88bab9b1127833633ca3ae0a477224c');
         done();
       });
     });
 
+    it('should publish error on shasum invaild', done => {
+      mm(config, 'privatePackages', ['testmodule-new-1']);
+      var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
+      pkg.versions['0.0.88'].dependencies = {
+        'bytetest-1': '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      pkg.versions['0.0.88'].dist.shasum = 'fa475605f88bab9b1127833633ca3ae0a47wrong';
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        assert(!err);
+        assert(res.body.error === '[invalid] dist.shasum invalid');
+        assert(res.body.reason === '[invalid] dist.shasum invalid');
+        done();
+      });
+    });
+
+    it('should publish error on integrity invaild', done => {
+      mm(config, 'privatePackages', ['testmodule-new-1']);
+      var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
+      pkg.versions['0.0.88'].dependencies = {
+        'bytetest-1': '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      pkg.versions['0.0.88'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfBBB==';
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(400, function (err, res) {
+        assert(!err);
+        assert(res.body.error === '[invalid] dist.integrity invalid');
+        assert(res.body.reason === '[invalid] dist.integrity invalid');
+        done();
+      });
+    });
+
+    it('should publish success with integrity and shasum', done => {
+      mm(config, 'privatePackages', ['testmodule-new-1']);
+      var pkg = utils.getPackage('testmodule-new-1', '0.0.88', utils.admin);
+      pkg.versions['0.0.88'].dependencies = {
+        'bytetest-1': '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      pkg.versions['0.0.88'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==';
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should publish success with integrity and without shasum', done => {
+      mm(config, 'privatePackages', ['testmodule-new-1']);
+      var pkg = utils.getPackage('testmodule-new-1', '0.0.881', utils.admin);
+      pkg.versions['0.0.881'].dependencies = {
+        'bytetest-1': '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      pkg.versions['0.0.881'].dist.integrity = 'sha512-n+4CQg0Rp1Qo0p9a0R5E5io67T9iD3Lcgg6exmpmt0s8kd4XcOoHu2kiu6U7xd69cGq0efkNGWUBP229ObfRSA==';
+      delete pkg.versions['0.0.881'].dist;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
+    it('should publish success without integrity and without shasum', done => {
+      mm(config, 'privatePackages', ['testmodule-new-1']);
+      var pkg = utils.getPackage('testmodule-new-1', '0.0.882', utils.admin);
+      pkg.versions['0.0.882'].dependencies = {
+        'bytetest-1': '~0.0.1',
+        mocha: '~1.0.0'
+      };
+      delete pkg.versions['0.0.882'].dist.integrity;
+      delete pkg.versions['0.0.882'].dist;
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', utils.adminAuth)
+      .send(pkg)
+      .expect(201, done);
+    });
+
     it('should publish new package and fire globalHook', done => {
       done = pedding(done, 2);
       mm(config, 'globalHook', function* (envelope) {
@@ -215,6 +305,27 @@ describe('test/controllers/registry/package/save.test.js', function () {
       .expect(403, done);
     });
 
+    it('should publish when maintainers dont contain current user in token mode', function* () {
+      var token = yield tokenService.createToken(utils.admin);
+
+      var pkg = utils.getPackage('testmodule-new-4', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].maintainers[0].name += '-testuser';
+
+      yield request(app)
+        .put('/' + pkg.name)
+        .set('authorization', 'Bearer ' + token.token)
+        .send(pkg)
+        .expect(201);
+
+      yield tokenService.deleteToken(utils.admin, token.token);
+
+      var maintainers = yield packageService.listMaintainers(pkg.name);
+      maintainers.should.eql([{
+        name: 'cnpmjstest10',
+        email: 'fengmk2@gmail.com',
+      }]);
+    });
+
     it('should 400 when attachments missing', function (done) {
       var pkg = utils.getPackage('testmodule-new-1', '0.0.1', utils.admin);
       delete pkg._attachments;

test/controllers/registry/package/show.test.js

@@ -142,7 +142,7 @@ describe('test/controllers/registry/package/show.test.js', function () {
     });
   });
 
-  it('should return other_urls on query exists', function (done) {
+  it('should ignore query', function (done) {
     request(app)
     .get('/@cnpmtest/testmodule-show/latest?bucket=us1')
     .expect(200, function (err, res) {
@@ -153,7 +153,7 @@ describe('test/controllers/registry/package/show.test.js', function () {
       data['dist-tags'].should.eql({
         latest: '1.1.0',
       });
-      data.dist.tarball.should.equal('http://r.cnpmjs.org/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-1.1.0.tgz?bucket=us1&other_urls=http%3A%2F%2Fr.cnpmjs.org%2F%40cnpmtest%2Ftestmodule-show%2Fdownload%2F%40cnpmtest%2Ftestmodule-show-1.1.0.tgz');
+      data.dist.tarball.should.equal('http://r.cnpmjs.org/@cnpmtest/testmodule-show/download/@cnpmtest/testmodule-show-1.1.0.tgz');
       done();
     });
   });

test/controllers/registry/token/create.test.js

@@ -5,6 +5,7 @@ var request = require('supertest');
 var app = require('../../../../servers/registry');
 var tokenService = require('../../../../services/token');
 var TestUtil = require('../../../utils');
+var create = require('../../../../controllers/registry/token/create');
 
 describe('test/controllers/registry/token/create.test.js', function () {
   describe('POST /-/npm/v1/tokens', function () {
@@ -76,4 +77,28 @@ describe('test/controllers/registry/token/create.test.js', function () {
       });
     });
   });
+
+  describe('inject tokenService', () => {
+    it('should work', function* () {
+      const ctx = {
+        request: {
+          body: {
+            readonly: false,
+            cidr_whitelist: [],
+            password: TestUtil.admin,
+          },
+        },
+        user: {
+          name: TestUtil.admin,
+        },
+        tokenService: {
+          createToken(userId) {
+            return `${userId}:token`
+          }
+        },
+      };
+      yield Reflect.apply(create, ctx, []);
+      ctx.body.should.equal(`${TestUtil.admin}:token`);
+    });
+  });
 });

test/controllers/registry/user/show.test.js

@@ -31,7 +31,7 @@ describe('test/controllers/registry/user/show.test.js, GET /-/user/org.couchdb.u
     });
   });
 
-  it('should return npm user info', function (done) {
+  it.skip('should return npm user info', function (done) {
     request(app)
     .get('/-/user/org.couchdb.user:fengmk2')
     .expect(200, function (err, res) {
@@ -57,7 +57,7 @@ describe('test/controllers/registry/user/show.test.js, GET /-/user/org.couchdb.u
       mm(config, 'customUserService', true);
     });
 
-    it('should show npm user', function (done) {
+    it.skip('should show npm user', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:fengmk2')
       .expect(200, function (err, res) {

test/controllers/sync_module_worker.test.js

@@ -146,6 +146,185 @@ describe('test/controllers/sync_module_worker.test.js', () => {
     yield end();
   });
 
+  it('should sync mk2test-module-cnpmsync add os, cpu success', function* () {
+    mm(config, 'enableAbbreviatedMetadata', true);
+    mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.com');
+    var log = yield logService.create({
+      name: 'mk2test-module-cnpmsync',
+      username: 'fengmk2',
+    });
+    log.id.should.above(0);
+    var worker = new SyncModuleWorker({
+      logId: log.id,
+      name: 'mk2test-module-cnpmsync',
+      username: 'fengmk2',
+      noDep: true,
+    });
+    worker.start();
+    var end = thunkify.event(worker, 'end');
+    yield end();
+
+    let pkg;
+    let pkgV2;
+    let pkgV3;
+    let lastResHeaders;
+    function checkResult() {
+      return function (done) {
+        request(app)
+        .get('/mk2test-module-cnpmsync')
+        .set('accept', 'application/vnd.npm.install-v1+json')
+        .expect(function (res) {
+          lastResHeaders = res.headers;
+          pkg = res.body.versions['1.0.0'];
+          assert(pkg.os[0] === 'linux');
+          assert(pkg.cpu[0] === 'x64');
+          assert(!pkg.peerDependenciesMeta);
+          pkgV2 = res.body.versions['2.0.0'];
+          assert(pkgV2.os[0] === 'linux');
+          assert(pkgV2.cpu[0] === 'x64');
+          assert(pkgV2.peerDependenciesMeta);
+          pkgV3 = res.body.versions['3.0.0'];
+          assert(!pkgV3.os);
+          assert(!pkgV3.cpu);
+          assert(pkgV3.peerDependenciesMeta);
+        })
+        .expect(200, done);
+      };
+    }
+    yield checkResult();
+    const oldEtag = lastResHeaders.etag;
+    // check etag keep same again
+    yield checkResult();
+    assert(oldEtag == lastResHeaders.etag);
+
+    // modify result
+    yield packageService.updateModuleAbbreviatedPackage({
+      name: pkg.name,
+      version: pkg.version,
+      os: undefined,
+      cpu: undefined,
+      peerDependenciesMeta: undefined,
+    });
+
+    yield packageService.updateModuleAbbreviatedPackage({
+      name: pkgV2.name,
+      version: pkgV2.version,
+      os: undefined,
+      cpu: undefined,
+      peerDependenciesMeta: undefined,
+    });
+    yield packageService.updateModuleAbbreviatedPackage({
+      name: pkgV3.name,
+      version: pkgV3.version,
+      peerDependenciesMeta: undefined,
+    });
+
+    function checkModifiyResult() {
+      return function (done) {
+        request(app)
+        .get('/mk2test-module-cnpmsync')
+        .set('accept', 'application/vnd.npm.install-v1+json')
+        .expect(function (res) {
+          // console.log(JSON.stringify(res.body, null, 2));
+          assert(!res.body.versions['1.0.0'].os);
+          assert(!res.body.versions['1.0.0'].cpu);
+          assert(!res.body.versions['2.0.0'].peerDependenciesMeta);
+          assert(!res.body.versions['3.0.0'].peerDependenciesMeta);
+        })
+        .expect(200, done);
+      };
+    }
+    yield checkModifiyResult();
+
+    // sync again
+    worker = new SyncModuleWorker({
+      logId: log.id,
+      name: 'mk2test-module-cnpmsync',
+      username: 'fengmk2',
+      noDep: true,
+    });
+    worker.start();
+    end = thunkify.event(worker, 'end');
+    yield end();
+
+    // check again still work
+    yield checkResult();
+    const newEtag = lastResHeaders.etag;
+    assert(newEtag !== oldEtag);
+
+    // check etag keep same again
+    yield checkResult();
+    assert(newEtag == lastResHeaders.etag);
+  });
+
+  it('should sync mk2test-module-cnpmsync-issue-1667 with remoteAbbreviatedVersion success', function* () {
+    mm(config, 'enableAbbreviatedMetadata', true);
+    mm(config, 'sourceNpmRegistry', 'https://registry.npmjs.com');
+    var log = yield logService.create({
+      name: 'mk2test-module-cnpmsync-issue-1667',
+      username: 'fengmk2',
+    });
+    log.id.should.above(0);
+    var worker = new SyncModuleWorker({
+      logId: log.id,
+      name: 'mk2test-module-cnpmsync-issue-1667',
+      username: 'fengmk2',
+      noDep: true,
+    });
+    worker.start();
+    var end = thunkify.event(worker, 'end');
+    yield end();
+
+    let pkg;
+    let pkgV2;
+    let pkgV3;
+    let lastResHeaders;
+    function checkResult() {
+      return function (done) {
+        request(app)
+        .get('/mk2test-module-cnpmsync-issue-1667')
+        .set('accept', 'application/vnd.npm.install-v1+json')
+        .expect(function (res) {
+          lastResHeaders = res.headers;
+          console.log('%j', res.body);
+          pkg = res.body.versions['3.0.0'];
+          assert(pkg.hasInstallScript === true);
+          // no scripts
+          assert(!pkg.scripts);
+          assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz');
+          assert(!('noattachment' in pkg.dist));
+        })
+        .expect(200, done);
+      };
+    }
+    yield checkResult();
+
+    function checkFullResult() {
+      return function (done) {
+        request(app)
+        .get('/mk2test-module-cnpmsync-issue-1667')
+        .set('accept', 'application/json')
+        .expect(function (res) {
+          lastResHeaders = res.headers;
+          console.log('%j', res.body);
+          pkg = res.body.versions['3.0.0'];
+          assert(pkg.hasInstallScript === true);
+          // has scripts
+          assert(pkg.scripts);
+          // console.log(pkg.dist);
+          assert(pkg.dist.key === '/mk2test-module-cnpmsync-issue-1667/-/mk2test-module-cnpmsync-issue-1667-3.0.0.tgz');
+          assert(pkg.dist.integrity === 'sha512-pwnnZyjvr29UxwFAIx7xHvVCkFpGVAYgaFllr/m5AZoD1CR2uHHPw16ISEO/A2rZ0WM3UoAghwd5bAZ4pYzD2Q==');
+          assert(pkg.dist.shasum === 'c31af371a6cdc10dd5b9ad26625a4c863249198d');
+          assert(pkg.dist.fileCount === 2);
+          assert(pkg.dist.unpackedSize === 232);
+          assert(pkg.dist.size === 271);
+        })
+        .expect(200, done);
+      };
+    }
+    yield checkFullResult();
+  });
+
   it('should sync upstream first', function* () {
     mm(config, 'sourceNpmRegistryIsCNpm', true);
     var log = yield logService.create({
@@ -607,7 +786,7 @@ describe('test/controllers/sync_module_worker.test.js', () => {
       });
     });
 
-    describe('package unpublished', () => {
+    describe.skip('package unpublished', () => {
       it('should sync unpublished info', function* () {
         var worker = new SyncModuleWorker({
           name: ['afp'],

test/controllers/web/package/search.test.js

@@ -58,12 +58,11 @@ describe('test/controllers/web/package/search.test.js', function () {
 
     it('should list no match ok', function (done) {
       request(app)
-      .get('/browse/keyword/notexistpackage')
-      .expect(200)
-      .expect(/Can not found package match notexistpackage/, done);
+        .get('/browse/keyword/notexistpackage')
+        .expect(200)
+        .expect(/Can not found package match notexistpackage/, done);
     });
 
-
     describe('GET /browse/keyword/:word searchlist', function () {
 
       before(function (done) {

test/controllers/web/package/show.test.js

@@ -1,11 +1,13 @@
 'use strict';
 
 var should = require('should');
+var assert = require('assert');
 var request = require('supertest');
 var mm = require('mm');
 var config = require('../../../../config');
 var app = require('../../../../servers/web');
 var registry = require('../../../../servers/registry');
+var blocklistService = require('../../../../services/blocklist');
 var utils = require('../../../utils');
 
 describe('test/controllers/web/package/show.test.js', () => {
@@ -55,6 +57,41 @@ describe('test/controllers/web/package/show.test.js', () => {
       });
     });
 
+    it('should get block package', function* () {
+      var pkg = utils.getPackage('@cnpmtest/testmodule-web-show-block', '0.0.1', utils.admin);
+      pkg.versions['0.0.1'].dependencies = {
+        bytetest: '~0.0.1',
+        mocha: '~1.0.0',
+        'testmodule-web-show': '0.0.1'
+      };
+      yield request(registry)
+        .put('/' + pkg.name)
+        .set('authorization', utils.adminAuth)
+        .send(pkg)
+        .expect(201);
+      
+      yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '0.0.1', 'unittest');
+      let res = yield request(app)
+        .get('/package/@cnpmtest/testmodule-web-show-block')
+        .expect(451)
+        .expect('content-type', 'text/plain; charset=utf-8');
+      assert(res.text === '[block] package@0.0.1 was blocked, reason: unittest');
+
+      yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '0.0.1', 'unittest');
+      res = yield request(app)
+        .get('/package/@cnpmtest/testmodule-web-show-block/0.0.1')
+        .expect(451)
+        .expect('content-type', 'text/plain; charset=utf-8');
+      assert(res.text === '[block] package@0.0.1 was blocked, reason: unittest');
+
+      yield blocklistService.blockPackageVersion('@cnpmtest/testmodule-web-show-block', '*', 'block all');
+      res = yield request(app)
+        .get('/package/@cnpmtest/testmodule-web-show-block')
+        .expect(451)
+        .expect('content-type', 'text/plain; charset=utf-8');
+      assert(res.text === '[block] package@0.0.1 was blocked, reason: block all');
+    });
+
     it('should get scoped package', function (done) {
       request(app)
       .get('/package/@cnpmtest/testmodule-web-show')
@@ -76,6 +113,36 @@ describe('test/controllers/web/package/show.test.js', () => {
       .get('/package/@cnpmtest/not-exist-module')
       .expect(404, done);
     });
+
+    it('should get 404 show sync button on scoped package', done => {
+      mm(config, 'syncModel', 'all');
+      request(app)
+        .get('/package/@cnpmtest/testmodule-repo-short-https-404')
+        .expect(404)
+        .expect('content-type', 'text/html; charset=utf-8')
+        .expect(/>SYNC<\/a> from official npm registry/)
+        .expect(/Can not found package match @cnpmtest\/testmodule-repo-short-https-404/, done);
+    });
+
+    it('should get 404 show sync button on scoped package with encode url', done => {
+      mm(config, 'syncModel', 'all');
+      request(app)
+        .get('/package/%40foo%2Fawdawda')
+        .expect(404)
+        .expect('content-type', 'text/html; charset=utf-8')
+        .expect(/>SYNC<\/a> from official npm registry/)
+        .expect(/Can not found package match @foo\/awdawda/, done);
+    });
+
+    it('should get 404 show sync button on non-scoped package', done => {
+      mm(config, 'syncModel', 'all');
+      request(app)
+        .get('/package/testmodule-repo-short-https-404')
+        .expect(404)
+        .expect('content-type', 'text/html; charset=utf-8')
+        .expect(/>SYNC<\/a> from official npm registry/)
+        .expect(/Can not found package match testmodule-repo-short-https-404/, done);
+    });
   });
 
   describe('GET /package/:name/:version', function () {

test/fixtures/scope-package/Makefile

@@ -4,7 +4,7 @@ TIMEOUT = 1000
 MOCHA_OPTS =
 
 install:
-	@npm install --registry=https://registry.npm.taobao.org --disturl=https://npm.taobao.org/dist
+	@npm install --registry=https://registry.npmmirror.com --disturl=https://npmmirror.com/dist
 
 jshint: install
 	@./node_modules/.bin/jshint .

test/lib/common.test.js

@@ -11,28 +11,50 @@
 
 'use strict';
 
+const { co } = require('co');
+const mm = require('mm');
+const config = require('../../config');
+const createModule = require('../utils').createModule;
+const packageService = require('../../services/package');
 /**
  * Module dependencies.
  */
 
 var common = require('../../lib/common');
 
-describe('test/lib/common.test.js', function () {
-  describe('isAdmin()', function () {
-    it('should admin is admin', function () {
-      common.isAdmin('admin').should.equal(true);
-      common.isAdmin('fengmk2').should.equal(true);
-      common.isAdmin('constructor').should.equal(false);
-      common.isAdmin('toString').should.equal(false);
+describe("test/lib/common.test.js", function () {
+  describe("isAdmin()", function () {
+    it("should admin is admin", function () {
+      common.isAdmin("admin").should.equal(true);
+      common.isAdmin("fengmk2").should.equal(true);
+      common.isAdmin("constructor").should.equal(false);
+      common.isAdmin("toString").should.equal(false);
     });
   });
 
-  describe('getCDNKey()', function () {
-    it('should auto fix scope filename', function () {
-      common.getCDNKey('foo', 'foo-1.0.0.tgz').should.equal('/foo/-/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', 'foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', '@bar/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar/foo-1.0.0.tgz');
-      common.getCDNKey('@bar/foo', '@bar1/foo-1.0.0.tgz').should.equal('/@bar/foo/-/@bar1/foo-1.0.0.tgz');
+  describe("getCDNKey()", function () {
+    it("should auto fix scope filename", function () {
+      common
+        .getCDNKey("foo", "foo-1.0.0.tgz")
+        .should.equal("/foo/-/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "@bar/foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar/foo-1.0.0.tgz");
+      common
+        .getCDNKey("@bar/foo", "@bar1/foo-1.0.0.tgz")
+        .should.equal("/@bar/foo/-/@bar1/foo-1.0.0.tgz");
+    });
+  });
+
+  describe("isLocalModule", function () {
+    it("should ignore private packages", function * () {
+      yield createModule("banana", "1.0.0");
+      const modules = yield packageService.listModulesByName('banana');
+      mm(config, "privatePackages", ["banana"]);
+      common.isLocalModule(modules).should.equal(true);
     });
   });
 });

test/mocks/package/koa/default.js

@@ -5,7 +5,7 @@ module.exports = {
     name: 'koa',
     version: '0.13.0',
     description: 'description: Koa web app framework',
-    registryUrl: 'https://registry.npm.taobao.org/koa',
+    registryUrl: 'https://registry.npmmirror.com/koa',
     engines: {
       node: {
         version: '>= 0.11.13',

test/services/blocklist.test.js

@@ -0,0 +1,28 @@
+'use strict';
+
+const assert = require('assert');
+const blocklist = require('../../services/blocklist');
+
+describe('test/services/blocklist.test.js', () => {
+  describe('blockPackageVersion()', () => {
+    it('should block one package version ', function* () {
+      yield blocklist.blockPackageVersion('test-block-name-other', '1.0.0', 'only for test');
+
+      yield blocklist.blockPackageVersion('test-block-name', '1.0.0', 'only for test');
+      const blocks1 = yield blocklist.findBlockPackageVersions('test-block-name');
+      assert(Object.keys(blocks1).length === 1);
+      assert(blocks1['1.0.0'].reason === 'only for test');
+      // block again
+      yield blocklist.blockPackageVersion('test-block-name', '1.0.0', 'only for test new');
+      const blocks2 = yield blocklist.findBlockPackageVersions('test-block-name');
+      assert(Object.keys(blocks2).length === 1);
+      assert(blocks2['1.0.0'].reason === 'only for test new');
+
+      // block all versions
+      yield blocklist.blockPackageVersion('test-block-name', '*', 'only for test all');
+      const blocks3 = yield blocklist.findBlockPackageVersions('test-block-name');
+      assert(Object.keys(blocks3).length === 2);
+      assert(blocks3['*'].reason === 'only for test all');
+    });
+  });
+});

test/services/common.test.js

@@ -1,19 +1,5 @@
-/**!
- * cnpmjs.org - test/services/common.test.js
- *
- * Copyright(c) fengmk2 and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var mm = require('mm');
 var config = require('../../config');
 var common = require('../../services/common');

test/services/default_user_service.test.js

@@ -1,17 +1,5 @@
-/**
- * Copyright(c) cnpm and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var should = require('should');
 var mm = require('mm');
 var User = require('../../models').User;
@@ -89,7 +77,7 @@ describe('services/default_user_service.test.js', function () {
       });
     });
 
-    it('should get a npm sync user by login name', function* () {
+    it.skip('should get a npm sync user by login name', function* () {
       var user = yield userService.get('fengmk2');
       should.exist(user);
       user.should.eql({
@@ -113,7 +101,7 @@ describe('services/default_user_service.test.js', function () {
   describe('list()', function () {
     it('should return all exists users', function* () {
       var users = yield userService.list(['cnpmjstest10', 'fengmk2', 'cnpmjstest101']);
-      users.should.length(3);
+      users.should.length(2);
     });
 
     it('should return some exists users', function* () {

test/services/package.test.js

@@ -5,6 +5,8 @@ var sleep = require('co-sleep');
 var Package = require('../../services/package');
 var utils = require('../utils');
 var common = require('../../services/common');
+var config = require('../../config');
+var mm = require('mm');
 
 describe('test/services/package.test.js', function () {
   describe('addModuleTag()', function () {
@@ -130,6 +132,71 @@ describe('test/services/package.test.js', function () {
     });
   });
 
+  describe('listModelSince()', function () {
+    it('list tags since', function* () {
+      mm(config, 'changesDelay', 0);
+      yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
+      yield sleep(2100);
+      var start = Date.now() - 1000;
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.0');
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta');
+      yield utils.createModule('test-listModuleSince-module-2', '1.0.0');
+      var tags = yield Package.listTagSince(start);
+
+      var modules = tags.map(function (item) {
+        return { name: item.id, tag: item.changes[0].tag };
+      });
+
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", tag: "latest" },
+          { name: "test-listModuleSince-module-1", tag: "beta" },
+          { name: "test-listModuleSince-module-2", tag: "latest" },
+        ]);
+
+      tags = yield Package.listTagSince(start, 2);
+      modules = tags.map(function (item) {
+        return { name: item.id, tag: item.changes[0].tag};
+      });
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", tag: "latest" },
+          { name: "test-listModuleSince-module-1", tag: "beta" },
+        ]);
+    });
+    it('list package version since', function* () {
+      mm(config, 'changesDelay', 0);
+      yield utils.createModule('test-listModuleSince-module-0', '1.0.0');
+      yield sleep(2100);
+      var start = Date.now() - 1000;
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.0');
+      yield utils.createModule('test-listModuleSince-module-1', '1.0.1', null, 'beta');
+      yield utils.createModule('test-listModuleSince-module-2', '1.0.0');
+      var tags = yield Package.listVersionSince(start);
+
+      var modules = tags.map(function (item) {
+        return { name: item.id, version: item.changes[0].version };
+      });
+
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", version: "1.0.0" },
+          { name: "test-listModuleSince-module-1", version: "1.0.1" },
+          { name: "test-listModuleSince-module-2", version: "1.0.0" },
+        ]);
+
+      tags = yield Package.listVersionSince(start, 2);
+      modules = tags.map(function (item) {
+        return { name: item.id, version: item.changes[0].version};
+      });
+      modules
+        .should.eql([
+          { name: "test-listModuleSince-module-1", version: "1.0.0" },
+          { name: "test-listModuleSince-module-1", version: "1.0.1" },
+        ]);
+    });
+  });
+
   describe('listPublicModuleNamesSince(), listAllPublicModuleNames()', function () {
     it('should got those module names', function* () {
       yield utils.createModule('test-listPublicModuleNamesSince-module-0', '1.0.0');

test/sync/sync_exist.test.js

@@ -1,19 +1,5 @@
-/*!
- * cnpmjs.org - test/sync/sync_exist.test.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var mm = require('mm');
 var config = require('../../config');
 var sync = require('../../sync/sync_exist');
@@ -34,37 +20,45 @@ describe('test/sync/sync_exist.test.js', function () {
     });
 
     it('should sync first time ok', function *() {
-      mm.data(npmService, 'getShort', ['byte']);
+      mm.data(npmService, 'listChanges', [
+        {
+          seq: 1,
+          id: 'byte',
+        }
+      ]);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: 0});
       var data = yield sync();
       data.successes[0].should.equal('byte');
     });
 
     it('should sync common ok', function *() {
-      mm.data(npmService, 'getAllSince', {
-        _updated: Date.now(),
-        byte: {},
-      });
-      mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
-      var data = yield sync();
-      data.successes[0].should.equal('byte');
-
-      mm.data(npmService, 'getAllSince', []);
-      var data = yield sync();
-      data.successes.should.eql([]);
-    });
-
-    it('should sync with array format data', function *() {
-      mm.data(npmService, 'getAllSince', [
+      mm.data(npmService, 'listChanges', [
         {
-          name: 'byte',
+          seq: 2,
+          id: 'byte',
         }
       ]);
       mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield sync();
       data.successes[0].should.equal('byte');
 
-      mm.data(npmService, 'getAllSince', []);
+      mm.data(npmService, 'listChanges', []);
+      var data = yield sync();
+      data.successes.should.eql([]);
+    });
+
+    it('should sync with array format data', function *() {
+      mm.data(npmService, 'listChanges', [
+        {
+          seq: 3,
+          id: 'byte',
+        }
+      ]);
+      mm.data(totalService, 'getTotalInfo', {last_exist_sync_time: Date.now()});
+      var data = yield sync();
+      data.successes[0].should.equal('byte');
+
+      mm.data(npmService, 'listChanges', []);
       var data = yield sync();
       data.successes.should.eql([]);
     });

view/web/package.html

@@ -20,7 +20,9 @@
           <% for (var k in package.engines) {
             var engine = package.engines[k];
           %>
+            <% if (engine.badgeURL) { %>
             <img title="<%= engine.title %>" src="<%- engine.badgeURL %>">
+            <% } %>
         <% }
         } %>
         <% if (package.snyk) { %>
@@ -44,7 +46,7 @@
             </div>
         <% } %>
 
-        <% if (package.license) { %>
+        <% if (package.license && package.license.url) { %>
           <a href="<%= package.license.url %>" target="_blank"><%= package.license.name %></a>
         <% } %>
 
@@ -53,12 +55,12 @@
           <a href="<%= package.repository.weburl %>" target="_blank">Repository</a>
         <% } %>
 
-        <% if (package.bugs.url) { %>
+        <% if (package.bugs && package.bugs.url) { %>
           <% if (package.license || package.repository) {%> &middot; <% } %>
           <a href="<%= package.bugs.url %>" target="_blank">Bugs</a>
         <% } %>
 
-        <% if (package.license || package.repository || package.bugs.url) {%> &middot; <% } %>
+        <% if (package.license || package.repository || (package.bugs && package.bugs.url)) {%> &middot; <% } %>
 
         <% if (!package.isPrivate) { %>
         <a href="https://www.npmjs.org/package/<%= package.name %>/v/<%= package.version %>" target="_blank">Original npm</a>
@@ -159,37 +161,44 @@
       </div>
     </div>
 
+    <% if (download) { %>
     <div class="pack-downloads cols-box-item">
       <div class="cols-box-title">
         Downloads
       </div>
       <div class="downloads">
+        <div class="row">
+          Total
+          <span class="pack-download-num"><%= download.total %></span>
+        </div>
         <div class="row">
           Today
           <span class="pack-download-num"><%= download.today %></span>
-          </div>
-      <div class="row">
+        </div>
+        <div class="row">
           This Week
           <span class="pack-download-num"><%= download.thisweek %></span>
-          </div>
-      <div class="row">
+        </div>
+        <div class="row">
           This Month
           <span class="pack-download-num"><%= download.thismonth %></span>
-          </div>
-      <div class="row">
+        </div>
+        <div class="row">
           Last Day
           <span class="pack-download-num"><%= download.lastday %></span>
-          </div>
-      <div class="row">
+        </div>
+        <div class="row">
           Last Week
           <span class="pack-download-num"><%= download.lastweek %></span>
-          </div>
-      <div class="row">
+        </div>
+        <div class="row">
           Last Month
           <span class="pack-download-num"><%= download.lastmonth %></span>
         </div>
       </div>
     </div>
+    <% } %>
+
     <div class="pack-dep cols-box-item">
       <div class="cols-box-title">
         <%
@@ -239,6 +248,7 @@
       <% } %>
     </div>
 
+    <% if (package.dependents) { %>
     <div class="pack-dep cols-box-item">
       <div class="cols-box-title">
         <%
@@ -262,6 +272,7 @@
         None
       <% } %>
     </div>
+    <% } %>
 
   </div>
 </div>

view/web/scope_sync.html

@@ -0,0 +1,124 @@
+<div id="scopeSync">
+  <h2>Scope Packages Sync</h2>
+  <div>
+    <div><%= scope %></div>
+    <div>source registry ：<%= sourceCnpmRegistry %></div>
+    <div>scope package number：<%= packages.length %> </div>
+    <div id="last-sync-time">Last Sync Time: </div>
+    <button id='start-button' onclick="startAllSync()">Start sync all scope packages</button>
+  </div>
+  <ul>
+    <% for(var i=0;i<packages.length;i++){ %>
+      <li id="sync-result-<%= packages[i] %>"><%= packages[i] %>： wait sync</li>
+    <% } %>
+  </ul>
+  <h2>Log</h2>
+  <div id="sync-logs">
+    <% for(var i=0;i<packages.length;i++){ %>
+      <h2>Sync <span style="color:#09f;"><%= packages[i] %></span></h2>
+      <div id="sync-notify-<%= packages[i] %>">
+        <div class="ant-alert ant-alert-success">Sync started, please wait patiently.</div>
+      </div>
+      <pre style="min-height: 400px;" id="sync-log-<%= packages[i] %>"></pre>
+    <% } %>
+  </div>
+</div>
+<script>
+  var scope = '<%= scope %>';
+  var packages = '<%= packages %>';
+  var excutePackages = packages.split(',').map(function (name) {
+    var logId = localStorage.getItem(name)
+    var obj = {
+      name: name,
+      logTimer: '',
+      logId: logId,
+      syncDone: false,
+      hasFail: false
+    }
+    logId && getSyncLog(obj)
+
+    return obj
+  })
+
+  var lastSyncStartTime = localStorage.getItem(`${scope}_last_sync_start_time`)
+  lastSyncStartTime && $(document.getElementById('last-sync-time')).html('<div class="ant-alert ant-alert-success">本浏览器上次同步时间为：' + lastSyncStartTime + '</div>')
+
+  function startAllSync () {
+    if (lastSyncStartTime) {
+      if (!window.confirm(`laste sync time is ${lastSyncStartTime}，are you sure restart ?`)) return
+    }
+
+    $.ajax({
+        url: `/scopeSync/${scope}`,
+        type: 'PUT',
+        dataType: 'json',
+        success: function (data) {
+          if (!data.ok) {
+            console.log(data)
+            alert('Sync request error.')
+            return
+          }
+          localStorage.setItem(`${scope}_last_sync_start_time`, new Date())
+
+          data.logIds.forEach(({ name, logId }) => {
+            localStorage.setItem(name, logId)
+            $(document.getElementById('sync-result-' + name)).html(name + '---------------- Syncing ----------------')
+            var obj = excutePackages.find(obj => obj.name === name)
+            obj.logId = logId
+            obj.logTimer = setInterval(function () { getSyncLog(obj) }, 2000);
+          })
+        },
+        error: function (err) {
+          console.log(err)
+          alert(JSON.stringify(err))
+        }
+      });
+  }
+
+  function getSyncLog(obj) {
+    var name = obj.name
+    var id = obj.logId
+    var timer = obj.logTimer
+    var $log = $(document.getElementById('sync-log-' + name));
+    var $notify = $(document.getElementById('sync-notify-' + name));
+    var $syncResult = $(document.getElementById('sync-result-' + name));
+
+    $.ajax({
+      url: '/sync/' + name + '/log/' + id,
+      type: 'GET',
+      dataType: 'json',
+      success: function (data) {
+        if (!data.ok) {
+          return;
+        }
+
+        var log = data.log || '';
+
+        obj.syncDone = data.syncDone;
+
+        if (log.indexOf('Fail: [') >= 0) {
+          var failInfo = log.match(/Fail: \[ (.*?) \]/);
+          if (failInfo && failInfo[1]) {
+            obj.hasFail = true;
+          }
+        }
+        if (obj.syncDone) {
+          log += '\nSync ' + name + ' complete!';
+          if (obj.hasFail) {
+            log += ' scome package sync faile，you can try cleae localStorage and restart sync';
+            location.hash = '';
+            $notify.html('<div class="ant-alert ant-alert-error">～～～～～～sync failed～～～～～～</div>');
+            $syncResult.html('<div style="color: red">' + name + '：Sync failed!!!!!!!!!!!!!' + '</div>')
+          } else {
+            $notify.html('<div class="ant-alert ant-alert-success">Sync success.</div>');
+            $syncResult.html('<div style="color: #4deb4d">' + name + '：Sync success' + '</div>')
+          }
+          clearInterval(timer);
+        } else {
+          $syncResult.html(name + '---------------- Syncing ----------------')
+        }
+        $log.html(log);
+      }
+    });
+  }
+</script>

view/web/sync.html

@@ -15,9 +15,13 @@
   if (type !== 'package') {
     name = type + ':' + name;
   }
+  <% if (syncTaskUrl) { %>
+  var resourceURL = '<%= syncTaskUrl %>';
+  <% } else { %>
   var resourceURL = '/sync/' + name;
+  <% } %>
   $(function() {
-    var checkLogId = location.hash.match(/logid=(\d+)/);
+    var checkLogId = location.hash.match(/logid=(\w+)/);
     var logid = checkLogId ? checkLogId[1] : '';
     if (logid) {
       return getSyncLog(logid);
@@ -26,6 +30,7 @@
       url: resourceURL + '?sync_upstream=true',
       type: 'PUT',
       dataType: 'json',
+      crossDomain: true,
       success: handleSyncSucess,
       error: function (err) {
         var alert = $('<div class="ant-alert ant-alert-error"></div>');
@@ -57,6 +62,7 @@
     $.ajax({
       url: resourceURL + '/log/' + id,
       type: 'GET',
+      crossDomain: true,
       dataType: 'json',
       success: function (data) {
         if (!data.ok) {
@@ -72,12 +78,13 @@
         }
         if (syncDone) {
           log += '\nSync ' + name + ' complete!';
+          var logLink = data.logUrl ? (' <a href="' + data.logUrl + '" target="_blank">' + data.logUrl + '</a>') : '';
           if (hasFail) {
             log += ' But some packages sync failed, you can refresh to sync again.';
             location.hash = '';
-            $notify.html('<div class="ant-alert ant-alert-error">Sync failed.</div>');
+            $notify.html('<div class="ant-alert ant-alert-error">Sync failed.' + logLink + '</div>');
           } else {
-            $notify.html('<div class="ant-alert ant-alert-success">Sync success.</div>');
+            $notify.html('<div class="ant-alert ant-alert-success">Sync success.' + logLink + '</div>');
           }
           clearInterval(timer);
         }