Release 3.0.0-rc.66

> 包名超长时，更新下载记录接口会无限重试
* 更新下载记录前，先判断当前包名是否存在

-------------

> When the package name is too long, the update download record
interface will retry indefinitely.

* Check pkg exists, before updating the download record.

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+npm-debug.log

.editorconfig

@@ -0,0 +1,17 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+

.github/workflows/nodejs.yml

@@ -0,0 +1,52 @@
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: true
+          MYSQL_DATABASE: cnpmjs_test
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [14, 16]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout Git Source
+      uses: actions/checkout@v2
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Dependencies
+      run: npm i -g npminstall && npminstall
+
+    - name: Continuous Integration
+      run: npm run ci
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -7,6 +7,7 @@ coverage.html
 *.pid
 *.gz
 dump.rdb
+.DS_Store
 
 pids
 logs
@@ -15,7 +16,17 @@ results
 node_modules
 npm-debug.log
 public/dist/
+.dist
 config/config.js
 backup/*.json
 backup/*.gz
 docs/web/history.md
+docs/web/_readme.md
+view/web/_layout.html
+bin/mysql.js
+bin/test.sql
+coverage/
+config/web_readme.md
+.tmp/
+*.sqlite
+

.jshintignore

@@ -0,0 +1,7 @@
+node_modules/
+coverage/
+.tmp/
+.git/
+tools/
+public/
+test/

.jshintrc

@@ -0,0 +1,95 @@
+{
+    // JSHint Default Configuration File (as on JSHint website)
+    // See http://jshint.com/docs/ for more details
+
+    "maxerr"        : 50,       // {int} Maximum error before stopping
+
+    // Enforcing
+    "bitwise"       : true,     // true: Prohibit bitwise operators (&, |, ^, etc.)
+    "camelcase"     : false,    // true: Identifiers must be in camelCase
+    "curly"         : true,     // true: Require {} for every new block or scope
+    "eqeqeq"        : true,     // true: Require triple equals (===) for comparison
+    "forin"         : false,    // true: Require filtering for..in loops with obj.hasOwnProperty()
+    "immed"         : false,    // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
+    "indent"        : false,    // {int} Number of spaces to use for indentation
+    "latedef"       : false,    // true: Require variables/functions to be defined before being used
+    "newcap"        : false,    // true: Require capitalization of all constructor functions e.g. `new F()`
+    "noarg"         : true,     // true: Prohibit use of `arguments.caller` and `arguments.callee`
+    "noempty"       : true,     // true: Prohibit use of empty blocks
+    "nonew"         : false,    // true: Prohibit use of constructors for side-effects (without assignment)
+    "plusplus"      : false,    // true: Prohibit use of `++` & `--`
+    "quotmark"      : false,    // Quotation mark consistency:
+                                //   false    : do nothing (default)
+                                //   true     : ensure whatever is used is consistent
+                                //   "single" : require single quotes
+                                //   "double" : require double quotes
+    "undef"         : true,     // true: Require all non-global variables to be declared (prevents global leaks)
+    "unused"        : true,     // true: Require all defined variables be used
+    "strict"        : true,     // true: Requires all functions run in ES5 Strict Mode
+    "trailing"      : false,    // true: Prohibit trailing whitespaces
+    "maxparams"     : false,    // {int} Max number of formal params allowed per function
+    "maxdepth"      : false,    // {int} Max depth of nested blocks (within functions)
+    "maxstatements" : false,    // {int} Max number statements per function
+    "maxcomplexity" : false,    // {int} Max cyclomatic complexity per function
+    "maxlen"        : false,    // {int} Max number of characters per line
+
+    // Relaxing
+    "asi"           : false,     // true: Tolerate Automatic Semicolon Insertion (no semicolons)
+    "boss"          : true,      // true: Tolerate assignments where comparisons would be expected
+    "debug"         : false,     // true: Allow debugger statements e.g. browser breakpoints.
+    "eqnull"        : false,     // true: Tolerate use of `== null`
+    "es5"           : false,     // true: Allow ES5 syntax (ex: getters and setters)
+    "esnext"        : true,      // true: Allow ES.next (ES6) syntax (ex: `const`)
+    "moz"           : false,     // true: Allow Mozilla specific syntax (extends and overrides esnext features)
+                                 // (ex: `for each`, multiple try/catch, function expression…)
+    "evil"          : false,     // true: Tolerate use of `eval` and `new Function()`
+    "expr"          : true,      // true: Tolerate `ExpressionStatement` as Programs
+    "funcscope"     : false,     // true: Tolerate defining variables inside control statements"
+    "globalstrict"  : false,     // true: Allow global "use strict" (also enables 'strict')
+    "iterator"      : false,     // true: Tolerate using the `__iterator__` property
+    "lastsemic"     : false,     // true: Tolerate omitting a semicolon for the last statement of a 1-line block
+    "laxbreak"      : true,      // true: Tolerate possibly unsafe line breakings
+    "laxcomma"      : false,     // true: Tolerate comma-first style coding
+    "loopfunc"      : false,     // true: Tolerate functions being defined in loops
+    "multistr"      : true,      // true: Tolerate multi-line strings
+    "proto"         : false,     // true: Tolerate using the `__proto__` property
+    "scripturl"     : false,     // true: Tolerate script-targeted URLs
+    "smarttabs"     : false,     // true: Tolerate mixed tabs/spaces when used for alignment
+    "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
+    "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
+    "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
+    "validthis"     : true,      // true: Tolerate using this in a non-constructor function
+
+    // Environments
+    "browser"       : true,     // Web Browser (window, document, etc)
+    "couch"         : false,    // CouchDB
+    "devel"         : true,     // Development/debugging (alert, confirm, etc)
+    "dojo"          : false,    // Dojo Toolkit
+    "jquery"        : false,    // jQuery
+    "mootools"      : false,    // MooTools
+    "node"          : true,     // Node.js
+    "nonstandard"   : false,    // Widely adopted globals (escape, unescape, etc)
+    "prototypejs"   : false,    // Prototype and Scriptaculous
+    "rhino"         : false,    // Rhino
+    "worker"        : false,    // Web Workers
+    "wsh"           : false,    // Windows Scripting Host
+    "yui"           : false,    // Yahoo User Interface
+    "noyield"       : true,     // allow generators without a yield
+
+    // Legacy
+    "nomen"         : false,    // true: Prohibit dangling `_` in variables
+    "onevar"        : false,    // true: Allow only one `var` statement per function
+    "passfail"      : false,    // true: Stop on first error
+    "white"         : false,    // true: Check against strict whitespace and indentation rules
+
+    // Custom Globals
+    "globals"       : {         // additional predefined global variables
+      // mocha
+      "describe": true,
+      "it": true,
+      "before": true,
+      "afterEach": true,
+      "beforeEach": true,
+      "after": true
+    }
+}

.node-dev.json

@@ -0,0 +1,3 @@
+{
+  "notify": false
+}

.npmignore

@@ -8,3 +8,18 @@ logo.png
 public/dist/
 backup/*.json
 backup/*.gz
+docs/web/history.md
+docs/web/_readme.md
+view/web/_layout.html
+bin/mysql.js
+bin/test.sql
+coverage/
+.jshintrc
+.jshintignore
+.DS_Store
+config/web_readme.md
+.dist/
+config/config.js
+*.sqlite
+.node-dev.json
+nohup.out

.travis.yml

@@ -1,5 +0,0 @@
-language: node_js
-node_js:
-  - '0.10'
-install: npm install --registry=http://registry.cnpmjs.org
-script: make test-coveralls

AUTHORS

@@ -1,7 +1,12 @@
 # Ordered by date of first contribution.
-# Auto-generated by 'contributors' on Fri, 20 Dec 2013 09:33:09 GMT.
-# https://github.com/xingrz/node-contributors
 
 fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)
-dead_horse <dead_horse@qq.com> (https://github.com/dead-horse)
-AlsoTang <alsotang@gmail.com> (https://github.com/alsotang)
+dead-horse <dead_horse@qq.com> (https://github.com/dead-horse)
+alsotang <alsotang@gmail.com> (https://github.com/alsotang)
+4simple <wondger@qq.com> (https://github.com/4simple)
+afc163 <afc163@gmail.com> (https://github.com/afc163)
+Yuwei Ba <i@xiaoba.me> (https://github.com/ibigbug)
+dickeylth <dickeylth@gmail.com> (https://github.com/dickeylth)
+wenbing <wenbing@users.noreply.github.com> (https://github.com/wenbing)
+21paradox <1036339815@qq.com> (https://github.com/21paradox)
+

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# How to contribute
+
+Third-party patches are essential for keeping `cnpmjs.org` great.
+We want to keep it as easy as possible to contribute changes that
+get things working in your environment. There are a few guidelines that we
+need contributors to follow so that we can have a chance of keeping on
+top of things.
+
+## Getting Started
+
+* Make sure you have a [GitHub account](https://github.com/signup/free)
+* Fork the repository on GitHub
+
+## Making Changes
+
+* Create a topic branch from where you want to base your work.
+  * This is usually the master branch.
+  * Only target release branches if you are certain your fix must be on that
+    branch.
+  * To quickly create a topic branch based on master.
+    Please avoid working directly on the `master` branch.
+* Make commits of logical units and including unit tests.
+* Check for unnecessary whitespace with `git diff --check` before committing.
+* Make sure your commit messages are in the proper format.
+* Make sure you have added the necessary tests for your changes.
+* Run _all_ the tests to assure nothing else was accidentally broken.
+* Follow [node style guide](https://github.com/felixge/node-style-guide)
+
+## Submitting Changes
+
+* Push your changes to a topic branch in your fork of the repository.
+* Submit a pull request.
+* Make sure travis-ci test pass.
+
+# Additional Resources
+
+* [General GitHub documentation](http://help.github.com/)
+* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
+* [cnpmjs.org](http://cnpmjs.org)

Dockerfile

@@ -0,0 +1,28 @@
+FROM node:12
+MAINTAINER Bono Lv <lvscar  {aT} gmail.com>
+
+# Working enviroment
+ENV \
+    CNPM_DIR="/var/app/cnpmjs.org" \
+    CNPM_DATA_DIR="/var/data/cnpm_data" 
+
+RUN mkdir  -p ${CNPM_DIR}
+
+WORKDIR ${CNPM_DIR}
+
+COPY package.json ${CNPM_DIR}
+
+RUN npm set registry https://registry.npmmirror.com
+
+RUN npm install --production
+
+COPY .  ${CNPM_DIR}
+COPY docs/dockerize/config.js  ${CNPM_DIR}/config/
+
+EXPOSE 7001/tcp 7002/tcp
+
+VOLUME ["/var/data/cnpm_data"]
+
+# Entrypoint
+CMD ["node", "dispatch.js"]
+

LICENSE.txt

@@ -1,6 +1,6 @@
 This software is licensed under the MIT License.
 
-Copyright(c) cnpmjs.org and other contributors.
+Copyright(c) cnpm and other contributors.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,30 +1,105 @@
 TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
-REPORTER = tap
-TIMEOUT = 30000
+REPORTER = spec
+TIMEOUT = 600000
 MOCHA_OPTS =
+DB = sqlite
 
-install:
-	@npm install --registry=http://registry.cnpmjs.org --cache=${HOME}/.npm/.cache/cnpm
+jshint:
+	@node_modules/.bin/jshint .
 
-test: install
-	@NODE_ENV=test ./node_modules/mocha/bin/mocha \
+init-database:
+	@NODE_ENV=test node test/init_db.js
+
+init-mysql:
+	@mysql -uroot -h 127.0.0.1 --port 3306 -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@mysql -uroot -h 127.0.0.1 --port 3306 -e 'CREATE DATABASE cnpmjs_test;'
+
+init-pg:
+	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@psql -c 'CREATE DATABASE cnpmjs_test;'
+
+test: init-database
+	@NODE_ENV=test DB=${DB} node_modules/.bin/mocha \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
+		--require should \
+		--require thunk-mocha \
+		--require ./test/init.js \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
-test-cov:
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
+test-sqlite:
+	@$(MAKE) test DB=sqlite
 
-test-cov-html:
-	@rm -f coverage.html
-	@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
-	@ls -lh coverage.html
+test-mysql: init-mysql
+	@$(MAKE) test DB=mysql
 
-test-coveralls: test
-	@echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
-	@-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
+test-pg: init-pg
+	@DB_PORT=5432 DB_USER=$(USER) $(MAKE) test DB=postgres
 
-test-all: test test-cov
+test-all: test-sqlite test-mysql
+
+test-cov cov: init-database
+	@NODE_ENV=test DB=${DB} node \
+		node_modules/.bin/istanbul cover \
+		node_modules/.bin/_mocha \
+		-- -u exports \
+		--reporter $(REPORTER) \
+		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
+		--require should \
+		--require thunk-mocha \
+		--require ./test/init.js \
+		$(MOCHA_OPTS) \
+		$(TESTS)
+
+test-cov-sqlite:
+	@$(MAKE) test-cov DB=sqlite
+
+test-cov-mysql: init-mysql
+	@$(MAKE) test-cov DB=mysql
+
+test-travis: init-database
+	@NODE_ENV=test DB=${DB} \
+		node \
+		node_modules/.bin/istanbul cover \
+		node_modules/.bin/_mocha \
+		-- -u exports \
+		--reporter dot \
+		--timeout $(TIMEOUT) \
+		--require intelli-espower-loader \
+		--require should \
+		--require thunk-mocha \
+		--require ./test/init.js \
+		$(MOCHA_OPTS) \
+		$(TESTS)
+
+test-travis-sqlite:
+	@$(MAKE) test-travis DB=sqlite
+
+test-travis-mysql: init-mysql
+	@$(MAKE) test-travis DB=mysql
+
+test-travis-pg:
+	@psql -c 'DROP DATABASE IF EXISTS cnpmjs_test;' -U postgres
+	@psql -c 'CREATE DATABASE cnpmjs_test;' -U postgres
+	@DB_PORT=5432 DB_USER=postgres $(MAKE) test-travis DB=postgres
+
+test-travis-all: jshint test-travis-sqlite test-travis-mysql test-travis-pg
+
+dev:
+	@NODE_ENV=development node node_modules/.bin/node-dev dispatch.js
+
+contributors:
+	@node_modules/.bin/contributors -f plain -o AUTHORS
+
+autod:
+	@node_modules/.bin/autod -w \
+		--registry https://r.cnpmjs.org \
+		--prefix "^" \
+		--exclude public,view,docs,backup,coverage \
+		--dep mysql \
+		--keep should,supertest,chunkstream,mm,pedding
 
 .PHONY: test

README.md

@@ -1,69 +1,188 @@
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+# ‼️ ‼️ ‼️ ‼️ DEPRECATED, please use https://github.com/cnpm/cnpmcore instead ‼️ ‼️ ‼️ ‼️
+
+------
+
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org)[![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
+[![npm version][npm-image]][npm-url]
+[![Node.js CI](https://github.com/cnpm/cnpmjs.org/actions/workflows/nodejs.yml/badge.svg)](https://github.com/cnpm/cnpmjs.org/actions/workflows/nodejs.yml)
+[![Test coverage][codecov-image]][codecov-url]
+[![Known Vulnerabilities][snyk-image]][snyk-url]
+[![npm download][download-image]][download-url]
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org?ref=badge_shield)
 
-[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
+[npm-image]: http://cnpmjs.org/badge/v/cnpmjs.org.svg?style=flat-square
+[npm-url]: http://cnpmjs.org/package/cnpmjs.org
+[codecov-image]: https://codecov.io/gh/cnpm/cnpmjs.org/branch/master/graph/badge.svg
+[codecov-url]: https://codecov.io/gh/cnpm/cnpmjs.org
+[snyk-image]: https://snyk.io/test/npm/cnpmjs.org/badge.svg?style=flat-square
+[snyk-url]: https://snyk.io/test/npm/cnpmjs.org
+[download-image]: https://img.shields.io/npm/dm/cnpmjs.org.svg?style=flat-square
+[download-url]: https://npmjs.org/package/cnpmjs.org
 
-![logo](https://raw.github.com/fengmk2/cnpmjs.org/master/logo.png)
+![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
-## What is this?
+## Description
 
-Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
+Private npm registry and web for Enterprise, base on [koa](http://koajs.com/),
+MySQL and [Simple Store Service](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
 
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
+Our goal is to provide a low cost maintenance, easy to use, and easy to scale solution for private npm.
 
-![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
+## What can you do with `cnpmjs.org`?
 
-## Install
+* Build a private npm for your own enterprise. ([alibaba](http://www.alibaba.com/) is using `cnpmjs.org` now)
+* Build a npm mirror. (we use it to build a mirror in China: [https://npmmirror.com/](https://npmmirror.com/))
+* Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
+
+## Features
+
+* **Support "scoped" packages**: [npm/npm#5239](https://github.com/npm/npm/issues/5239)
+* **Support [CORS](http://en.wikipedia.org/wiki/Cross-origin_resource_sharing)**
+* **Simple to deploy**: only need `mysql` and a [simple store system](https://github.com/cnpm/cnpmjs.org/wiki/NFS-Guide).
+* **Low cost and easy maintenance**: `package.json` info can store in MySQL, MariaDB, SQLite or PostgreSQL.
+tarball(tgz file) can store in Amazon S3 or other object storage service.
+* **Automatic synchronization**: automatically sync from any registry specified. support two sync modes:
+  - Sync all modules from upstream
+  - Only sync the modules after first access.
+* **Manual synchronization**: automatic synchronization may has little delay. you can sync manually on web page.
+* **Customized client**: we provide a client [cnpm](https://github.com/cnpm/cnpm)
+to extend `npm` with more features(`sync` command, [gzip](https://github.com/npm/npm-registry-client/pull/40) support).
+And it is easy to wrap for your own registry which build with `cnpmjs.org`.
+* **Compatible with npm client**: you can use the official npm client with `cnpmjs.org`.
+you only need to change the registry in client config.
+* **Support http_proxy**: if you're behind a firewall, you can provide a http proxy for cnpmjs.org.
+
+## Docs
+
+* [How to deploy](https://github.com/cnpm/cnpmjs.org/wiki/Deploy)
+* cnpm client: [cnpm](https://github.com/cnpm/cnpm), `npm install -g cnpm`
+* [Sync packages through `http_proxy`](https://github.com/cnpm/cnpmjs.org/wiki/Sync-packages-through-http_proxy)
+* [Migrating from 1.x to 2.x](https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x)
+* [New features in 2.x](https://github.com/cnpm/cnpmjs.org/wiki/New-features-in-2.x).
+* [wiki](https://github.com/cnpm/cnpmjs.org/wiki)
+
+## Develop on your local machine
+
+### Dependencies
+
+* [node](http://nodejs.org) >= 8.0.0
+* Databases: only required one type
+  * [sqlite3](https://npmmirror.com/package/sqlite3) >= 3.0.2, we use `sqlite3` by default
+  * [MySQL](http://dev.mysql.com/downloads/) >= 5.6.16, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
+  * MariaDB
+  * PostgreSQL
+
+### Clone code and run test
 
 ```bash
-$ npm install
+# clone from git
+$ git clone https://github.com/cnpm/cnpmjs.org.git
+
+# install dependencies
+$ make install
+
+# test
+$ make test
+
+# coverage
+$ make test-cov
+
+# update dependencies
+$ make autod
+
+# start server with development mode
+$ make dev
 ```
 
-## Usage
+### Dockerized cnpmjs.org Installation Guide
 
-```js
-$ node dispatch.js
-```
+Cnpmjs.org shipped with a simple but pragmatic Docker Compose configuration.With the configuration, you can set up a MySQL backend cnpmjs.org instance by executing just one command on Docker installed environment.
 
-## Authors
+#### Preparation
+
+* [Install Docker](https://www.docker.com/community-edition)
+* [Install Docker Compose](https://docs.docker.com/compose/install/) (Docker for Mac, Docker for Windows include Docker Compose, so most Mac and Windows users do not need to install Docker Compose separately)
+* (Optional) Speed up Docker images downloading by setting up [Docker images download accelerator](https://yq.aliyun.com/articles/29941)
+
+
+#### Dockerized cnpmjs.org control command
+
+Make sure your current working directory is the root of this GitHub repository.
+
+##### Run dockerized cnpmjs.org
 
 ```bash
-$ git summary
+ $docker-compose up
+ ```
 
- project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
- authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
+This command will build a Docker image using the current code of repository. Then set up a dockerized MySQL instance with data initialized. After Docker container running, you can access your cnpmjs.org web portal at http://127.0.0.1:7002 and npm register at http://127.0.0.1:7001.
+
+#### Run cnpmjs.org in the backend
+
+```bash
+$docker-compose up -d
 ```
 
+#### Rebuild cnpmjs.org Docker image
+
+```bash
+$docker-compose build
+```
+
+#### Remove current dockerized cnpmjs.org instance
+
+The current configuration set 2 named Docker Volume for your persistent data. If you haven't change the repository directory name, them will be "cnpmjsorg_cnpm-files-volume" & "cnpmjsorg_cnpm-db-volume".
+
+Be Careful, the following commands will remove them.
+
+```bash
+$docker-compose rm
+$docker volume rm cnpmjsorg_cnpm-files-volume
+$docker volume rm cnpmjsorg_cnpm-db-volume
+```
+
+You can get more information about your data volumes using the below commands:
+
+```bash
+$docker volume ls  // list all of your Docker volume
+$docker volume inspect cnpmjsorg_cnpm-files-volume
+$docker volume inspect cnpmjsorg_cnpm-db-volume
+```
+
+## How to contribute
+
+* Clone the project
+* Checkout a new branch
+* Add new features or fix bugs in the new branch
+* Make a pull request and we will review it ASAP
+
+Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
+
+## Sponsors
+
+- [![阿里云](https://static.aliyun.com/images/www-summerwind/logo.gif)](http://click.aliyun.com/m/4288/) [![阿里云云效](https://img.alicdn.com/tfs/TB116yt3fb2gK0jSZK9XXaEgFXa-106-20.png)](https://devops.aliyun.com/?channel=pd_cnpm_github) (2016.2 - now)
+
 ## License
 
-(The MIT License)
+[MIT](LICENSE.txt)
 
-Copyright(c) cnpmjs.org and other contributors.
+<!-- GITCONTRIBUTOR_START -->
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+## Contributors
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/543405?v=4" width="100px;"/><br/><sub><b>ibigbug</b></sub>](https://github.com/ibigbug)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|
+| :---: | :---: | :---: | :---: | :---: | :---: |
+|[<img src="https://avatars.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars.githubusercontent.com/u/327019?v=4" width="100px;"/><br/><sub><b>JacksonTian</b></sub>](https://github.com/JacksonTian)<br/>|[<img src="https://avatars.githubusercontent.com/u/11251401?v=4" width="100px;"/><br/><sub><b>Solais</b></sub>](https://github.com/Solais)<br/>|[<img src="https://avatars.githubusercontent.com/u/1134761?v=4" width="100px;"/><br/><sub><b>4simple</b></sub>](https://github.com/4simple)<br/>|[<img src="https://avatars.githubusercontent.com/u/6622122?v=4" width="100px;"/><br/><sub><b>21paradox</b></sub>](https://github.com/21paradox)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1294440?v=4" width="100px;"/><br/><sub><b>albertZhang2013</b></sub>](https://github.com/albertZhang2013)<br/>|[<img src="https://avatars.githubusercontent.com/u/6111524?v=4" width="100px;"/><br/><sub><b>ankurk91</b></sub>](https://github.com/ankurk91)<br/>|[<img src="https://avatars.githubusercontent.com/u/1935436?v=4" width="100px;"/><br/><sub><b>huangbowen521</b></sub>](https://github.com/huangbowen521)<br/>|[<img src="https://avatars.githubusercontent.com/u/5365267?v=4" width="100px;"/><br/><sub><b>gwenaelp</b></sub>](https://github.com/gwenaelp)<br/>|[<img src="https://avatars.githubusercontent.com/u/324440?v=4" width="100px;"/><br/><sub><b>KidkArolis</b></sub>](https://github.com/KidkArolis)<br/>|[<img src="https://avatars.githubusercontent.com/u/922240?v=4" width="100px;"/><br/><sub><b>tq0fqeu</b></sub>](https://github.com/tq0fqeu)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1587797?v=4" width="100px;"/><br/><sub><b>limianwang</b></sub>](https://github.com/limianwang)<br/>|[<img src="https://avatars.githubusercontent.com/u/900947?v=4" width="100px;"/><br/><sub><b>LoicMahieu</b></sub>](https://github.com/LoicMahieu)<br/>|[<img src="https://avatars.githubusercontent.com/u/1614482?v=4" width="100px;"/><br/><sub><b>paulvi</b></sub>](https://github.com/paulvi)<br/>|[<img src="https://avatars.githubusercontent.com/u/36651530?v=4" width="100px;"/><br/><sub><b>XXBeii</b></sub>](https://github.com/XXBeii)<br/>|[<img src="https://avatars.githubusercontent.com/u/1422472?v=4" width="100px;"/><br/><sub><b>jpuncle</b></sub>](https://github.com/jpuncle)<br/>|[<img src="https://avatars.githubusercontent.com/u/20092391?v=4" width="100px;"/><br/><sub><b>vincentmrlau</b></sub>](https://github.com/vincentmrlau)<br/>|
+[<img src="https://avatars.githubusercontent.com/u/4470552?v=4" width="100px;"/><br/><sub><b>stoneChen</b></sub>](https://github.com/stoneChen)<br/>|[<img src="https://avatars.githubusercontent.com/u/2196373?v=4" width="100px;"/><br/><sub><b>dickeylth</b></sub>](https://github.com/dickeylth)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|[<img src="https://avatars.githubusercontent.com/u/1941756?v=4" width="100px;"/><br/><sub><b>gniavaj</b></sub>](https://github.com/gniavaj)<br/>|[<img src="https://avatars.githubusercontent.com/u/10371891?v=4" width="100px;"/><br/><sub><b>hellojukay</b></sub>](https://github.com/hellojukay)<br/>|[<img src="https://avatars.githubusercontent.com/u/5040076?v=4" width="100px;"/><br/><sub><b>liyangready</b></sub>](https://github.com/liyangready)<br/>
 
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sun Mar 20 2022 09:50:54 GMT+0800`.
+
+<!-- GITCONTRIBUTOR_END -->
+
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmjs.org?ref=badge_large)

backup/dump.js

@@ -1,142 +0,0 @@
-/**!
- * cnpmjs.org - backup/dump.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-/**
- * 1. dump module
- * 2. dump tag
- * 3. dump user
- * 4. total
- * 5. download_total
- */
-
-var path = require('path');
-var fs = require('fs');
-var moment = require('moment');
-var eventproxy = require('eventproxy');
-var util = require('util');
-var zlib = require('zlib');
-var mysql = require('../common/mysql');
-var nfs = require('../common/nfs');
-var config = require('../config');
-
-function dumpTable(name, lastRow, callback) {
-  var sql = 'SELECT * from ' + name + ' WHERE gmt_modified >=? ORDER BY gmt_modified ASC LIMIT 10000;';
-  mysql.query(sql, [lastRow.gmt_modified], function (err, rows) {
-    if (err || rows.length === 0) {
-      return callback(err, rows);
-    }
-    if (rows[0].id === lastRow.id) {
-      rows = rows.slice(1);
-    }
-    callback(null, rows);
-  });
-}
-
-function log() {
-  var str = '[' + moment().format('YYYY-MM-DD HH:mm:ss') + '] ' + util.format.apply(util, arguments);
-  console.log(str);
-}
-
-function syncTable(name, callback) {
-  var datadir = __dirname;
-  var dataFile = path.join(datadir, moment().format('YYYY-MM-DD-HH') + '_' + name + '.json');
-  var lastRowFile = path.join(datadir, name + '_lastdate.json');
-  var lastRow = null;
-  if (fs.existsSync(lastRowFile)) {
-    lastRow = require(lastRowFile);
-    lastRow.gmt_modified = new Date(Date.parse(lastRow.gmt_modified));
-  } else {
-    lastRow = {
-      gmt_modified: new Date('2011-11-11'),
-    };
-  }
-
-  log('getting "%s" since %j', name, lastRow);
-  dumpTable(name, lastRow, function (err, rows) {
-    console.log('[%s] got %d rows', Date(), rows && rows.length || 0);
-    if (err) {
-      return callback(err);
-    }
-
-    if (!rows || rows.length === 0) {
-      return callback();
-    }
-
-    var writeStream = fs.createWriteStream(dataFile, {flags: 'a'});
-    writeStream.once('error', callback);
-    for (var i = 0; i < rows.length; i++) {
-      writeStream.write(JSON.stringify(rows[i]) + '\n');
-    }
-    writeStream.end();
-    writeStream.on('finish', function () {
-      log('append %d rows to %s', rows.length, dataFile);
-      var gzfile = dataFile + '.gz';
-      var gzip = zlib.createGzip();
-      var inp = fs.createReadStream(dataFile);
-      var out = fs.createWriteStream(gzfile);
-      inp.pipe(gzip).pipe(out);
-      out.once('error', callback);
-      out.on('finish', function () {
-        var key = path.join(config.backupFilePrefix, path.basename(gzfile));
-        log('saving %s to %s ...', gzfile, key);
-        nfs.upload(gzfile, {key: key}, function (err, result) {
-          if (err) {
-            return callback(err);
-          }
-
-          lastRow = rows[rows.length - 1];
-          lastRow = {gmt_modified: lastRow.gmt_modified, id: lastRow.id};
-          fs.writeFileSync(lastRowFile, JSON.stringify(lastRow));
-          log('save %s data file to %j, lastrow: %j', name, result, lastRow);
-          callback();
-        });
-      });
-    });
-  });
-}
-
-var ep = eventproxy.create();
-ep.fail(function (err) {
-  log('error: %s', err.stack);
-  process.exit(1);
-});
-
-syncTable('module', ep.done('module'));
-
-ep.on('module', function () {
-  syncTable('tag', ep.done('tag'));
-});
-
-ep.on('tag', function () {
-  syncTable('user', ep.done('user'));
-});
-
-ep.on('user', function () {
-  syncTable('total', ep.done('total'));
-});
-
-ep.on('total', function () {
-  syncTable('download_total', ep.done('download_total'));
-});
-
-ep.on('download_total', function () {
-  ep.emit('finish');
-});
-
-ep.on('finish', function () {
-  log('finish, %d process exit', process.pid);
-  process.exit(0);
-});

bin/change_password.js

@@ -0,0 +1,24 @@
+"use strict";
+// Only support for ./services/DefaultUserService. If you use custom user service, ignore this file.
+// call with:
+// $ node ./bin/change_password.js 'username' 'new_password'
+
+var UserModel = require('../models').User;
+var co = require('co');
+var utility = require('utility');
+
+var username = process.argv[2];
+var newPassword = process.argv[3];
+
+co(function * () {
+  var user = yield UserModel.find({where: {name: username}});
+  var salt = user.salt;
+  console.log(`user original password_sha: ${user.password_sha}`);
+  var newPasswordSha = utility.sha1(newPassword + salt);
+  user.password_sha = newPasswordSha;
+  user = yield user.save();
+  console.log(`change user password successful!! user new password_sha: ${user.password_sha}`);
+  process.exit(0);
+}).catch(function (e) {
+  console.log(e);
+});

bin/cli.js

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+
+/**!
+ * Copyright(c) cnpm and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:cli');
+var program = require('commander');
+var path = require('path');
+var fs = require('fs');
+var mkdirp = require('mkdirp');
+var treekill = require('treekill');
+var version = require('../package.json').version;
+
+function list(val) {
+  return val.split(',');
+}
+
+program
+  .version(version);
+
+program
+  .command('start')
+  .description('start cnpmjs.org server')
+  .option('--admins <admins>', 'set admins', list)
+  .option('--scopes <scopes>', 'set scopes', list)
+  // .option('--cluster', 'enable cluster mode')
+  .option('--dataDir <dataDir>', 'cnpmjs.org data dir, default is `$HOME/.cnpmjs.org`')
+  .action(start);
+
+program
+  .command('stop')
+  .description('stop cnpmjs.org server')
+  .option('--dataDir <dataDir>', 'cnpmjs.org data dir, default is `$HOME/.cnpmjs.org`')
+  .action(stop);
+
+program.parse(process.argv);
+
+
+function start(options) {
+  stop(options)
+    // wait for "stop" method to remove the pid file
+    .then(function () {
+      var dataDir = options.dataDir || path.join(process.env.HOME, '.cnpmjs.org');
+      mkdirp.sync(dataDir);
+
+      var configfile = path.join(dataDir, 'config.json');
+      var config = {};
+      if (fs.existsSync(configfile)) {
+        try {
+          config = require(configfile);
+        } catch (err) {
+          console.warn('load old %s error: %s', configfile, err);
+        }
+      }
+      // config.enableCluster = !!options.cluster;
+      if (options.admins) {
+        config.admins = {};
+        for (var i = 0; i < options.admins.length; i++) {
+          config.admins[options.admins[i]] = options.admins[i] + '@localhost.com';
+        }
+      }
+      if (options.scopes) {
+        config.scopes = options.scopes.map(function (name) {
+          if (name[0] !== '@') {
+            name = '@' + name;
+          }
+          return name;
+        });
+      }
+
+      var configJSON = JSON.stringify(config, null, 2);
+      fs.writeFileSync(configfile, configJSON);
+
+      debug('save config %s to %s', configJSON, configfile);
+
+      // if sqlite db file not exists, init first
+      initDatabase(function() {
+        require('../dispatch');
+      });
+
+      fs.writeFileSync(path.join(dataDir, 'pid'), process.pid + '');
+    });
+}
+
+function stop(options) {
+  var dataDir = options.dataDir || path.join(process.env.HOME, '.cnpmjs.org');
+  var pidfile = path.join(dataDir, 'pid');
+  return new Promise(function (resolve) {
+    if (!fs.existsSync(pidfile)) {
+      resolve();
+      return;
+    }
+    var pid = Number(fs.readFileSync(pidfile, 'utf8'));
+    treekill(pid, function (err) {
+      if (err) {
+        console.log(err);
+        throw err;
+      }
+      console.log('cnpmjs.org server:%d stop', pid);
+      fs.unlinkSync(pidfile);
+      resolve();
+    });
+  });
+}
+
+function initDatabase(callback) {
+  var models = require('../models');
+
+  models.sequelize.sync({ force: false })
+    .then(function () {
+      models.Total.init(function (err) {
+        if (err) {
+          console.error('[models/init_script.js] sequelize init fail');
+          console.error(err);
+          throw err;
+        } else {
+          console.log('[models/init_script.js] `sqlite` sequelize sync and init success');
+          callback();
+        }
+      });
+    })
+    .catch(function (err) {
+      console.error('[models/init_script.js] sequelize sync fail');
+      console.error(err);
+      throw err;
+    });
+}

bin/nodejsctl

@@ -7,7 +7,7 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS=node
+NODEJS='node'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
 STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\
@@ -52,9 +52,9 @@ stop()
     kill -15 $PID
     sleep 2
 
-    node_num=`ps -ef |grep node|grep ${PROJECT_NAME}|grep -v grep|wc -l`
+    node_num=`ps -ef | grep ${PROJECT_NAME} | grep -v grep | wc -l`
     if [ $node_num != 0 ]; then
-      ps -ef |grep node | grep ${PROJECT_NAME} |grep -v grep|awk '{print $2}'|xargs kill -9
+      ps -ef | grep ${PROJECT_NAME} |grep -v grep|awk '{print $2}'|xargs kill -9
       ipcs -s | grep 0x | awk '{print $2}' | xargs -n1 ipcrm -s  > /dev/null 2>&1
       ipcs -m | grep 0x | awk '{print $2}' | xargs -n1 ipcrm -m  > /dev/null 2>&1
     fi

common/cache.js

@@ -0,0 +1,16 @@
+'use strict';
+
+const debug = require('debug')('cnpmjs.org:cache');
+const Redis = require('ioredis');
+const config = require('../config');
+
+let client;
+
+if (config.redisCache.enable) {
+  client = new Redis(config.redisCache.connectOptions);
+  client.on('ready', () => {
+    debug('connect ready, getBuiltinCommands: %j', client.getBuiltinCommands());
+  });
+}
+
+module.exports = client;

common/logger.js

@@ -1,70 +1,56 @@
-/*!
- * cnpmjs.org - common/logger.js
- * Copyright(c) 2013 
- * Author: dead_horse <undefined>
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+const debug = require('debug')('cnpmjs.org:logger');
+const formater = require('error-formater');
+const Logger = require('mini-logger');
+const utility = require('utility');
+const util = require('util');
+const os = require('os');
+const config = require('../config');
+const mail = require('./mail');
 
-var config = require('../config');
-var util = require('util');
-var moment = require('moment');
-var logstream = require('logfilestream');
-var ms = require('ms');
+const isTEST = process.env.NODE_ENV === 'test';
+const categories = ['sync_info', 'sync_error'];
 
-var isTEST = process.env.NODE_ENV === 'test';
-var ONE_DAY = ms('1d');
-var levels = ['info', 'warn', 'error'];
-
-levels.forEach(function (catetory) {
-  var options = {
-    logdir: config.logdir,
-    duration: ONE_DAY,
-    nameformat: '[' + catetory + '.]YYYY-MM-DD[.log]'
-  };
-  var stream = logstream(options);
-  function write(msg) {
-    var time = moment().format('YYYY-MM-DD HH:mm:ss.SSS');
-    if (msg instanceof Error) {
-      var err = {
-        name: msg.name,
-        code: msg.code,
-        message: msg.message,
-        stack: msg.stack,
-        host: msg.host,
-        url: msg.url,
-        data: msg.data
-      };
-      if (err.name === 'Error' && typeof err.code === 'string') {
-        err.name = err.code + err.name;
-      }
-      err.name += 'Exception';
-      if (err.host) {
-        err.message += ' (' + err.host + ')';
-      }
-      msg = util.format('%s nodejs.%s: %s\nURL: %s\nData: %j\n%s\n\n',
-        time,
-        err.name,
-        err.stack,
-        err.url,
-        err.data,
-        time
-      );
-    } else {
-      msg = time + ' ' + util.format.apply(util, arguments) + '\n';
-    }
-    if (!isTEST) {
-      if (config.debug) {
-        var level = catetory;
-        console.log('[' + level + '] ' + msg);
-      } else {
-        stream.write(msg);
-      }
-    }
-  }
-  exports[catetory] = write;
+const logger = module.exports = Logger({
+  categories: categories,
+  dir: config.logdir,
+  duration: '1d',
+  format: '[{category}.]YYYY-MM-DD[.log]',
+  stdout: config.debug && !isTEST,
+  errorFormater: errorFormater,
+  seperator: os.EOL,
 });
+
+const to = [];
+for (var user in config.admins) {
+  to.push(config.admins[user]);
+}
+
+function errorFormater(err) {
+  const msg = formater.both(err);
+  mail.error(to, msg.json.name, msg.text);
+  return msg.text;
+}
+
+logger.syncInfo = function () {
+  const args = [].slice.call(arguments);
+  if (typeof args[0] === 'string') {
+    args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
+  }
+  if (debug.enabled) {
+    debug.apply(debug, args);
+  }
+  logger.sync_info.apply(logger, args);
+};
+
+logger.syncError =function () {
+  const args = [].slice.call(arguments);
+  if (typeof args[0] === 'string') {
+    args[0] = util.format('[%s][%s] ', utility.logDate(), process.pid) + args[0];
+  }
+  if (debug.enabled) {
+    debug.apply(debug, args);
+  }
+  logger.sync_error.apply(logger, arguments);
+};

common/mail.js

@@ -1,24 +1,31 @@
-/*!
- * cnpmjs.org - common/mail.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
-var utility = require('utility');
 var nodemailer = require('nodemailer');
+var utility = require('utility');
 var os = require('os');
 var mailConfig = require('../config').mail;
 
+var smtpConfig;
+if (mailConfig.auth) {
+  // new style
+  smtpConfig = mailConfig;
+} else {
+  smtpConfig = {
+    enable: mailConfig.enable,
+    // backward compat
+    host: mailConfig.host,
+    port: mailConfig.port,
+    secure: mailConfig.secure || mailConfig.ssl,
+    debug: mailConfig.debug,
+    auth: {
+      user: mailConfig.user,
+      pass: mailConfig.pass
+    }
+  };
+}
+
+var transport;
+
 /**
  * Send notice email with mail level and appname.
  *
@@ -51,26 +58,23 @@ LEVELS.forEach(function (level) {
 exports.send = function (to, subject, html, callback) {
   callback = callback || utility.noop;
 
-  var transport = nodemailer.createTransport("SMTP", {
-    host: mailConfig.host,
-    port: mailConfig.port,
-    secureConnection: mailConfig.ssl,
-    debug: mailConfig.debug,
-    auth: {
-      user: mailConfig.user,
-      pass: mailConfig.pass,
-    }
-  });
+  if (mailConfig.enable === false) {
+    console.log('[send mail debug] [%s] to: %s, subject: %s\n%s', Date(), to, subject, html);
+    return callback();
+  }
+
+  if (!transport) {
+    transport = nodemailer.createTransport(smtpConfig);
+  }
 
   var message = {
-    sender: mailConfig.sender,
+    from: mailConfig.from || mailConfig.sender,
     to: to,
     subject: subject,
     html: html,
   };
 
   transport.sendMail(message, function (err, result) {
-    transport.close();
     callback(err, result);
   });
 };

common/markdown.js

@@ -0,0 +1,20 @@
+'use strict';
+
+var xss = require('xss');
+var MarkdownIt = require('markdown-it');
+
+// allow class attr on code
+xss.whiteList.code = ['class'];
+
+var md = new MarkdownIt({
+  html: true,
+  linkify: true,
+});
+
+exports.render = function (content, filterXss) {
+  var html = md.render(content);
+  if (filterXss !== false) {
+    html = xss(html);
+  }
+  return html;
+};

common/mysql.js

@@ -1,71 +0,0 @@
-/*!
- * cnpmjs.org - common/mysql.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var ready = require('ready');
-var mysql = require('mysql');
-var config = require('../config');
-
-var server = config.mysqlServers[0];
-
-// TODO: query timeout
-var pool = mysql.createPool({
-  host: server.host,
-  port: server.port,
-  user: server.user,
-  password: server.password,
-  database: config.mysqlDatabase,
-  connectionLimit: config.mysqlMaxConnections,
-  multipleStatements: true,
-});
-
-exports.pool = pool;
-
-exports.query = function (sql, values, cb) {
-  pool.query(sql, values, cb);
-};
-
-exports.queryOne = function (sql, values, cb) {
-  if (typeof values === 'function') {
-    cb = values;
-    values = null;
-  }
-  exports.query(sql, values, function (err, rows) {
-    if (rows) {
-      rows = rows[0];
-    }
-    cb(err, rows);
-  });
-};
-
-exports.escape = function (val) {
-  return pool.escape(val);
-};
-
-ready(exports);
-
-function init() {
-  exports.query('show tables', function (err, rows) {
-    if (err) {
-      console.error('[%s] [worker:%s] mysql init error: %s', Date(), process.pid, err);
-      setTimeout(init, 1000);
-      return;
-    }
-    console.log('[%s] [worker:%s] mysql ready, got %d tables', Date(), process.pid, rows.length);
-    exports.ready(true);
-  });
-}
-init();

common/nfs.js

@@ -1,11 +1,9 @@
 /*!
- * cnpmjs.org - common/nfs.js
- *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
  *
  * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
  */
 
 'use strict';
@@ -15,11 +13,5 @@
  */
 
 var config = require('../config');
-var nfs = config.nfs;
 
-if (!nfs) {
-  // use qnfs by default
-  nfs = require('./qnfs');
-}
-
-module.exports = nfs;
+module.exports = config.nfs;

common/qnfs.js

@@ -1,45 +0,0 @@
-/*!
- * cnpmjs.org - common/qnfs.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var qn = require('qn');
-var config = require('../config');
-
-var client = qn.create(config.qn);
-
-/**
- * Upload file
- *
- * @param {String} filepath
- * @param {Object} options
- *  - {String} key
- *  - {Number} size
- * @param {Function(err, result)} callback
- *  - {Object} result
- *   - {String} url
- */
-exports.upload = function (filepath, options, callback) {
-  client.delete(options.key, function (err, data) {
-    client.uploadFile(filepath, {key: options.key, size: options.size}, callback);
-  });
-};
-
-exports.url = function (key) {
-  return client.resourceURL(key);
-};
-
-exports.remove = function (key, callback) {
-  client.delete(key, callback);
-};

common/sequelize.js

@@ -0,0 +1,65 @@
+'use strict';
+
+var Sequelize = require('sequelize');
+var DataTypes = require('sequelize/lib/data-types');
+var config = require('../config');
+
+if (config.mysqlServers && config.database.dialect === 'sqlite') {
+  // https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x
+  // forward compat with old style on 1.x
+  // mysqlServers: [
+  //   {
+  //     host: '127.0.0.1',
+  //     port: 3306,
+  //     user: 'root',
+  //     password: ''
+  //   }
+  // ],
+  // mysqlDatabase: 'cnpmjs_test',
+  // mysqlMaxConnections: 4,
+  // mysqlQueryTimeout: 5000,
+
+  console.warn('[WARNNING] your config.js was too old, please @see https://github.com/cnpm/cnpmjs.org/wiki/Migrating-from-1.x-to-2.x');
+  var server = config.mysqlServers[0];
+  var dialectOptions = config.database && config.database.dialectOptions;
+  config.database = {
+    db: config.mysqlDatabase,
+    username: server.user,
+    password: server.password,
+    dialect: 'mysql',
+    host: server.host,
+    port: server.port,
+    pool: {
+      maxConnections: config.mysqlMaxConnections || 10,
+      minConnections: 0,
+      maxIdleTime: 30000,
+    },
+    logging: !!process.env.SQL_DEBUG,
+  };
+  if (dialectOptions) {
+    config.database.dialectOptions = dialectOptions;
+  }
+}
+
+var database = config.database;
+
+// sync database before app start, defaul is false
+database.syncFirst = false;
+
+// add longtext for mysql
+Sequelize.LONGTEXT = DataTypes.LONGTEXT = DataTypes.TEXT;
+if (database.dialect === 'mysql') {
+  Sequelize.LONGTEXT = DataTypes.LONGTEXT = 'LONGTEXT';
+}
+
+database.define = {
+  timestamps: true,
+  createdAt: 'gmt_create',
+  updatedAt: 'gmt_modified',
+  charset: 'utf8',
+  collate: 'utf8_general_ci',
+};
+
+var sequelize = new Sequelize(database.db, database.username, database.password, database);
+
+module.exports = sequelize;

common/session.js

@@ -1,41 +0,0 @@
-/*!
- * cnpmjs.org - common/session.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var connect = require('connect');
-var RedisStore = require('connect-redis')(connect);
-var config = require('../config');
-
-var session;
-var key = 'AuthSession';
-var cookie = { path: '/', httpOnly: true, maxAge: 3600000 * 24 * 30 };
-
-if (config.debug) {
-  session = connect.cookieSession({
-    secret: config.sessionSecret,
-    key: key,
-    cookie: cookie
-  });
-} else {
-  session = connect.session({
-    key: key,
-    secret: config.sessionSecret,
-    store: new RedisStore(config.redis),
-    cookie: cookie,
-  });
-}
-
-module.exports = session;

common/urllib.js

@@ -0,0 +1,84 @@
+'use strict';
+
+var urlparse = require('url').parse;
+var urllib = require('urllib');
+var HttpAgent = require('agentkeepalive');
+var HttpsAgent = require('agentkeepalive').HttpsAgent;
+var config = require('../config');
+var url = require('url');
+var URL = require('url').URL;
+
+var httpAgent;
+var httpsAgent;
+
+if (config.httpProxy) {
+  var tunnel = require('tunnel-agent');
+  var urlinfo = urlparse(config.httpProxy);
+  if (urlinfo.protocol === 'http:') {
+    httpAgent = tunnel.httpOverHttp({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+    httpsAgent = tunnel.httpsOverHttp({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+  } else if (urlinfo.protocol === 'https:') {
+    httpAgent = tunnel.httpOverHttps({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+    httpsAgent = tunnel.httpsOverHttps({
+      proxy: {
+        host: urlinfo.hostname,
+        port: urlinfo.port
+      }
+    });
+  } else {
+    throw new TypeError('httpProxy format error: ' + config.httpProxy);
+  }
+} else {
+  httpAgent = new HttpAgent({
+    timeout: 0,
+    keepAliveTimeout: 15000
+  });
+  httpsAgent = new HttpsAgent({
+    timeout: 0,
+    keepAliveTimeout: 15000
+  });
+}
+
+var client = urllib.create({
+  agent: httpAgent,
+  httpsAgent: httpsAgent
+});
+
+var request = urllib.HttpClient.prototype.request;
+
+function getAccelerateUrl(url) {
+  const urlObj = typeof url === 'string' ? new URL(url) : url;
+  const newHost = config.accelerateHostMap && config.accelerateHostMap[urlObj.host];
+  if (newHost) {
+    urlObj.host = newHost;
+  }
+  return urlObj.toString();
+}
+
+client.request = function (requestUrl, options) {
+  const accelerateUrl = getAccelerateUrl(requestUrl);
+  options = Object.assign({}, options, {
+    formatRedirectUrl: function (from, to) {
+      return getAccelerateUrl(url.resolve(from, to));
+    }
+  });
+  return Reflect.apply(request, client, [ accelerateUrl, options ]);
+};
+
+module.exports = client;
+module.exports.USER_AGENT = urllib.USER_AGENT;

config/index.js

@@ -1,96 +1,387 @@
-/*!
- * cnpmjs.org - config/index.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
+var mkdirp = require('mkdirp');
+var copy = require('copy-to');
 var path = require('path');
 var fs = require('fs');
-var mkdirp = require('mkdirp');
-
-fs.existsSync = fs.existsSync || path.existsSync;
-var pkg = require('../package.json');
+var os = require('os');
+var utility = require('utility');
 
+var version = require('../package.json').version;
+var Nfs = require('fs-cnpm');
 var root = path.dirname(__dirname);
+var dataDir = path.join(process.env.HOME || root, '.cnpmjs.org');
 
 var config = {
-  version: pkg.version,
+  version: version,
+  dataDir: dataDir,
+
+  /**
+   * Cluster mode
+   */
+  enableCluster: false,
+  numCPUs: os.cpus().length,
+
+  /*
+   * server configure
+   */
+
   registryPort: 7001,
   webPort: 7002,
-  enableCluster: false,
-  debug: true, // if debug
-  logdir: path.join(root, '.tmp', 'logs'),
-  viewCache: false,
-  // mysql config
-  mysqlServers: [
-    {
-      host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
-      port: 3306,
-      user: 'cnpmjs',
-      password: 'cnpmjs123'
-    }
-  ],
-  mysqlDatabase: 'cnpmjstest',
-  mysqlMaxConnections: 4,
-  mysqlQueryTimeout: 5000,
+  bindingHost: '127.0.0.1', // only binding on 127.0.0.1 for local access
+  // default is ctx.protocol
+  protocol: '',
+  // When sync package, cnpm not know the access protocol.
+  // So should set manually
+  backupProtocol: 'http',
 
+  // debug mode
+  // if in debug mode, some middleware like limit wont load
+  // logger module will print to stdout
+  debug: process.env.NODE_ENV === 'development',
+  // page mode, enable on development env
+  pagemock: process.env.NODE_ENV === 'development',
+  // session secret
   sessionSecret: 'cnpmjs.org test session secret',
-  redis: {
-    host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
-    port: 19533,
-    pass: 'cnpmjs_dev'
-  },
-  uploadDir: path.join(root, 'public', 'dist'),
-  // qiniu cdn: http://www.qiniu.com/, it free for dev.
-  qn: {
-    accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
-    secretKey: "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j",
-    bucket: "qtestbucket",
-    domain: "http://qtestbucket.qiniudn.com"
+  // max request json body size
+  jsonLimit: '10mb',
+  // log dir name
+  logdir: path.join(dataDir, 'logs'),
+  // update file template dir
+  uploadDir: path.join(dataDir, 'downloads'),
+  // web page viewCache
+  viewCache: false,
+
+  // registry http response cache control header
+  // if you are using CDN, can set it to 'max-age=0, s-maxage=10, must-revalidate'
+  // it meaning cache 10s on CDN server and no cache on client side.
+  registryCacheControlHeader: '',
+  // if you are using CDN, can set it to 'Accept, Accept-Encoding'
+  registryVaryHeader: '',
+  // disable package search
+  disableSearch: false,
+
+  // view files directory
+  viewDir: path.join(root, 'view', 'web'),
+
+  customRegistryMiddlewares: [],
+  customWebMiddlewares: [],
+
+  // config for koa-limit middleware
+  // for limit download rates
+  limit: {
+    enable: false,
+    token: 'koa-limit:download',
+    limit: 1000,
+    interval: 1000 * 60 * 60 * 24,
+    whiteList: [],
+    blackList: [],
+    message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
   },
 
-  mail: {
-    appname: 'cnpmjs.org',
-    sender: 'cnpmjs.org mail sender <adderss@gmail.com>',
-    host: 'smtp.gmail.com',
-    port: 465,
-    user: 'address@gmail.com',
-    pass: 'your password',
-    ssl: true,
-    debug: false 
-  },
+  enableCompress: false, // enable gzip response or not
 
-  sourceNpmRegistry: 'http://registry.npmjs.org',
-  enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
+  // default system admins
   admins: {
-    admin: 'admin@cnpmjs.org',
+    // name: email
     fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
     dead_horse: 'dead_horse@qq.com',
-    cnpmjstest10: 'cnpmjstest10@cnpmjs.org',
   },
+
+  // email notification for errors
+  // check https://github.com/andris9/Nodemailer for more informations
+  mail: {
+    enable: false,
+    appname: 'cnpmjs.org',
+    from: 'cnpmjs.org mail sender <adderss@gmail.com>',
+    service: 'gmail',
+    auth: {
+      user: 'address@gmail.com',
+      pass: 'your password'
+    }
+  },
+
+  logoURL: 'https://os.alipayobjects.com/rmsportal/oygxuIUkkrRccUz.jpg', // cnpm logo image url
+  adBanner: '',
+  customHeader: '',
+  customReadmeFile: '', // you can use your custom readme file instead the cnpm one
+  customFooter: '', // you can add copyright and site total script html here
+  npmClientName: 'cnpm', // use `${name} install package`
+
+  // max handle number of package.json `dependencies` property
+  maxDependencies: 200,
+
+  /**
+   * database config
+   */
+
+  database: {
+    db: 'cnpmjs_test',
+    username: 'root',
+    password: '',
+
+    // the sql dialect of the database
+    // - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
+    dialect: 'sqlite',
+
+    // custom host; default: 127.0.0.1
+    host: '127.0.0.1',
+
+    // custom port; default: 3306
+    port: 3306,
+
+    // use pooling in order to reduce db connection overload and to increase speed
+    // currently only for mysql and postgresql (since v1.5.0)
+    pool: {
+      maxConnections: 10,
+      minConnections: 0,
+      maxIdleTime: 30000
+    },
+
+    dialectOptions: {
+      // if your server run on full cpu load, please set trace to false
+      trace: true,
+    },
+
+    // the storage engine for 'sqlite'
+    // default store into ~/.cnpmjs.org/data.sqlite
+    storage: path.join(dataDir, 'data.sqlite'),
+
+    logging: !!process.env.SQL_DEBUG,
+  },
+
+  // return total modules and versions, default is true
+  // it will use `SELECT count(DISTINCT name) FROM module` SQL on Database
+  enableTotalCount: true,
+
+  // enable proxy npm audits request or not
+  enableNpmAuditsProxy: true,
+
+  // package tarball store in local filesystem by default
+  nfs: new Nfs({
+    dir: path.join(dataDir, 'nfs')
+  }),
+  // if set true, will 302 redirect to `nfs.url(dist.key)`
+  downloadRedirectToNFS: false,
+  // don't check database and just download tgz from nfs
+  downloadTgzDontCheckModule: false,
+  // remove original tarball when publishing
+  unpublishRemoveTarball: true,
+
+  // registry url name
+  registryHost: 'r.cnpmjs.org',
+
+  /**
+   * registry mode config
+   */
+
+  // enable private mode or not
+  // private mode: only admins can publish, other users just can sync package from source npm
+  // public mode: all users can publish
+  enablePrivate: false,
+
+  // registry scopes, if don't set, means do not support scopes
+  scopes: [ '@cnpm', '@cnpmtest', '@cnpm-test' ],
+
+  // some registry already have some private packages in global scope
+  // but we want to treat them as scoped private packages,
+  // so you can use this white list.
+  privatePackages: [],
+
+  /**
+   * sync configs
+   */
+
+  // the official npm registry
+  // cnpm wont directly sync from this one
+  // but sometimes will request it for some package infomations
+  // please don't change it if not necessary
+  officialNpmRegistry: 'https://registry.npmjs.com',
+  officialNpmReplicate: 'https://replicate.npmjs.com',
+  cnpmRegistry: 'https://r.cnpmjs.com',
+
+  // /-/all/changes
+  // since different changes are aggregated through many tables
+  // prevent changesStream changes collisions
+  changesDelay: 5000,
+
+  // sync source, upstream registry
+  // If you want to directly sync from official npm's registry
+  // please drop them an email first
+  sourceNpmRegistry: 'https://registry.npmmirror.com',
+  sourceNpmWeb: 'https://npmmirror.com',
+
+  // set remote registry to show web page data
+  enableWebDataRemoteRegistry: false,
+  webDataRemoteRegistry: '',
+
+  // upstream registry is base on cnpm/cnpmjs.org or not
+  // if your upstream is official npm registry, please turn it off
+  sourceNpmRegistryIsCNpm: true,
+
+  // if install return 404, try to sync from source registry
   syncByInstall: true,
-  backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
+
+  // sync mode select
+  // none: do not sync any module, proxy all public modules from sourceNpmRegistry
+  // exist: only sync exist modules
+  // all: sync all modules
   syncModel: 'none', // 'none', 'all', 'exist'
+  // sync package.json/dist-tag.json to sync dir
+  syncBackupFiles: false,
+
   syncConcurrency: 1,
+  // sync interval, default is 10 minutes
+  syncInterval: '10m',
+
+  // sync polular modules, default to false
+  // because cnpm can't auto sync tag change for now
+  // so we want to sync popular modules to ensure their tags
+  syncPopular: false,
+  syncPopularInterval: '1h',
+  // top 100
+  topPopular: 100,
+
+  // sync devDependencies or not, default is false
+  syncDevDependencies: false,
+  // try to remove all deleted versions from original registry
+  syncDeletedVersions: true,
+
+  // changes streaming sync
+  syncChangesStream: false,
+  syncDownloadOptions: {
+    // formatRedirectUrl: function (url, location)
+  },
+
+  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
+  syncScope: false,
+  syncScopeInterval: '12h',
+  // scope package sync config
+  /**
+ * sync scope package from assign registry
+ * @param {Array<scope>} scopes
+ * @param {String} scope.scope scope name
+ * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
+ * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
+ */
+  syncScopeConfig: [],
+
+  handleSyncRegistry: 'http://127.0.0.1:7001',
+
+  // default badge subject
+  badgeSubject: 'cnpm',
+  // defautl use https://badgen.net/
+  badgeService: {
+    url: function(subject, status, options) {
+      options = options || {};
+      let url = `https://badgen.net/badge/${utility.encodeURIComponent(subject)}/${utility.encodeURIComponent(status)}`;
+      if (options.color) {
+        url += `/${utility.encodeURIComponent(options.color)}`;
+      }
+      if (options.icon) {
+        url += `?icon=${utility.encodeURIComponent(options.icon)}`;
+      }
+      return url;
+    },
+  },
+
+  packagephobiaURL: 'https://packagephobia.now.sh',
+  packagephobiaSupportPrivatePackage: false,
+  packagephobiaMinDownloadCount: 1000,
+
+  // custom user service, @see https://github.com/cnpm/cnpmjs.org/wiki/Use-Your-Own-User-Authorization
+  // when you not intend to ingegrate with your company's user system, then use null, it would
+  // use the default cnpm user system
+  userService: null,
+
+  // always-auth https://docs.npmjs.com/misc/config#always-auth
+  // Force npm to always require authentication when accessing the registry, even for GET requests.
+  alwaysAuth: false,
+
+  // if you're behind firewall, need to request through http proxy, please set this
+  // e.g.: `httpProxy: 'http://proxy.mycompany.com:8080'`
+  httpProxy: null,
+
+  // snyk.io root url
+  snykUrl: 'https://snyk.io',
+
+  // https://github.com/cnpm/cnpmjs.org/issues/1149
+  // if enable this option, must create module_abbreviated and package_readme table in database
+  enableAbbreviatedMetadata: false,
+
+  // enable package or package version block list, must create package_version_blocklist table in database
+  enableBlockPackageVersion: false,
+
+  // enable bug version hotfix by https://github.com/cnpm/bug-versions
+  enableBugVersion: false,
+
+  // global hook function: function* (envelope) {}
+  // envelope format please see https://github.com/npm/registry/blob/master/docs/hooks/hooks-payload.md#payload
+  globalHook: null,
+
+  opensearch: {
+    host: '',
+  },
+
+  // redis cache
+  redisCache: {
+    enable: false,
+    connectOptions: null,
+  },
+
+  // custom format full package list
+  // change `GET /:name` request response body
+  // use on `controllers/registry/list.js`
+  formatCustomFullPackageInfoAndVersions: (ctx, packageInfo) => {
+    return packageInfo;
+  },
+  // custom format one package version
+  // change `GET /:name/:version` request response body
+  // use on `controllers/registry/show.js`
+  formatCustomOnePackageVersion: (ctx, packageVersion) => {
+    return packageVersion;
+  },
+  // registry download accelerate map
+  accelerateHostMap: {},
 };
 
-// load config/config.js, everything in config.js will cover the same key in index.js
-var customConfig = path.join(root, 'config/config.js');
-if (fs.existsSync(customConfig)) {
-  var options = require(customConfig);
-  for (var k in options) {
-    config[k] = options[k];
+if (process.env.NODE_ENV === 'test') {
+  config.enableAbbreviatedMetadata = true;
+  config.customRegistryMiddlewares.push((app) => {
+    return function* (next) {
+      this.set('x-custom-middleware', 'true');
+      this.set('x-custom-app-models', typeof app.models.query === 'function' ? 'true' : 'false');
+      yield next;
+    };
+  });
+
+  config.customWebMiddlewares.push((app) => {
+    return function* (next) {
+      this.set('x-custom-web-middleware', 'true');
+      this.set('x-custom-web-app-models', typeof app.models.query === 'function' ? 'true' : 'false');
+      yield next;
+    };
+  });
+
+  config.enableBlockPackageVersion = true;
+  config.enableBugVersion = true;
+}
+
+if (process.env.NODE_ENV !== 'test') {
+  var customConfig;
+  if (process.env.NODE_ENV === 'development') {
+    customConfig = path.join(root, 'config', 'config.js');
+  } else {
+    // 1. try to load `$dataDir/config.json` first, not exists then goto 2.
+    // 2. load config/config.js, everything in config.js will cover the same key in index.js
+    customConfig = path.join(dataDir, 'config.json');
+    if (!fs.existsSync(customConfig)) {
+      customConfig = path.join(root, 'config', 'config.js');
+    }
+  }
+  if (fs.existsSync(customConfig)) {
+    copy(require(customConfig)).override(config);
   }
 }
 
@@ -98,3 +389,10 @@ mkdirp.sync(config.logdir);
 mkdirp.sync(config.uploadDir);
 
 module.exports = config;
+
+config.loadConfig = function (customConfig) {
+  if (!customConfig) {
+    return;
+  }
+  copy(customConfig).override(config);
+};

controllers/download.js

@@ -1,76 +0,0 @@
-/**!
- * cnpmjs.org - controllers/download.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var moment = require('moment');
-var DownloadTotal = require('../proxy/download');
-
-exports.total = function (name, callback) {
-  var end = moment();
-  var start = end.clone().subtract('months', 1).startOf('month');
-  var lastday = end.clone().subtract('days', 1).format('YYYY-MM-DD');
-  var lastweekStart = end.clone().subtract('weeks', 1).startOf('week');
-  var lastweekEnd = lastweekStart.clone().endOf('week').format('YYYY-MM-DD');
-  var lastmonthEnd = start.clone().endOf('month').format('YYYY-MM-DD');
-  var thismonthStart = end.clone().startOf('month').format('YYYY-MM-DD');
-  var thisweekStart = end.clone().startOf('week').format('YYYY-MM-DD');
-  start = start.format('YYYY-MM-DD');
-  end = end.format('YYYY-MM-DD');
-  lastweekStart = lastweekStart.format('YYYY-MM-DD');
-  var method = name ? 'getModuleTotal' : 'getTotal';
-  var args = [start, end];
-  if (name) {
-    args.unshift(name);
-  }
-  args.push(function (err, rows) {
-    if (err) {
-      return callback(err);
-    }
-
-    var download = {
-      today: 0,
-      thisweek: 0,
-      thismonth: 0,
-      lastday: 0,
-      lastweek: 0,
-      lastmonth: 0,
-    };
-    for (var i = 0; i < rows.length; i++) {
-      var r = rows[i];
-      if (r.date === end) {
-        download.today += r.count;
-      }
-      if (r.date >= thismonthStart) {
-        download.thismonth += r.count;
-      }
-      if (r.date >= thisweekStart) {
-        download.thisweek += r.count;
-      }
-
-      if (r.date === lastday) {
-        download.lastday += r.count;
-      }
-      if (r.date >= lastweekStart && r.date <= lastweekEnd) {
-        download.lastweek += r.count;
-      }
-      if (r.date >= start && r.date <= lastmonthEnd) {
-        download.lastmonth += r.count;
-      }
-    }
-    callback(null, download);
-  });
-
-  DownloadTotal[method].apply(DownloadTotal, args);
-};

controllers/registry/module.js

@@ -1,635 +0,0 @@
-/**!
- * cnpmjs.org - controllers/registry/module.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:controllers:registry:module');
-var path = require('path');
-var fs = require('fs');
-var crypto = require('crypto');
-var utility = require('utility');
-var eventproxy = require('eventproxy');
-var Bagpipe = require('bagpipe');
-var urlparse = require('url').parse;
-var config = require('../../config');
-var Module = require('../../proxy/module');
-var Total = require('../../proxy/total');
-var nfs = require('../../common/nfs');
-var common = require('../../lib/common');
-var Log = require('../../proxy/module_log');
-var DownloadTotal = require('../../proxy/download');
-var SyncModuleWorker = require('../../proxy/sync_module_worker');
-var logger = require('../../common/logger');
-var semver = require('semver');
-
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Module.listTags(name, ep.done('tags'));
-  Module.listByName(name, ep.done('rows'));
-
-  ep.all('tags', 'rows', function (tags, rows) {
-    debug('show module, user: %s, allowSync: %s, isAdmin: %s', req.session.name, req.session.allowSync, req.session.isAdmin);
-    // if module not exist in this registry,
-    // sync the module backend and return package info from official registry
-    // TODO: Need to change this logic, it will cause unpublish sync again.
-    // TODO: Hard to detect this show is on install flow or unpublish flow
-    if (rows.length === 0) {
-      if (!req.session.allowSync) {
-        return next();
-      }
-      var username = (req.session && req.session.name) || 'anonymous';
-      SyncModuleWorker.sync(name, username, function (err, result) {
-        if (err) {
-          return next(err);
-        }
-        if (!result.ok) {
-          return res.json(result.statusCode, result.pkg);
-        }
-        res.json(200, result.pkg);
-      });
-      return;
-    }
-
-    var nextMod = null;
-    var latestMod = null;
-    var distTags = {};
-    for (var i = 0; i < tags.length; i++) {
-      var t = tags[i];
-      distTags[t.tag] = t.version;
-    }
-
-    var versions = {};
-    var times = {};
-    var attachments = {};
-    for (var i = 0; i < rows.length; i++) {
-      var row = rows[i];
-      if (row.version === 'next') {
-        nextMod = row;
-        continue;
-      }
-      var pkg = row.package;
-      common.downloadURL(pkg, req);
-      versions[pkg.version] = pkg;
-      times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
-      if ((!distTags.latest && !latestMod) || distTags.latest === row.version) {
-        latestMod = row;
-      }
-    }
-
-    if (!latestMod) {
-      latestMod = nextMod || rows[0];
-    }
-
-    if (!nextMod) {
-      nextMod = latestMod;
-    }
-
-    var rev = '';
-    if (nextMod) {
-      rev = String(nextMod.id);
-    }
-
-    var info = {
-      _id: name,
-      _rev: rev,
-      name: name,
-      description: latestMod.package.description,
-      "dist-tags": distTags,
-      maintainers: latestMod.package.maintainers,
-      time: times,
-      author: latestMod.package.author,
-      repository: latestMod.package.repository,
-      versions: versions,
-      readme: latestMod.package.readme,
-      _attachments: attachments,
-    };
-
-    debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
-
-    res.json(info);
-  });
-};
-
-exports.get = function (req, res, next) {
-  var name = req.params.name;
-  var tag = req.params.version;
-  var version = semver.valid(tag);
-
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  var method = version ? 'get' : 'getByTag';
-  var queryLabel = version ? version : tag;
-
-  Module[method](name, queryLabel, ep.done(function (mod) {
-    if (mod) {
-      common.downloadURL(mod.package, req);
-      return res.json(mod.package);
-    }
-    ep.emit('notFound');
-  }));
-
-  ep.once('notFound', function () {
-    if (!req.session.allowSync) {
-      return next();
-    }
-
-    var username = (req.session && req.session.username) || 'anonymous';
-    SyncModuleWorker.sync(name, username, function (err, result) {
-      if (err) {
-        return next(err);
-      }
-      var pkg = result.pkg && result.pkg.versions[version];
-      if (!pkg) {
-        return res.json(404, {
-          error: 'not exist',
-          reason: 'version not found: ' + version
-        });
-      }
-      res.json(pkg);
-    });
-  });
-};
-
-var _downloads = {};
-
-exports.download = function (req, res, next) {
-  var name = req.params.name;
-  var filename = req.params.filename;
-  var cdnurl = nfs.url(common.getCDNKey(name, filename));
-  res.statusCode = 302;
-  res.setHeader('Location', cdnurl);
-  res.end();
-  _downloads[name] = (_downloads[name] || 0) + 1;
-};
-
-setInterval(function () {
-  // save download count
-  var totals = [];
-  for (var name in _downloads) {
-    var count = _downloads[name];
-    totals.push([name, count]);
-  }
-  _downloads = {};
-
-  if (totals.length === 0) {
-    return;
-  }
-
-  debug('save download total: %j', totals);
-
-  var date = utility.YYYYMMDD();
-  var next = function () {
-    var item = totals.shift();
-    if (!item) {
-      // done
-      return;
-    }
-
-    DownloadTotal.plusTotal({name: item[0], date: date, count: item[1]}, function (err) {
-      if (!err) {
-        return next();
-      }
-
-      logger.error(err);
-      debug('save download %j error: %s', item, err);
-
-      totals.push(item);
-      // save to _downloads
-      for (var i = 0; i < totals.length; i++) {
-        var v = totals[i];
-        var name = v[0];
-        _downloads[name] = (_downloads[name] || 0) + v[1];
-      }
-      // end
-    });
-  };
-  next();
-}, 5000);
-
-exports.upload = function (req, res, next) {
-  var length = Number(req.headers['content-length']) || 0;
-  if (!length || req.headers['content-type'] !== 'application/octet-stream') {
-    return next();
-  }
-
-  var username = req.session.name;
-  var name = req.params.name;
-  var id = Number(req.params.rev);
-  var filename = req.params.filename;
-  var version = filename.substring(name.length + 1);
-  version = version.replace(/\.tgz$/, '');
-  // save version on pkg upload
-
-  debug('%s: upload %s, file size: %d', username, req.url, length);
-  Module.getById(id, function (err, mod) {
-    if (err || !mod) {
-      return next(err);
-    }
-    var match = mod.package.maintainers.filter(function (item) {
-      return item.name === username;
-    });
-    if (match.length === 0 || mod.name !== name) {
-      return res.json(403, {
-        error: 'no_perms',
-        reason: 'Current user can not publish this module'
-      });
-    }
-
-    if (mod.version !== 'next') {
-      // rev wrong
-      return res.json(403, {
-        error: 'rev_wrong',
-        reason: 'rev not match next module'
-      });
-    }
-
-    var filepath = common.getTarballFilepath(filename);
-    var ws = fs.createWriteStream(filepath);
-    var shasum = crypto.createHash('sha1');
-    req.pipe(ws);
-    var dataSize = 0;
-    req.on('data', function (data) {
-      shasum.update(data);
-      dataSize += data.length;
-    });
-    ws.on('finish', function () {
-      if (dataSize !== length) {
-        return res.json(403, {
-          error: 'size_wrong',
-          reason: 'Header size ' + length + ' not match download size ' + dataSize,
-        });
-      }
-      shasum = shasum.digest('hex');
-      var key = common.getCDNKey(name, filename);
-      nfs.upload(filepath, {key: key, size: length}, function (err, result) {
-        // remove tmp file whatever
-        fs.unlink(filepath, utility.noop);
-        if (err) {
-          return next(err);
-        }
-
-        var dist = {
-          tarball: result.url,
-          shasum: shasum,
-          size: length
-        };
-        mod.package.dist = dist;
-        mod.package.version = version;
-        debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
-          id, filepath, length, shasum, dist, version);
-        Module.update(mod, function (err, result) {
-          if (err) {
-            return next(err);
-          }
-          res.json(201, {ok: true, rev: String(result.id)});
-        });
-      });
-    });
-  });
-};
-
-exports.updateLatest = function (req, res, next) {
-  var username = req.session.name;
-  var name = req.params.name;
-  var version = semver.valid(req.params.version);
-  if (!version) {
-    return res.json(400, {
-      error: 'Params Invalid',
-      reason: 'Invalid version: ' + req.params.version,
-    });
-  }
-  Module.get(name, 'next', function (err, nextMod) {
-    if (err) {
-      return next(err);
-    }
-    if (!nextMod) {
-      return next();
-    }
-    var match = nextMod.package.maintainers.filter(function (item) {
-      return item.name === username;
-    });
-    if (match.length === 0) {
-      return res.json(401, {
-        error: 'noperms',
-        reason: 'Current user can not publish this module'
-      });
-    }
-
-    // check version if not match pkg upload
-    if (nextMod.package.version !== version) {
-      return res.json(403, {
-        error: 'version_wrong',
-        reason: 'version not match'
-      });
-    }
-
-    var body = req.body;
-    nextMod.version = version;
-    nextMod.author = username;
-    body.dist = nextMod.package.dist;
-    body.maintainers = nextMod.package.maintainers;
-    if (!body.author) {
-      body.author = {
-        name: username,
-      };
-    }
-    body._publish_on_cnpm = true;
-    nextMod.package = body;
-    debug('update %s:%s %j', nextMod.package.name, nextMod.package.version, nextMod.package.dist);
-    // change latest to version
-    Module.update(nextMod, function (err) {
-      if (err) {
-        return next(err);
-      }
-      // set latest tag
-      Module.addTag(name, 'latest', version, function (err) {
-        if (err) {
-          return next(err);
-        }
-        // add a new next version
-        nextMod.version = 'next';
-        Module.add(nextMod, function (err, result) {
-          if (err) {
-            return next(err);
-          }
-          res.json(201, {ok: true, rev: String(result.id)});
-        });
-      });
-    });
-  });
-};
-
-exports.add = function (req, res, next) {
-  var username = req.session.name;
-  var name = req.params.name;
-  var pkg = req.body;
-  var maintainers = pkg.maintainers || [];
-  var match = maintainers.filter(function (item) {
-    return item.name === username;
-  });
-  if (match.length === 0) {
-    return res.json(403, {
-      error: 'no_perms',
-      reason: 'Current user can not publish this module'
-    });
-  }
-
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Module.getLatest(name, ep.doneLater('latest'));
-  Module.get(name, 'next', ep.done(function (nextMod) {
-    if (nextMod) {
-      nextMod.exists = true;
-      return ep.emit('next', nextMod);
-    }
-    // ensure next module exits
-    // because updateLatest will create next module fail
-    nextMod = {
-      name: name,
-      version: 'next',
-      author: username,
-      package: {
-        name: name,
-        version: 'next',
-        description: pkg.description,
-        readme: pkg.readme,
-        maintainers: pkg.maintainers,
-      },
-    };
-    Module.add(nextMod, ep.done(function (result) {
-      nextMod.id = result.id;
-      ep.emit('next', nextMod);
-    }));
-  }));
-
-  ep.all('latest', 'next', function (latestMod, nextMod) {
-    var maintainers = latestMod ? latestMod.package.maintainers : nextMod.package.maintainers;
-    var match = maintainers.filter(function (item) {
-      return item.name === username;
-    });
-
-    if (match.length === 0) {
-      return res.json(403, {
-        error: 'no_perms',
-        reason: 'Current user can not publish this module'
-      });
-    }
-
-    if (latestMod || nextMod.exists) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-
-    res.json(201, {
-      ok: true,
-      id: name,
-      rev: String(nextMod.id),
-    });
-  });
-};
-
-exports.removeWithVersions = function (req, res, next) {
-  debug('removeWithVersions module %s, with info %j', req.params.name, req.body);
-  var name = req.params.name;
-  var username = req.session.name;
-  var versions = req.body.versions || {};
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Module.listByName(name, ep.doneLater('list'));
-  ep.once('list', function (mods) {
-    if (!mods || !mods.length) {
-      return next();
-    }
-    //TODO replace this maintainer check
-    var match = mods[0].package.maintainers.filter(function (item) {
-      return item.name === username;
-    });
-
-    if (!match.length || mods[0].name !== name) {
-      return res.json(403, {
-        error: 'no_perms',
-        reason: 'Current user can not update this module'
-      });
-    }
-
-    var removeVersions = [];
-    for (var i = 0; i < mods.length; i++) {
-      var v = mods[i].version;
-      if (v !== 'next' && !versions[v]) {
-        removeVersions.push(v);
-      }
-    }
-    if (!removeVersions.length) {
-      return res.json(201, {ok: true});
-    }
-    Module.removeByNameAndVersions(name, removeVersions, ep.done(function () {
-      res.json(201, {ok: true});
-    }));
-  });
-};
-
-
-exports.removeTar = function (req, res, next) {
-  debug('remove tarball with filename: %s, id: %s', req.params.filename, req.params.rev);
-  var id = Number(req.params.rev);
-  var filename = req.params.filename;
-  var name = req.params.name;
-  var username = req.session.name;
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Module.getById(id, ep.doneLater('get'));
-  ep.once('get', function (mod) {
-    if (!mod) {
-      return next();
-    }
-    //TODO replace this maintainer check
-    var match = mod.package.maintainers.filter(function (item) {
-      return item.name === username;
-    });
-    if (!match.length || mod.name !== name) {
-      return res.json(403, {
-        error: 'no_perms',
-        reason: 'Current user can not delete this tarball'
-      });
-    }
-
-    var key = common.getCDNKey(mod.name, filename);
-    nfs.remove(key, ep.done(function () {
-      res.json(200, {ok: true});
-    }));
-  });
-};
-
-exports.removeAll = function (req, res, next) {
-  debug('remove all the module with name: %s, id: %s', req.params.name, req.params.rev);
-  var id = Number(req.params.rev);
-  var name = req.params.name;
-  var username = req.session.name;
-
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Module.listByName(name, ep.doneLater('list'));
-  ep.once('list', function (mods) {
-    var mod = mods[0];
-    if (!mod) {
-      return next();
-    }
-    //TODO replace this maintainer check
-    var match = mod.package.maintainers.filter(function (item) {
-      return item.name === username;
-    });
-    if (!match.length || mod.name !== name) {
-      return res.json(403, {
-        error: 'no_perms',
-        reason: 'Current user can not delete this tarball'
-      });
-    }
-    Total.plusDeleteModule(utility.noop);
-    Module.removeByName(name, ep.done('remove'));
-    Module.removeTags(name, ep.done('removeTags'));
-  });
-
-  ep.all('list', 'remove', 'removeTags', function (mods) {
-    var keys = [];
-    for (var i = 0; i < mods.length; i++) {
-      var key = urlparse(mods[i].dist_tarball).path;
-      key && keys.push(key);
-    }
-    var queue = new Bagpipe(5);
-    keys.forEach(function (key) {
-      queue.push(nfs.remove, key, function () {
-        //ignore err here
-        ep.emit('removeTar');
-      });
-    });
-    ep.after('removeTar', keys.length, function () {
-      res.json(200, {ok: true});
-    });
-  });
-};
-
-function parseModsForList(updated, mods, req) {
-  var results = {
-    _updated: updated
-  };
-
-  for (var i = 0; i < mods.length; i++) {
-    var mod = mods[i];
-    var pkg = {};
-    try {
-      pkg = JSON.parse(mod.package);
-    } catch (e) {
-      //ignore this pkg
-      continue;
-    }
-    pkg['dist-tags'] = {
-      latest: pkg.version
-    };
-    common.downloadURL(pkg, req);
-    results[mod.name] = pkg;
-  }
-  return results;
-}
-
-exports.listAllModules = function (req, res, next) {
-  var updated = Date.now();
-  Module.listSince(0, function (err, mods) {
-    if (err) {
-      return next(err);
-    }
-    return res.json(parseModsForList(updated, mods, req));
-  });
-};
-
-exports.listAllModulesSince = function (req, res, next) {
-  var query = req.query || {};
-  if (query.stale !== 'update_after') {
-    return res.json(400, {
-      error: 'query_parse_error',
-      reason: 'Invalid value for `stale`.'
-    });
-  }
-  debug('list all modules from %s', req.startkey);
-  var startkey = Number(query.startkey) || 0;
-  var updated = Date.now();
-  Module.listSince(startkey, function (err, mods) {
-    if (err) {
-      return next(err);
-    }
-    res.json(parseModsForList(updated, mods, req));
-  });
-};
-
-exports.listAllModuleNames = function (req, res, next) {
-  Module.listShort(function (err, mods) {
-    if (err) {
-      return next(err);
-    }
-    var results = mods.map(function (m) {
-      return m.name;
-    });
-    res.json(results);
-  });
-};

controllers/registry/package/changes.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+var lodash = require('lodash');
+var gather = require('co-gather');
+
+// GET /-/_changes?since={timestamp}&limit={number}&cursorId={number}
+// List changes since the timestamp
+// Similar with https://registry.npmmirror.com/_changes?since=1658974943840
+// Change types:
+// 1. ✅ PACKAGE_VERSION_ADDED
+// 2. ✅ PACKAGE_TAG_ADDED
+// 3. 🆕 PACKAGE_UNPUBLISHED
+// 5. ❎ PACKAGE_MAINTAINER_REMOVED
+// 6. ❎ PACKAGE_MAINTAINER_CHANGED
+// 7. ❎ PACKAGE_TAG_CHANGED
+//
+// Since we don't have the previous data,
+// We can't compute the reliable seqId
+// use gmt_modified cinstead of seqId
+module.exports = function* listSince() {
+  var query = this.query;
+  var since = query.since;
+  var limit = Number(query.limit);
+
+  // ensure limit
+  if (Number.isNaN(limit)) {
+    limit = 1000;
+  }
+  var queryResults = yield gather(
+    [
+      "listVersionSince",
+      "listTagSince",
+      "listUnpublishedModuleSince",
+    ].map(function (method) {
+      return packageService[method](since, limit);
+    })
+  );
+
+  var validResults = queryResults.map(function (result) {
+    if (!result.isError) {
+      return result.value;
+    }
+    return [];
+  });
+
+  var results = lodash.orderBy(
+    lodash.flatten(validResults).filter(Boolean),
+    "gmt_modified",
+    "asc"
+  ).slice(0, limit);
+  this.body = { results };
+};

controllers/registry/package/deprecate.js

@@ -0,0 +1,47 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+
+module.exports = deprecateVersions;
+
+/**
+ * @see https://github.com/cnpm/cnpmjs.org/issues/415
+ */
+function* deprecateVersions() {
+  var body = this.request.body;
+  var name = this.params.name || this.params[0];
+
+  var tasks = [];
+  for (var version in body.versions) {
+    tasks.push(packageService.getModule(name, version));
+  }
+  var rs = yield tasks;
+
+  var updateTasks = [];
+  for (var i = 0; i < rs.length; i++) {
+    var row = rs[i];
+    if (!row) {
+      // some version not exists
+      this.status = 400;
+      const error = '[version_error] Some versions: ' + JSON.stringify(Object.keys(body.versions)) + ' not found';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+    var data = body.versions[row.package.version];
+    if (typeof data.deprecated === 'string') {
+      row.package.deprecated = data.deprecated;
+      updateTasks.push(packageService.updateModulePackage(row.id, row.package));
+    }
+  }
+  yield updateTasks;
+  // update last modified
+  yield packageService.updateModuleLastModified(name);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+  };
+}

controllers/registry/package/dist_tag.js

@@ -0,0 +1,117 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+var hook = require('../../../services/hook');
+
+function ok() {
+  return {
+    ok: "dist-tags updated"
+  };
+}
+
+// GET /-/package/:pkg/dist-tags -- returns the package's dist-tags
+exports.index = function* () {
+  var name = this.params.name || this.params[0];
+  var rows = yield packageService.listModuleTags(name);
+  var tags = {};
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    tags[row.tag] = row.version;
+  }
+  this.body = tags;
+};
+
+// PUT /-/package/:pkg/dist-tags -- Set package's dist-tags to provided object body (removing missing)
+exports.save = function* () {
+  var name = this.params.name || this.params[0];
+  yield packageService.removeModuleTags(name);
+  yield exports.update.call(this);
+};
+
+// POST /-/package/:pkg/dist-tags -- Add/modify dist-tags from provided object body (merge)
+exports.update = function* () {
+  var name = this.params.name || this.params[0];
+  var tags = this.request.body;
+  for (var tag in tags) {
+    var version = tags[tag];
+    yield packageService.addModuleTag(name, tag, version);
+
+    // hooks
+    const envelope = {
+      event: 'package:dist-tag',
+      name: name,
+      tag: tag,
+      type: 'package',
+      version: version,
+      hookOwner: null,
+      payload: null,
+      change: null,
+    };
+    hook.trigger(envelope);
+  }
+  this.status = 201;
+  this.body = ok();
+};
+
+// PUT /-/package/:pkg/dist-tags/:tag -- Set package's dist-tags[tag] to provided string body
+// POST /-/package/:pkg/dist-tags/:tag -- Same as PUT /-/package/:pkg/dist-tags/:tag
+exports.set = function* () {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  var version = this.request.body;
+  // make sure version exists
+  var pkg = yield packageService.getModule(name, version);
+  if (!pkg) {
+    this.status = 400;
+    const error = '[version_error] ' + name + '@' + version + ' not exists';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  yield packageService.addModuleTag(name, tag, version);
+  this.status = 201;
+  this.body = ok();
+  // hooks
+  const envelope = {
+    event: 'package:dist-tag',
+    name: name,
+    tag: tag,
+    type: 'package',
+    version: version,
+    hookOwner: null,
+    payload: null,
+    change: null,
+  };
+  hook.trigger(envelope);
+};
+
+// DELETE /-/package/:pkg/dist-tags/:tag -- Remove tag from dist-tags
+exports.destroy = function* () {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  if (tag === 'latest') {
+    this.status = 400;
+    const error = '[dist_tag_error] Can\'t not delete latest tag';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+  yield packageService.removeModuleTagsByNames(name, tag);
+  this.body = ok();
+  // hooks
+  const envelope = {
+    event: 'package:dist-tag:rm',
+    name: name,
+    tag: tag,
+    type: 'package',
+    hookOwner: null,
+    payload: null,
+    change: null,
+  };
+  hook.trigger(envelope);
+};

controllers/registry/package/download.js

@@ -0,0 +1,142 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:download');
+var mime = require('mime');
+var utility = require('utility');
+var defer = require('co-defer');
+var is = require('is-type-of');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+var common = require('../../../lib/common');
+var downloadAsReadStream = require('../../utils').downloadAsReadStream;
+var packageService = require('../../../services/package');
+var downloadTotalService = require('../../../services/download_total');
+var config = require('../../../config');
+
+let globalDownloads = new Map();
+
+module.exports = function* download(next) {
+  var name = this.params.name || this.params[0];
+  var filename = this.params.filename || this.params[1];
+  // scope pkg and download with out scope
+  if (name.startsWith('@') && !filename.startsWith('@')) {
+    var scope = name.slice(0, name.indexOf('/'));
+    // fix filename with scope
+    filename = `${scope}/${filename}`;
+  }
+
+  var version = filename.slice(name.length + 1, -4);
+  // can not get dist
+  var url = null;
+
+  var query = this.query || {};
+  // allow download from specific store bucket
+  var options = query.bucket ? { bucket: query.bucket } : null;
+
+  if (typeof nfs.url === 'function') {
+    if (is.generatorFunction(nfs.url)) {
+      url = yield nfs.url(common.getCDNKey(name, filename), options);
+    } else {
+      url = nfs.url(common.getCDNKey(name, filename), options);
+    }
+  }
+
+  debug('download %s %s %s %s', name, filename, version, url);
+  // don't check database and just download tgz from nfs
+  if (config.downloadTgzDontCheckModule && url) {
+    this.status = 302;
+    this.set('Location', url);
+    const count = (globalDownloads.get(name) || 0) + 1;
+    globalDownloads.set(name, count);
+    return;
+  }
+
+  var row = yield packageService.getModule(name, version);
+  if (!row || !row.package || !row.package.dist) {
+    if (!url) {
+      return yield next;
+    }
+    this.status = 302;
+    this.set('Location', url);
+    const count = (globalDownloads.get(name) || 0) + 1;
+    globalDownloads.set(name, count);
+    return;
+  }
+
+  const count = (globalDownloads.get(name) || 0) + 1;
+  globalDownloads.set(name, count);
+
+  if (config.downloadRedirectToNFS && url) {
+    this.status = 302;
+    this.set('Location', url);
+    return;
+  }
+
+  var dist = row.package.dist;
+  if (!dist.key) {
+    // try to use nsf.url() first
+    url = url || dist.tarball;
+    debug('get tarball by 302, url: %s', url);
+    this.status = 302;
+    this.set('Location', url);
+    return;
+  }
+
+  // else use `dist.key` to get tarball from nfs
+  if (typeof dist.size === 'number' && dist.size > 0) {
+    this.length = dist.size;
+  }
+  this.type = mime.lookup(dist.key);
+  this.attachment(filename);
+  this.etag = dist.shasum;
+
+  this.body = yield downloadAsReadStream(dist.key);
+};
+
+var saving = false;
+defer.setInterval(function* () {
+  if (saving) {
+    return;
+  }
+
+  // save download count
+  var totals = [];
+  var allCount = 0;
+  for (const [ name, count ] of globalDownloads) {
+    if (name !== '__all__') {
+      totals.push([name, count]);
+    }
+    allCount += count;
+  }
+  globalDownloads = new Map();
+
+  if (allCount === 0) {
+    return;
+  }
+
+  saving = true;
+  totals.push([ '__all__', allCount ]);
+  debug('save download total: %j', totals);
+
+  var date = utility.YYYYMMDD();
+  for (var i = 0; i < totals.length; i++) {
+    var item = totals[i];
+    var name = item[0];
+    var count = item[1];
+    try {
+      yield downloadTotalService.plusModuleTotal({ name: name, date: date, count: count });
+    } catch (err) {
+      if (err.name !== 'SequelizeUniqueConstraintError') {
+        err.message += '; name: ' + name + ', count: ' + count + ', date: ' + date;
+        logger.error(err);
+      }
+      var pkgExist = yield packageService.getModuleLastModified(name);
+      if (pkgExist) {
+        // save back to globalDownloads, try again next time
+        count = (globalDownloads.get(name) || 0) + count;
+        globalDownloads.set(name, count);
+      }
+    }
+  }
+  saving = false;
+}, 5000 + Math.ceil(Math.random() * 1000));

controllers/registry/package/download_total.js

@@ -0,0 +1,67 @@
+'use strict';
+
+var DownloadTotal = require('../../../services/download_total');
+var DATE_REG = /^\d{4}-\d{2}-\d{2}$/;
+
+module.exports = function* downloadTotal() {
+  var range = this.params.range || this.params[0] || '';
+  var name = this.params.name || this.params[1];
+
+  range = range.split(':');
+  if (range.length !== 2
+      || !range[0].match(DATE_REG)
+      || !range[1].match(DATE_REG)) {
+    this.status = 400;
+    const error = '[range_error] range must be YYYY-MM-DD:YYYY-MM-DD style';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  this.body = name
+    ? yield getPackageTotal(name, range[0], range[1])
+    : yield getTotal(range[0], range[1]);
+};
+
+function* getPackageTotal(name, start, end) {
+  var res = yield DownloadTotal.getModuleTotal(name, start, end);
+  var downloads = res.map(function (row) {
+    return {
+      day: row.date,
+      downloads: row.count
+    };
+  });
+
+  downloads.sort(function (a, b) {
+    return a.day > b.day ? 1 : -1;
+  });
+
+  return {
+    downloads: downloads,
+    package: name,
+    start: start,
+    end: end
+  };
+}
+
+function* getTotal(start, end) {
+  var res = yield DownloadTotal.getTotal(start, end);
+  var downloads = res.map(function (row) {
+    return {
+      day: row.date,
+      downloads: row.count
+    };
+  });
+
+  downloads.sort(function (a, b) {
+    return a.day > b.day ? 1 : -1;
+  });
+
+  return {
+    downloads: downloads,
+    start: start,
+    end: end
+  };
+}

controllers/registry/package/list.js

@@ -0,0 +1,489 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:list');
+var utility = require('utility');
+var packageService = require('../../../services/package');
+var blocklistService = require('../../../services/blocklist');
+var bugVersionService = require('../../../services/bug_version');
+var common = require('../../../lib/common');
+var SyncModuleWorker = require('../../sync_module_worker');
+var config = require('../../../config');
+const cache = require('../../../common/cache');
+const logger = require('../../../common/logger');
+
+// https://forum.nginx.org/read.php?2,240120,240120#msg-240120
+// should set weak etag avoid nginx remove it
+function etag(objs) {
+  return 'W/"' + utility.md5(JSON.stringify(objs)) + '"';
+}
+
+function filterBlockVerions(rows, blocks) {
+  if (!blocks) {
+    return rows;
+  }
+  return rows.filter(row => !blocks[row.version]);
+}
+
+/**
+ * list all version of a module
+ * GET /:name
+ */
+module.exports = function* list() {
+  const name = this.params.name || this.params[0];
+  const isSyncWorkerRequest = common.isSyncWorkerRequest(this);
+  const isJSONPRequest = this.query.callback;
+  let cacheKey = '';
+  let needAbbreviatedMeta = false;
+  let abbreviatedMetaType = 'application/vnd.npm.install-v1+json';
+  if (config.enableAbbreviatedMetadata && this.accepts([ 'json', abbreviatedMetaType ]) === abbreviatedMetaType) {
+    needAbbreviatedMeta = true;
+    // don't cache result on sync request
+    if (cache && !isJSONPRequest && !isSyncWorkerRequest) {
+      cacheKey = `list-${name}-v1`;
+    }
+  }
+  
+  if (cacheKey) {
+    const values = yield cache.hmget(cacheKey, 'etag', 'body');
+    if (values && values[0] && values[1]) {
+      this.body = values[1];
+      this.type = 'json';
+      this.etag = values[0];
+      this.set('x-hit-cache', cacheKey);
+      debug('hmget %s success, etag:%j', cacheKey, values[0]);
+      if (config.registryCacheControlHeader) {
+        this.set('cache-control', config.registryCacheControlHeader);
+      }
+      if (config.registryVaryHeader) {
+        this.set('vary', config.registryVaryHeader);
+      }
+      return;
+    }
+    debug('hmget %s missing, %j', cacheKey, values);
+  }
+
+  var rs = yield [
+    packageService.getModuleLastModified(name),
+    packageService.listModuleTags(name),
+    blocklistService.findBlockPackageVersions(name),
+  ];
+  var modifiedTime = rs[0];
+  var tags = rs[1];
+  var blocks = rs[2];
+
+  if (blocks && blocks['*']) {
+    this.status = 451;
+    const error = `[block] package was blocked, reason: ${blocks['*'].reason}`;
+    this.jsonp = {
+      name,
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  debug('show %s, last modified: %s, tags: %j', name, modifiedTime, tags);
+  if (modifiedTime) {
+    // find out the latest modfied time
+    // because update tags only modfied tag, wont change module gmt_modified
+    for (var i = 0; i < tags.length; i++) {
+      var tag = tags[i];
+      if (tag.gmt_modified > modifiedTime) {
+        modifiedTime = tag.gmt_modified;
+      }
+    }
+
+    // must set status first
+    this.status = 200;
+    if (this.fresh) {
+      debug('%s not change at %s, 304 return', name, modifiedTime);
+      this.status = 304;
+      return;
+    }
+  }
+
+  if (needAbbreviatedMeta) {
+    var rows = yield packageService.listModuleAbbreviatedsByName(name);
+    rows = filterBlockVerions(rows, blocks);
+    if (rows.length > 0) {
+      yield handleAbbreviatedMetaRequest(this, name, modifiedTime, tags, rows, cacheKey, isSyncWorkerRequest);
+      return;
+    }
+    var fullRows = yield packageService.listModulesByName(name);
+    fullRows = filterBlockVerions(fullRows, blocks);
+    if (fullRows.length > 0) {
+      // no abbreviated meta rows, use the full meta convert to abbreviated meta
+      yield handleAbbreviatedMetaRequestWithFullMeta(this, name, modifiedTime, tags, fullRows, isSyncWorkerRequest);
+      return;
+    }
+  }
+
+  var r = yield [
+    packageService.listModulesByName(name),
+    packageService.listStarUserNames(name),
+    packageService.listMaintainers(name),
+  ];
+  var rows = filterBlockVerions(r[0], blocks);
+  var starUsers = r[1];
+  var maintainers = r[2];
+
+  debug('show %s got %d rows, %d tags, %d star users, maintainers: %j',
+    name, rows.length, tags.length, starUsers.length, maintainers);
+
+  var starUserMap = {};
+  for (var i = 0; i < starUsers.length; i++) {
+    var starUser = starUsers[i];
+    if (starUser[0] !== '"' && starUser[0] !== "'") {
+      starUserMap[starUser] = true;
+    }
+  }
+  starUsers = starUserMap;
+
+  if (rows.length === 0) {
+    // check if unpublished
+    var unpublishedInfo = yield packageService.getUnpublishedModule(name);
+    debug('show unpublished %j', unpublishedInfo);
+    if (unpublishedInfo) {
+      this.status = 404;
+      this.jsonp = {
+        _id: name,
+        name: name,
+        time: {
+          modified: unpublishedInfo.package.time,
+          unpublished: unpublishedInfo.package,
+        },
+        _attachments: {},
+      };
+      return;
+    }
+  }
+
+  // if module not exist in this registry,
+  // sync the module backend and return package info from official registry
+  if (rows.length === 0) {
+    if (!this.allowSync) {
+      this.status = 404;
+      const error = '[not_found] document not found';
+      this.jsonp = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+
+    // start sync
+    var logId = yield SyncModuleWorker.sync(name, 'sync-by-install');
+    debug('start sync %s, get log id %s', name, logId);
+
+    return this.redirect(config.officialNpmRegistry + this.url);
+  }
+
+  var latestMod = null;
+  var readme = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  var allVersionString = '';
+  var times = {};
+  var attachments = {};
+  var createdTime = null;
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    var pkg = row.package;
+    // pkg is string ... ignore it
+    if (typeof pkg === 'string') {
+      continue;
+    }
+    common.setDownloadURL(pkg, this);
+    pkg._cnpm_publish_time = row.publish_time;
+    pkg.publish_time = pkg.publish_time || row.publish_time;
+
+    versions[pkg.version] = pkg;
+    allVersionString += pkg.version + ',';
+
+    var t = times[pkg.version] = row.publish_time ? new Date(row.publish_time) : row.gmt_modified;
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+      readme = pkg.readme;
+    }
+
+    delete pkg.readme;
+    if (maintainers.length > 0) {
+      pkg.maintainers = maintainers;
+    }
+
+    if (!createdTime || t < createdTime) {
+      createdTime = t;
+    }
+  }
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
+  }
+
+  if (modifiedTime && createdTime) {
+    var ts = {
+      modified: modifiedTime,
+      created: createdTime,
+    };
+    for (var t in times) {
+      ts[t] = times[t];
+    }
+    times = ts;
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  var rev = String(latestMod.id);
+  var pkg = latestMod.package;
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = pkg.version;
+  }
+
+  if (!readme && config.enableAbbreviatedMetadata) {
+    var packageReadme = yield packageService.getPackageReadme(name);
+    if (packageReadme) {
+      readme = packageReadme.readme;
+    }
+  }
+
+  var info = {
+    _id: name,
+    _rev: rev,
+    name: name,
+    description: pkg.description,
+    'dist-tags': distTags,
+    maintainers: pkg.maintainers,
+    time: times,
+    users: starUsers,
+    author: pkg.author,
+    repository: pkg.repository,
+    versions: versions,
+    readme: readme,
+    _attachments: attachments,
+  };
+
+  info.readmeFilename = pkg.readmeFilename;
+  info.homepage = pkg.homepage;
+  info.bugs = pkg.bugs;
+  info.license = pkg.license;
+
+  if (typeof config.formatCustomFullPackageInfoAndVersions === 'function') {
+    info = config.formatCustomFullPackageInfoAndVersions(this, info);
+  }
+
+  debug('show module %s: %s, latest: %s', name, rev, latestMod.version);
+  this.jsonp = info;
+  // use faster etag
+  this.etag = etag([
+    modifiedTime,
+    distTags,
+    pkg.maintainers,
+    allVersionString,
+  ]);
+
+  if (config.registryCacheControlHeader) {
+    this.set('cache-control', config.registryCacheControlHeader);
+  }
+  if (config.registryVaryHeader) {
+    this.set('vary', config.registryVaryHeader);
+  }
+};
+
+function* handleAbbreviatedMetaRequest(ctx, name, modifiedTime, tags, rows, cacheKey, isSyncWorkerRequest) {
+  debug('show %s got %d rows, %d tags, modifiedTime: %s, cacheKey: %s, isSyncWorkerRequest: %s',
+    name, rows.length, tags.length, modifiedTime, cacheKey, isSyncWorkerRequest);
+  const isJSONPRequest = ctx.query.callback;
+  var latestMod = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  var allVersionString = '';
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    var pkg = row.package;
+    common.setDownloadURL(pkg, ctx);
+    pkg._publish_on_cnpm = undefined;
+    pkg.publish_time = pkg.publish_time || row.publish_time;
+
+    versions[pkg.version] = pkg;
+    allVersionString += pkg.version + ',';
+
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+    }
+    // abbreviatedMeta row maybe update by syncer on missing attributes add
+    if (!modifiedTime || row.gmt_modified > modifiedTime) {
+      modifiedTime = row.gmt_modified;
+    }
+  }
+  // don't use bug-versions hotfix on sync request
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = latestMod.package.version;
+  }
+
+  var info = {
+    name: name,
+    modified: modifiedTime,
+    'dist-tags': distTags,
+    versions: versions,
+  };
+
+  debug('show %j', info);
+  // use faster etag
+  const resultEtag = etag([
+    modifiedTime,
+    distTags,
+    allVersionString,
+  ]);
+
+  if (isJSONPRequest) {
+    ctx.jsonp = info;
+  } else {
+    ctx.body = JSON.stringify(info);
+    ctx.type = 'json';
+    // set cache
+    if (cacheKey) {
+      // set cache async, dont block the response
+      cache.pipeline()
+        .hmset(cacheKey, 'etag', resultEtag, 'body', ctx.body)
+        // cache 120s
+        .expire(cacheKey, 120)
+        .exec()
+        .catch(err => {
+          logger.error(err);
+        });
+    }
+  }
+  ctx.etag = resultEtag;
+  if (config.registryCacheControlHeader) {
+    ctx.set('cache-control', config.registryCacheControlHeader);
+  }
+  if (config.registryVaryHeader) {
+    ctx.set('vary', config.registryVaryHeader);
+  }
+}
+
+function* handleAbbreviatedMetaRequestWithFullMeta(ctx, name, modifiedTime, tags, rows, isSyncWorkerRequest) {
+  debug('show %s got %d rows, %d tags, isSyncWorkerRequest: %s',
+    name, rows.length, tags.length, isSyncWorkerRequest);
+  var latestMod = null;
+  // set tags
+  var distTags = {};
+  for (var i = 0; i < tags.length; i++) {
+    var t = tags[i];
+    distTags[t.tag] = t.version;
+  }
+
+  // set versions and times
+  var versions = {};
+  var allVersionString = '';
+  for (var i = 0; i < rows.length; i++) {
+    var row = rows[i];
+    // pkg is string ... ignore it
+    if (typeof row.package === 'string') {
+      continue;
+    }
+    // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object
+    var hasInstallScript;
+    if (row.package.scripts) {
+      // https://www.npmjs.com/package/fix-has-install-script
+      if (row.package.scripts.install || row.package.scripts.preinstall || row.package.scripts.postinstall) {
+        hasInstallScript = true;
+      }
+    }
+    var pkg = {
+      name: row.package.name,
+      version: row.package.version,
+      deprecated: row.package.deprecated,
+      dependencies: row.package.dependencies,
+      optionalDependencies: row.package.optionalDependencies,
+      devDependencies: row.package.devDependencies,
+      bundleDependencies: row.package.bundleDependencies,
+      peerDependencies: row.package.peerDependencies,
+      peerDependenciesMeta: row.package.peerDependenciesMeta,
+      bin: row.package.bin,
+      os: row.package.os,
+      cpu: row.package.cpu,
+      libc: row.package.libc,
+      directories: row.package.directories,
+      dist: row.package.dist,
+      engines: row.package.engines,
+      workspaces: row.package.workspaces,
+      _hasShrinkwrap: row.package._hasShrinkwrap,
+      hasInstallScript: hasInstallScript,
+      publish_time: row.package.publish_time || row.publish_time,
+    };
+    common.setDownloadURL(pkg, ctx);
+
+    versions[pkg.version] = pkg;
+    row.package = pkg;
+    allVersionString += pkg.version + ',';
+
+    if ((!distTags.latest && !latestMod) || distTags.latest === pkg.version) {
+      latestMod = row;
+    }
+  }
+  if (!isSyncWorkerRequest) {
+    yield bugVersionService.hotfix(rows);
+  }
+
+  if (!latestMod) {
+    latestMod = rows[0];
+  }
+
+  if (tags.length === 0) {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = latestMod.package.version;
+  }
+
+  var info = {
+    name: name,
+    modified: modifiedTime,
+    'dist-tags': distTags,
+    versions: versions,
+  };
+
+  debug('show %j', info);
+  ctx.jsonp = info;
+  // use faster etag
+  ctx.etag = etag([
+    modifiedTime,
+    distTags,
+    allVersionString,
+  ]);
+
+  if (config.registryCacheControlHeader) {
+    ctx.set('cache-control', config.registryCacheControlHeader);
+  }
+  if (config.registryVaryHeader) {
+    ctx.set('vary', config.registryVaryHeader);
+  }
+}

controllers/registry/package/list_all.js

@@ -0,0 +1,16 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+
+// GET /-/all
+// List all packages names
+// https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L86
+module.exports = function* () {
+  var updated = Date.now();
+  var names = yield packageService.listAllPublicModuleNames();
+  var result = { _updated: updated };
+  names.forEach(function (name) {
+    result[name] = true;
+  });
+  this.body = result;
+};

controllers/registry/package/list_by_user.js

@@ -0,0 +1,29 @@
+/**!
+ * list packages by username
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+const packageService = require('../../../services/package');
+
+module.exports = function*() {
+  const username = this.params.user;
+  const packages = yield packageService.listModulesByUser(username);
+
+  this.body = {
+    user: {
+      name: username,
+    },
+    packages: packages,
+  };
+};

controllers/registry/package/list_dependents.js

@@ -0,0 +1,26 @@
+/**!
+ * list package's dependents
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <m@fengmk2.com> (http://fengmk2.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+const packageService = require('../../../services/package');
+
+module.exports = function*() {
+  const name = this.params.name || this.params[0];
+  const dependents = yield packageService.listDependents(name);
+
+  this.body = {
+    dependents: dependents,
+  };
+};

controllers/registry/package/list_shorts.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const packageService = require('../../../services/package');
+const config = require('../../../config');
+
+// GET /-/short
+// List public all packages names only
+module.exports = function* () {
+  if (this.query.private_only) {
+    const tasks = [];
+    for (let i = 0; i < config.scopes.length; i++) {
+      const scope = config.scopes[i];
+      tasks.push(packageService.listPrivateModulesByScope(scope));
+    }
+
+    if (config.privatePackages && config.privatePackages.length > 0) {
+      tasks.push(packageService.listModules(config.privatePackages));
+    }
+
+    const results = yield tasks;
+    const names = [];
+    for (const rows of results) {
+      for (const row of rows) {
+        names.push(row.name);
+      }
+    }
+    this.body = names;
+    return;
+  }
+
+  this.body = yield packageService.listAllPublicModuleNames();
+};

controllers/registry/package/list_since.js

@@ -0,0 +1,48 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+
+var A_WEEK_MS = 3600000 * 24 * 7;
+var TWA_DAYS_MS = 3600000 * 24 * 2;
+
+// GET /-/all/since?stale=update_after&startkey={key}
+// List packages names since startkey
+// https://github.com/npm/npm-registry-client/blob/master/lib/get.js#L89
+module.exports = function* listSince() {
+  var query = this.query;
+  if (query.stale !== 'update_after') {
+    this.status = 400;
+    const error = '[query_parse_error] Invalid value for `stale`.';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var startkey = Number(query.startkey);
+  if (!startkey) {
+    this.status = 400;
+    const error = '[query_parse_error] Invalid value for `startkey`.';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var updated = Date.now();
+  if (updated - startkey > A_WEEK_MS) {
+    startkey = updated - TWA_DAYS_MS;
+    console.warn('[%s] list modules since time out of range: query: %j, ip: %s, limit to %s',
+      Date(), query, this.ip, startkey);
+  }
+
+  var names = yield packageService.listPublicModuleNamesSince(startkey);
+  var result = { _updated: updated };
+  names.forEach(function (name) {
+    result[name] = true;
+  });
+
+  this.body = result;
+};

controllers/registry/package/list_versions.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const moment = require('moment');
+const packageService = require('../../../services/package');
+
+// GET /-/allversions?date={2020-02-20}
+// List all packages versions sync at date(gmt_modified)
+
+module.exports = function* () {
+  const query = this.query;
+  const date = moment(query.date, 'YYYY-MM-DD');
+  if (!date.isValid()) {
+    this.status = 400;
+    const error = '[query_parse_error] Invalid value for `date`, should be `YYYY-MM-DD` format.';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  const today = date.format('YYYY-MM-DD');
+  const rows = yield packageService.findAllModuleAbbreviateds({
+    gmt_modified: {
+      $gte: `${today} 00:00:00`,
+      $lte: `${today} 23:59:59`,
+    },
+  });
+  this.body = rows.map(row => {
+    return {
+      name: row.name,
+      version: row.version,
+      publish_time: new Date(row.publish_time),
+      gmt_modified: row.gmt_modified,
+    };
+  });
+};

controllers/registry/package/remove.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:remove');
+var urlparse = require('url').parse;
+var packageService = require('../../../services/package');
+var totalService = require('../../../services/total');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+var config = require('../../../config');
+
+// DELETE /:name/-rev/:rev
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L25
+module.exports = function* remove(next) {
+  var name = this.params.name || this.params[0];
+  var rev = this.params.rev || this.params[1];
+  debug('remove all the module with name: %s, id: %s', name, rev);
+
+  var mods = yield packageService.listModulesByName(name);
+  debug('removeAll module %s: %d', name, mods.length);
+  var mod = mods[0];
+  if (!mod) {
+    return yield next;
+  }
+
+  yield [
+    packageService.removeModulesByName(name),
+    packageService.removeModuleTags(name),
+    totalService.plusDeleteModule(),
+  ];
+
+  if (config.unpublishRemoveTarball) {
+    var keys = [];
+    for (var i = 0; i < mods.length; i++) {
+      var row = mods[i];
+      var dist = row.package.dist;
+      var key = dist.key;
+      if (!key) {
+        key = urlparse(dist.tarball).pathname;
+      }
+      key && keys.push(key);
+    }
+
+    try {
+      yield keys.map(function (key) {
+        return nfs.remove(key);
+      });
+    } catch (err) {
+      logger.error(err);
+    }
+  }
+
+  // remove the maintainers
+  yield packageService.removeAllMaintainers(name);
+
+  this.body = { ok: true };
+};

controllers/registry/package/remove_version.js

@@ -0,0 +1,65 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:remove_version');
+var packageService = require('../../../services/package');
+var nfs = require('../../../common/nfs');
+var logger = require('../../../common/logger');
+var getCDNKey = require('../../../lib/common').getCDNKey;
+var config = require('../../../config');
+
+// DELETE /:name/download/:filename/-rev/:rev
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L97
+module.exports = function* removeOneVersion(next) {
+  var name = this.params.name || this.params[0];
+  var filename = this.params.filename || this.params[1];
+  var id = Number(this.params.rev || this.params[2]);
+  // cnpmjs.org-2.0.0.tgz
+  var version = filename.split(name + '-')[1];
+  if (version) {
+    // 2.0.0.tgz
+    version = version.substring(0, version.lastIndexOf('.tgz'));
+  }
+  if (!version) {
+    return yield next;
+  }
+
+  debug('remove tarball with filename: %s, version: %s, revert to => rev id: %s', filename, version, id);
+
+  if (isNaN(id)) {
+    return yield next;
+  }
+
+  var rs = yield [
+    packageService.getModuleById(id),
+    packageService.getModule(name, version),
+  ];
+  var revertTo = rs[0];
+  var mod = rs[1]; // module need to delete
+  if (!mod || mod.name !== name) {
+    return yield next;
+  }
+
+  if (config.unpublishRemoveTarball) {
+    var key = mod.package && mod.package.dist && mod.package.dist.key;
+    if (!key) {
+      key = getCDNKey(mod.name, filename);
+    }
+
+    if (revertTo && revertTo.package) {
+      debug('removing key: %s from nfs, revert to %s@%s', key, revertTo.name, revertTo.package.version);
+    } else {
+      debug('removing key: %s from nfs, no revert mod', key);
+    }
+
+    try {
+      yield nfs.remove(key);
+    } catch (err) {
+      logger.error(err);
+    }
+  }
+
+  // remove version from table
+  yield packageService.removeModulesByNameAndVersions(name, [version]);
+  debug('removed %s@%s', name, version);
+  this.body = { ok: true };
+};

controllers/registry/package/save.js

@@ -0,0 +1,332 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:save');
+var ssri = require('ssri');
+var deprecateVersions = require('./deprecate');
+var packageService = require('../../../services/package');
+var logger = require('../../../common/logger');
+var common = require('../../../lib/common');
+var nfs = require('../../../common/nfs');
+var config = require('../../../config');
+var hook = require('../../../services/hook');
+
+// old flows:
+// 1. add()
+// 2. upload()
+// 3. updateLatest()
+//
+// new flows: only one request
+// PUT /:name
+// old publish: https://github.com/npm/npm-registry-client/blob/master/lib/publish.js#L84
+// new publish: https://github.com/npm/libnpmpublish/blob/main/publish.js#L91
+module.exports = function* save(next) {
+  // {
+  //   "_id": "@cnpm/foo",
+  //   "name": "@cnpm/foo",
+  //   "dist-tags": {
+  //     "latest": "1.0.0"
+  //   },
+  //   "versions": {
+  //     "1.0.0": {
+  //       "name": "@cnpm/foo",
+  //       "version": "1.0.0",
+  //       "dependencies": {
+  //         "xprofiler": "^1.2.6"
+  //       },
+  //       "readme": "ERROR: No README data found!",
+  //       "_id": "@cnpm/foo@1.0.0",
+  //       "_nodeVersion": "16.13.0",
+  //       "_npmVersion": "8.1.0",
+  //       "dist": {
+  //         "integrity": "sha512-7nm0vpDEWs7y+tTwlxd7YnGaBc+9Gk5KaPsx2cqQz6H84ndBXlw5nMxGtL4Uy0bCQIknPAZAVe+KNheInmmJrQ==",
+  //         "shasum": "afd05dcfb8759b9b1c7151492a04f2254365c602",
+  //         "tarball": "http://127.0.0.1:7001/@cnpm/foo/-/@cnpm/foo-1.0.0.tgz"
+  //       }
+  //     }
+  //   },
+  //   "access": null,
+  //   "_attachments": {
+  //     "@cnpm/foo-1.0.0.tgz": {
+  //       "content_type": "application/octet-stream",
+  //       "data": "H4sIAAAAAA...",
+  //       "length": 208
+  //     }
+  //   }
+  // }
+  var pkg = this.request.body;
+  var username = this.user.name;
+  var name = this.params.name || this.params[0];
+  var filename = Object.keys(pkg._attachments || {})[0];
+  var version = Object.keys(pkg.versions || {})[0];
+  if (!version) {
+    this.status = 400;
+    const error = '[version_error] package.versions is empty';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  // check maintainers
+  var result = yield packageService.authMaintainer(name, username);
+  if (!result.isMaintainer) {
+    this.status = 403;
+    const error = '[forbidden] ' +  username + ' not authorized to modify ' + name +
+      ', please contact maintainers: ' + result.maintainers.join(', ');
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  if (!filename) {
+    var hasDeprecated = false;
+    for (var v in pkg.versions) {
+      var row = pkg.versions[v];
+      if (typeof row.deprecated === 'string') {
+        hasDeprecated = true;
+        break;
+      }
+    }
+    if (hasDeprecated) {
+      return yield deprecateVersions.call(this, next);
+    }
+
+    this.status = 400;
+    const error = '[attachment_error] package._attachments is empty';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var attachment = pkg._attachments[filename];
+  var versionPackage = pkg.versions[version];
+  var maintainers = versionPackage.maintainers;
+
+  var authorizeType = common.getAuthorizeType(this);
+  if (!maintainers) {
+    if (authorizeType === common.AuthorizeType.BEARER) {
+      // With the token mode, pub lib with no maintainers
+      // make the maintainer to be puber
+      maintainers = [{
+        name: this.user.name,
+        email: this.user.email,
+      }];
+    } else {
+      // should never happened in normal request
+      this.status = 400;
+      const error = '[maintainers_error] request body need maintainers';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+  }
+
+  // notice that admins can not publish to all modules
+  // (but admins can add self to maintainers first)
+
+  var m = maintainers.filter(function (maintainer) {
+    return maintainer.name === username;
+  });
+
+  // package.json has maintainers and publisher in not in the list
+  if (authorizeType === common.AuthorizeType.BEARER && m.length === 0) {
+    var publisher = {
+      name: this.user.name,
+      email: this.user.email,
+    };
+    m = [ publisher ];
+    maintainers.push(publisher);
+  }
+
+  // make sure user in auth is in maintainers
+  // should never happened in normal request
+  if (m.length === 0) {
+    this.status = 403;
+    const error = '[maintainers_error] ' + username + ' does not in maintainer list';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  // TODO: add this info into some table
+  versionPackage._publish_on_cnpm = true;
+  var distTags = pkg['dist-tags'] || {};
+  var tags = []; // tag, version
+  for (var t in distTags) {
+    tags.push([t, distTags[t]]);
+  }
+
+  if (tags.length === 0) {
+    this.status = 400;
+    const error = '[invalid] dist-tags should not be empty';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  debug('%s publish new %s:%s, attachment size: %s, maintainers: %j, distTags: %j',
+    username, name, version, attachment.length, versionPackage.maintainers, distTags);
+
+  var exists = yield packageService.getModule(name, version);
+  if (exists) {
+    this.status = 403;
+    const error = '[forbidden] cannot modify pre-existing version: ' + version;
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  // upload attachment
+  var tarballBuffer;
+  tarballBuffer = Buffer.from(attachment.data, 'base64');
+
+  if (tarballBuffer.length !== attachment.length) {
+    this.status = 403;
+    const error = '[size_wrong] Attachment size ' + attachment.length
+      + ' not match download size ' + tarballBuffer.length;
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  if (!distTags.latest) {
+    // need to check if latest tag exists or not
+    var latest = yield packageService.getModuleByTag(name, 'latest');
+    if (!latest) {
+      // auto add latest
+      tags.push(['latest', tags[0][1]]);
+      debug('auto add latest tag: %j', tags);
+    }
+  }
+
+  var originDist = versionPackage.dist || {};
+  var shasum;
+  var integrity = originDist.integrity;
+  // for content security reason
+  // check integrity
+  if (integrity) {
+    var algorithm = ssri.checkData(tarballBuffer, integrity);
+    if (!algorithm) {
+      logger.error('[registry:save:integrity:invalid] %s@%s, dist:%j', name, version, originDist);
+      this.status = 400;
+      const error = '[invalid] dist.integrity invalid';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+    var integrityObj = ssri.fromData(tarballBuffer, {
+      algorithms: ['sha1'],
+    });
+    shasum = integrityObj.sha1[0].hexDigest();
+  } else {
+    var integrityObj = ssri.fromData(tarballBuffer, {
+      algorithms: ['sha512', 'sha1'],
+    });
+    integrity = integrityObj.sha512[0].toString();
+    shasum = integrityObj.sha1[0].hexDigest();
+    if (originDist.shasum && originDist.shasum !== shasum) {
+      // if integrity not exists, check shasum
+      logger.error('[registry:save:shasum:invalid] %s@%s, dist:%j', name, version, originDist);
+      this.status = 400;
+      const error = '[invalid] dist.shasum invalid';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+  }
+
+  var options = {
+    key: common.getCDNKey(name, filename),
+    shasum: shasum,
+    integrity: integrity,
+  };
+  var uploadResult = yield nfs.uploadBuffer(tarballBuffer, options);
+  debug('upload %j, options: %j', uploadResult, options);
+
+  var dist = Object.assign({}, originDist, {
+    tarball: '',
+    integrity: integrity,
+    shasum: shasum,
+    size: attachment.length,
+  });
+
+  // if nfs upload return a key, record it
+  if (uploadResult.url) {
+    dist.tarball = uploadResult.url;
+  } else if (uploadResult.key) {
+    dist.key = uploadResult.key;
+    dist.tarball = uploadResult.key;
+  }
+
+  var mod = {
+    name: name,
+    version: version,
+    author: username,
+    package: versionPackage
+  };
+
+  mod.package.dist = dist;
+  yield addDepsRelations(mod.package);
+
+  var addResult = yield packageService.saveModule(mod);
+  debug('%s module: save file to %s, size: %d, sha1: %s, dist: %j, version: %s',
+    addResult.id, dist.tarball, dist.size, shasum, dist, version);
+
+  if (tags.length) {
+    yield tags.map(function (tag) {
+      // tag: [tagName, version]
+      return packageService.addModuleTag(name, tag[0], tag[1]);
+    });
+  }
+
+  // ensure maintainers exists
+  var maintainerNames = maintainers.map(function (item) {
+    return item.name;
+  });
+  yield packageService.addPrivateModuleMaintainers(name, maintainerNames);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    rev: String(addResult.id)
+  };
+
+  // hooks
+  const envelope = {
+    event: 'package:publish',
+    name: mod.name,
+    type: 'package',
+    version: mod.version,
+    hookOwner: null,
+    payload: null,
+    change: null,
+  };
+  hook.trigger(envelope);
+};
+
+function* addDepsRelations(pkg) {
+  var dependencies = Object.keys(pkg.dependencies || {});
+  if (dependencies.length > config.maxDependencies) {
+    dependencies = dependencies.slice(0, config.maxDependencies);
+  }
+  yield packageService.addDependencies(pkg.name, dependencies);
+}

controllers/registry/package/show.js

@@ -0,0 +1,52 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:show');
+var packageService = require('../../../services/package');
+var SyncModuleWorker = require('../../sync_module_worker');
+var config = require('../../../config');
+
+/**
+ * [deprecate] api
+ *
+ * get the special version or tag of a module
+ *
+ * GET /:name/:version
+ * GET /:name/:tag
+ */
+module.exports = function* show() {
+  var name = this.params.name || this.params[0];
+  var tag = this.params.version || this.params[1];
+  var mod = yield packageService.showPackage(name, tag, this);
+
+  if (mod) {
+    if (typeof config.formatCustomOnePackageVersion === 'function') {
+      mod.package = config.formatCustomOnePackageVersion(this, mod.package);
+    }
+
+    this.jsonp = mod.package;
+    if (config.registryCacheControlHeader) {
+      this.set('cache-control', config.registryCacheControlHeader);
+    }
+    if (config.registryVaryHeader) {
+      this.set('vary', config.registryVaryHeader);
+    }
+    return;
+  }
+
+  // if not fond, sync from source registry
+  if (!this.allowSync) {
+    this.status = 404;
+    const error = '[not_exists] version not found: ' + tag;
+    this.jsonp = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  // start sync
+  var logId = yield SyncModuleWorker.sync(name, 'sync-by-install');
+  debug('start sync %s, get log id %s', name, logId);
+
+  this.redirect(config.officialNpmRegistry + this.url);
+};

controllers/registry/package/tag.js

@@ -0,0 +1,56 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:tag');
+var semver = require('semver');
+var util = require('util');
+var packageService = require('../../../services/package');
+
+// PUT /:name/:tag
+// https://github.com/npm/npm-registry-client/blob/master/lib/tag.js#L4
+// this.request("PUT", uri+"/"+tagName, { body : JSON.stringify(version) }, cb)
+module.exports = function* tag() {
+  var version = this.request.body;
+  var name = this.params.name || this.params[0];
+  var tag = this.params.tag || this.params[1];
+  debug('tag %j to %s/%s', version, name, tag);
+
+  if (!version) {
+    this.status = 400;
+    const error = '[version_missed] version not found';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  if (!semver.valid(version)) {
+    this.status = 403;
+    const error = util.format('[forbidden] setting tag %s to invalid version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var mod = yield packageService.getModule(name, version);
+  if (!mod) {
+    this.status = 403;
+    const error = util.format('[forbidden] setting tag %s to unknown version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var row = yield packageService.addModuleTag(name, tag, version);
+  this.status = 201;
+  this.body = {
+    ok: true,
+    modified: row.gmt_modified,
+  };
+};

controllers/registry/package/update.js

@@ -0,0 +1,194 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:package:update');
+var packageService = require('../../../services/package');
+var userService = require('../../../services/user');
+var config = require('../../../config');
+var hook = require('../../../services/hook');
+
+// PUT /:name/-rev/:rev
+//
+// * remove with versions, then will `DELETE /:name/download/:filename/-rev/:rev`
+// * ...
+module.exports = function* update(next) {
+  var name = this.params.name || this.params[0];
+  debug('update module %s, %s, %j', this.url, name, this.request.body);
+
+  var body = this.request.body;
+  if (body.versions) {
+    yield updateVersions.call(this, next);
+  } else if (body.maintainers) {
+    yield updateMaintainers.call(this, next);
+  } else {
+    yield next;
+  }
+};
+
+// update with versions
+// https://github.com/npm/npm-registry-client/blob/master/lib/unpublish.js#L63
+function* updateVersions(next) {
+  var name = this.params.name || this.params[0];
+  // left versions
+  var versions = this.request.body.versions;
+
+  // step1: list all the versions
+  var mods = yield packageService.listModulesByName(name);
+  debug('removeWithVersions module %s, left versions %j, %s mods',
+    name, Object.keys(versions), mods && mods.length);
+  if (!mods || !mods.length) {
+    return yield next;
+  }
+
+  // step3: calculate which versions need to remove and
+  // which versions need to remain
+  var removeVersions = [];
+  var removeVersionMaps = {};
+  var remainVersions = [];
+
+  for (var i = 0; i < mods.length; i++) {
+    var v = mods[i].version;
+    if (!versions[v]) {
+      removeVersions.push(v);
+      removeVersionMaps[v] = true;
+    } else {
+      remainVersions.push(v);
+    }
+  }
+
+  if (!removeVersions.length) {
+    debug('no versions need to remove');
+    this.status = 201;
+    this.body = { ok: true };
+    return;
+  }
+  debug('remove versions: %j, remain versions: %j', removeVersions, remainVersions);
+
+  // step 4: remove all the versions which need to remove
+  // let removeTar do remove versions from module table
+  var tags = yield packageService.listModuleTags(name);
+
+  var removeTags = [];
+  var latestRemoved = false;
+  tags.forEach(function (tag) {
+    // this tag need be removed
+    if (removeVersionMaps[tag.version]) {
+      removeTags.push(tag.id);
+      if (tag.tag === 'latest') {
+        latestRemoved = true;
+      }
+    }
+  });
+
+  debug('remove tags: %j', removeTags);
+  if (removeTags.length) {
+    // step 5: remove all the tags
+    yield packageService.removeModuleTagsByIds(removeTags);
+    if (latestRemoved && remainVersions[0]) {
+      debug('latest tags removed, generate a new latest tag with new version: %s',
+        remainVersions[0]);
+      // step 6: insert new latest tag
+      yield packageService.addModuleTag(name, 'latest', remainVersions[0]);
+    }
+  }
+
+  // step 7: update last modified, make sure etag change
+  yield packageService.updateModuleLastModified(name);
+
+  this.status = 201;
+  this.body = { ok: true };
+}
+
+function* updateMaintainers() {
+  var name = this.params.name || this.params[0];
+  var body = this.request.body;
+  debug('updateMaintainers module %s, %j', name, body);
+
+  var usernames = body.maintainers.map(function (user) {
+    return user.name;
+  });
+
+  if (usernames.length === 0) {
+    this.status = 403;
+    const error = '[invalid_operation] Can not remove all maintainers';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  if (config.customUserService) {
+    // ensure new authors are vaild
+    var maintainers = yield packageService.listMaintainerNamesOnly(name);
+    var map = {};
+    var newNames = [];
+    for (var i = 0; i < maintainers.length; i++) {
+      map[maintainers[i]] = 1;
+    }
+    for (var i = 0; i < usernames.length; i++) {
+      var username = usernames[i];
+      if (map[username] !== 1) {
+        newNames.push(username);
+      }
+    }
+    if (newNames.length > 0) {
+      var users = yield userService.list(newNames);
+      var map = {};
+      for (var i = 0; i < users.length; i++) {
+        var user = users[i];
+        map[user.login] = 1;
+      }
+      var invailds = [];
+      for (var i = 0; i < newNames.length; i++) {
+        var username = newNames[i];
+        if (map[username] !== 1) {
+          invailds.push(username);
+        }
+      }
+      if (invailds.length > 0) {
+        this.status = 403;
+        const error = '[invalid] User: `' + invailds.join(', ') + '` not exists';
+        this.body = {
+          error,
+          reason: error,
+        };
+        return;
+      }
+    }
+  }
+
+  var r = yield packageService.updatePrivateModuleMaintainers(name, usernames);
+  debug('result: %j', r);
+  if (r.add && r.add.length) {
+    const envelope = {
+      event: 'package:owner',
+      name: name,
+      type: 'package',
+      version: null,
+      hookOwner: null,
+      payload: null,
+      change: null,
+    };
+    hook.trigger(envelope);
+  }
+  if (r.remove && r.remove.length) {
+    const envelope = {
+      event: 'package:owner-rm',
+      name: name,
+      type: 'package',
+      version: null,
+      hookOwner: null,
+      payload: null,
+      change: null,
+    };
+    hook.trigger(envelope);
+  }
+
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: name,
+    rev: this.params.rev || this.params[1],
+  };
+}

controllers/registry/token/create.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var ipRegex = require('ip-regex');
+var tokenService = require('../../../services/token');
+var userService = require('../../../services/user');
+var ipv4 = ipRegex.v4({ exact: true });
+
+module.exports = function* createToken() {
+  var readonly = this.request.body.readonly;
+  if (typeof readonly !== 'undefined' && typeof readonly !== 'boolean') {
+    this.status = 400;
+    var error = '[bad_request] readonly ' + readonly + ' is not boolean';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+  var cidrWhitelist = this.request.body.cidr_whitelist;
+  if (typeof cidrWhitelist !== 'undefined') {
+    var isValidateWhiteList = Array.isArray(cidrWhitelist) && cidrWhitelist.every(function (cidr) {
+      return ipv4.test(cidr);
+    });
+    if (!isValidateWhiteList) {
+      this.status = 400;
+      var error = '[bad_request] cide white list ' + JSON.stringify(cidrWhitelist) + ' is not validate ip array';
+      this.body = {
+        error,
+        reason: error,
+      };
+      return;
+    }
+  }
+
+  var password = this.request.body.password;
+  var user = yield userService.auth(this.user.name, password);
+  if (!user) {
+    this.status = 401;
+    var error = '[unauthorized] incorrect or missing password.';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var tokenServiceImpl = this.tokenService || tokenService;
+
+  var token = yield tokenServiceImpl.createToken(this.user.name, {
+    readonly: !!readonly,
+    cidrWhitelist: cidrWhitelist || [],
+  });
+  this.status = 201;
+  this.body = token;
+};

controllers/registry/token/del.js

@@ -0,0 +1,8 @@
+'use strict';
+
+var tokenService = require('../../../services/token');
+
+module.exports = function* deleteToken() {
+  yield tokenService.deleteToken(this.user.name, this.params.UUID);
+  this.status = 204;
+};

controllers/registry/token/list.js

@@ -0,0 +1,60 @@
+'use strict';
+
+var tokenService = require('../../../services/token');
+
+var DEFAULT_PER_PAGE = 10;
+var MIN_PER_PAGE = 1;
+var MAX_PER_PAGE = 9999;
+
+module.exports = function* createToken() {
+  var perPage = typeof this.query.perPage === 'undefined' ? DEFAULT_PER_PAGE : parseInt(this.query.perPage);
+  if (Number.isNaN(perPage)) {
+    this.status = 400;
+    var error = 'perPage ' + this.query.perPage + ' is not a number';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+  if (perPage < MIN_PER_PAGE || perPage > MAX_PER_PAGE) {
+    this.status = 400;
+    var error = 'perPage ' + this.query.perPage + ' is out of boundary';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var page = typeof this.query.page === 'undefined' ? 0 : parseInt(this.query.page);
+  if (Number.isNaN(page)) {
+    this.status = 400;
+    var error = 'page ' + this.query.page + ' is not a number';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+  if (page < 0) {
+    this.status = 400;
+    var error = 'page ' + this.query.page + ' is invalidate';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var tokens = yield tokenService.listToken(this.user.name, {
+    page: page,
+    perPage: perPage,
+  });
+
+  this.status = 200;
+  this.body = {
+    objects: tokens,
+    urls: {},
+  };
+};

controllers/registry/user.js

@@ -1,162 +0,0 @@
-/*!
- * cnpmjs.org - controllers/registry/user.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var debug = require('debug')('cnpmjs.org:controllers:registry');
-var logger = require('../../common/logger');
-var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
-
-exports.show = function (req, res, next) {
-  var name = req.params.name;
-  User.get(name, function (err, row) {
-    if (err) {
-      return next(err);
-    }
-    if (!row) {
-      return next();
-    }
-
-    res.setHeader('etag', '"' + row.rev + '"');
-    var data = {
-      _id: 'org.couchdb.user:' + row.name,
-      _rev: row.rev,
-      name: row.name,
-      email: row.email,
-      type: 'user',
-      roles: [],
-      date: row.gmt_modified,
-    };
-    res.json(data);
-  });
-};
-
-// json:
-//  { name: 'fengmk2',
-//    salt: 'xxxx',
-//    password_sha: 'xxxxxx',
-//    email: 'fengmk2@gmail.com',
-//    _id: 'org.couchdb.user:fengmk2',
-//    type: 'user',
-//    roles: [],
-//    date: '2013-12-04T12:56:13.714Z' } }
-exports.add = function (req, res, next) {
-  var name = req.params.name;
-  var body = req.body || {};
-  var user = {
-    name: body.name,
-    salt: body.salt,
-    password_sha: body.password_sha,
-    email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
-    // roles: body.roles || [],
-  };
-  if (!user.name || !user.salt || !user.password_sha || !user.email) {
-    return res.json(422, {
-      error: 'paramError',
-      reason: 'params missing'
-    });
-  }
-  debug('add user: %j', user);
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  User.get(name, ep.doneLater(function (row) {
-    if (row) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    User.add(user, ep.done('add'));
-  }));
-
-  ep.once('add', function (result) {
-    res.setHeader('etag', '"' + result.rev + '"');
-    // location: 'http://registry.npmjs.org/_users/org.couchdb.user:cnpmjstest1',
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + name,
-      rev: result.rev
-    });
-  });
-};
-
-exports.authSession = function (req, res, next) {
-  // body: {"name":"foo","password":"****"}
-  var body = req.body || {};
-  var name = body.name;
-  var password = body.password;
-  User.auth(name, password, function (err, user) {
-    debug('authSession %s: %j', name, user);
-    if (err) {
-      return next(err);
-    }
-    if (!user) {
-      return res.json(401, {ok: false, name: null, roles: []});
-    }
-
-    req.session.name = user.name;
-    res.json(200, {ok: true, name: user.name, roles: []});
-  });
-};
-
-exports.update = function (req, res, next) {
-  var name = req.params.name;
-  var rev = req.params.rev;
-  if (!name || !rev) {
-    return next();
-  }
-
-  debug('update: %s, rev: %s, session.name: %s', name, rev, req.session.name);
-
-  if (name !== req.session.name) {
-    // must authSession first
-    res.statusCode = 401;
-    return res.json({
-      error: 'unauthorized',
-      reason: 'Name is incorrect.'
-    });
-  }
-
-  var body = req.body || {};
-  var user = {
-    name: body.name,
-    salt: body.salt,
-    password_sha: body.password_sha,
-    email: body.email,
-    ip: req.socket && req.socket.remoteAddress || '0.0.0.0',
-    rev: body.rev || body._rev,
-    // roles: body.roles || [],
-  };
-  User.update(user, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    //check rev error
-    if (!result) {
-      return res.json(409, {
-        error: 'conflict',
-        reason: 'Document update conflict.'
-      });
-    }
-    res.json(201, {
-      ok: true,
-      id: 'org.couchdb.user:' + user.name,
-      rev: result.rev
-    });
-  });
-};

controllers/registry/user/add.js

@@ -0,0 +1,140 @@
+'use strict';
+
+var ensurePasswordSalt = require('./common').ensurePasswordSalt;
+var userService = require('../../../services/user');
+var config = require('../../../config');
+var tokenService = require('../../../services/token');
+
+// npm 1.4.4
+// add new user first
+// @see https://github.com/npm/npm-registry-client/commit/effb4bc88d443f764f2c2e8b4dd583cc72cf6084
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'accept-encoding': 'gzip',
+//   'user-agent': 'node/v0.11.12 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '150',
+//   connection: 'close' } { name: 'mk2',
+//   password: '123456',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:33:19.465Z' }
+
+// old npm flow
+// json:
+// PUT /-/user/org.couchdb.user:mk2 { accept: 'application/json',
+//   'user-agent': 'node/v0.8.26 darwin x64',
+//   host: '127.0.0.1:7001',
+//   'content-type': 'application/json',
+//   'content-length': '258',
+//   connection: 'keep-alive' }
+// { name: 'mk2',
+//   salt: '12351936478446a5466d4fb1633b80f3838b4caaa03649a885ac722cd6',
+//   password_sha: '123408912a6db1d96b132a90856d99db029cef3d',
+//   email: 'fengmk2@gmail.com',
+//   _id: 'org.couchdb.user:mk2',
+//   type: 'user',
+//   roles: [],
+//   date: '2014-03-15T02:39:25.696Z' }
+module.exports = function* addUser() {
+  var name = this.params.name;
+  var body = this.request.body || {};
+
+  if (!body.password || !body.name) {
+    this.status = 422;
+    const error = '[param_error] params missing, name, email or password missing';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var loginedUser;
+  try {
+    loginedUser = yield userService.authAndSave(body.name, body.password);
+  } catch (err) {
+    this.status = err.status || 500;
+    this.body = {
+      error: err.message,
+      reason: err.message,
+    };
+    return;
+  }
+  if (loginedUser) {
+    var token = yield tokenService.createToken(body.name, {
+      readonly: !!body.readonly,
+      cidrWhitelist: body.cidr_whitelist || [],
+    });
+    this.status = 201;
+    this.body = {
+      token: token.token,
+      ok: true,
+      id: 'org.couchdb.user:' + loginedUser.login,
+      rev: Date.now() + '-' + loginedUser.login
+    };
+    return;
+  }
+
+  if (config.customUserService) {
+    // user login fail, not allow to add new user
+    this.status = 401;
+    const error = '[unauthorized] Login fail, please check your login name and password';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var user = {
+    name: body.name,
+    // salt: body.salt,
+    // password_sha: body.password_sha,
+    email: body.email,
+    ip: this.ip || '0.0.0.0',
+    // roles: body.roles || [],
+  };
+
+  ensurePasswordSalt(user, body);
+
+  if (!user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    const error = '[param_error] params missing, name, email or password missing';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var existUser = yield userService.get(name);
+  if (existUser) {
+    this.status = 409;
+    const error = '[conflict] User ' + name + ' already exists';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  // add new user
+  var result = yield userService.add(user);
+  this.etag = '"' + result.rev + '"';
+
+  var token = yield tokenService.createToken(body.name, {
+    readonly: !!body.readonly,
+    cidrWhitelist: body.cidr_whitelist || [],
+  });
+
+  this.status = 201;
+  this.body = {
+    token: token.token,
+    ok: true,
+    id: 'org.couchdb.user:' + name,
+    rev: result.rev
+  };
+};

controllers/registry/user/common.js

@@ -0,0 +1,26 @@
+/**!
+ * cnpmjs.org - controllers/registry/user/common.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var utility = require('utility');
+
+exports.ensurePasswordSalt = function (user, body) {
+  if (!user.password_sha && body.password) {
+    // create password_sha on server
+    user.salt = crypto.randomBytes(30).toString('hex');
+    user.password_sha = utility.sha1(body.password + user.salt);
+  }
+};

controllers/registry/user/ping.js

@@ -0,0 +1,7 @@
+'use strict';
+
+// https://docs.npmjs.com/cli/ping
+module.exports = function* () {
+  this.status = 200;
+  this.body = {};
+};

controllers/registry/user/show.js

@@ -0,0 +1,54 @@
+'use strict';
+
+var userService = require('../../../services/user');
+
+// GET /-/user/org.couchdb.user::name
+module.exports = function* show(next) {
+  var name = this.params.name;
+  var user = yield userService.getAndSave(name);
+  if (!user) {
+    return yield next;
+  }
+
+  var data = user.json;
+  if (!data) {
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
+      type: 'user',
+      roles: [],
+      date: user.gmt_modified,
+    };
+  }
+
+  if (data.login) {
+    // custom user format
+    // convert to npm user format
+    data = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
+      type: 'user',
+      roles: [],
+      date: user.gmt_modified,
+      avatar: data.avatar_url,
+      fullname: data.name || data.login,
+      homepage: data.html_url,
+      scopes: data.scopes,
+      site_admin: data.site_admin
+    };
+  }
+
+  data._cnpm_meta = {
+    id: user.id,
+    npm_user: user.isNpmUser,
+    custom_user: !!data.login,
+    gmt_create: user.gmt_create,
+    gmt_modified: user.gmt_modified,
+  };
+
+  this.body = data;
+};

controllers/registry/user/update.js

@@ -0,0 +1,85 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:registry:user:update');
+var ensurePasswordSalt = require('./common').ensurePasswordSalt;
+var userService = require('../../../services/user');
+
+// logined before update, no need to auth user again
+// { name: 'admin',
+// password: '123123',
+// email: 'fengmk2@gmail.com',
+// _id: 'org.couchdb.user:admin',
+// type: 'user',
+// roles: [],
+// date: '2014-08-05T16:08:22.645Z',
+// _rev: '1-1a18c3d73ba42e863523a399ff3304d8',
+// _cnpm_meta:
+//  { id: 14,
+//    npm_user: false,
+//    custom_user: false,
+//    gmt_create: '2014-08-05T15:46:58.000Z',
+//    gmt_modified: '2014-08-05T15:46:58.000Z',
+//    admin: true,
+//    scopes: [ '@cnpm', '@cnpmtest' ] } }
+module.exports = function* updateUser(next) {
+  var name = this.params.name;
+  var rev = this.params.rev;
+  if (!name || !rev) {
+    return yield next;
+  }
+  debug('update: %s, rev: %s, user.name: %s', name, rev, this.user.name);
+
+  if (name !== this.user.name) {
+    // must auth user first
+    this.status = 401;
+    const error = '[unauthorized] Name is incorrect';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var body = this.request.body || {};
+  var user = {
+    name: body.name,
+    // salt: body.salt,
+    // password_sha: body.password_sha,
+    email: body.email,
+    ip: this.ip || '0.0.0.0',
+    rev: body.rev || body._rev,
+    // roles: body.roles || [],
+  };
+
+  debug('update user %j', body);
+
+  ensurePasswordSalt(user, body);
+
+  if (!body.password || !user.name || !user.salt || !user.password_sha || !user.email) {
+    this.status = 422;
+    const error = '[param_error] params missing, name, email or password missing';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var result = yield userService.update(user);
+  if (!result) {
+    this.status = 409;
+    const error = '[conflict] Document update conflict';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    id: 'org.couchdb.user:' + user.name,
+    rev: result.rev
+  };
+};

controllers/registry/user/whoami.js

@@ -0,0 +1,9 @@
+'use strict';
+
+// https://docs.npmjs.com/cli/whoami
+module.exports = function* () {
+  this.status = 200;
+  this.body = {
+    username: this.user.name,
+  };
+};

controllers/registry/user_package.js

@@ -0,0 +1,52 @@
+/**!
+ * cnpmjs.org - controllers/registry/user_package.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var packageService = require('../../services/package');
+
+// GET /-/by-user/:user
+exports.list = function* () {
+  var users = this.params.user.split('|');
+  if (users.length > 20) {
+    this.status = 400;
+    const error = '[bad_request] reach max user names limit, must <= 20 user names';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var firstUser = users[0];
+  if (!firstUser) {
+    // params.user = '|'
+    this.body = {};
+    return;
+  }
+
+  var tasks = {};
+  for (var i = 0; i < users.length; i++) {
+    var username = users[i];
+    tasks[username] = packageService.listPublicModuleNamesByUser(username);
+  }
+
+  var data = yield tasks;
+  for (var k in data) {
+    if (data[k].length === 0) {
+      data[k] = undefined;
+    }
+  }
+  this.body = data;
+};

controllers/sync.js

@@ -1,50 +1,129 @@
-/**!
- * cnpmjs.org - controllers/download.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-var Log = require('../proxy/module_log');
-var SyncModuleWorker = require('../proxy/sync_module_worker');
+var debug = require('debug')('cnpmjs.org:controllers:sync');
+var Log = require('../services/module_log');
+var npmService = require('../services/npm');
+var SyncModuleWorker = require('./sync_module_worker');
+var config = require('../config');
 
-exports.sync = function (req, res, next) {
-  var username = req.session.name || 'anonymous';
-  var name = req.params.name;
-  SyncModuleWorker.sync(name, username, function (err, result) {
-    if (err) {
-      return next(err);
-    }
-    if (!result.ok) {
-      return res.json(result.statusCode, result.pkg);
-    }
-    res.json(201, {
-      ok: true,
-      logId: result.logId
-    });
-  });
+exports.sync = function* () {
+  var username = this.user.name || 'anonymous';
+  var name = this.params.name || this.params[0];
+  var type = 'package';
+  if (name.indexOf(':') > 0) {
+    // user:fengmk2
+    // package:pedding
+    var splits = name.split(':');
+    type = splits[0];
+    name = splits[1];
+  }
+  var publish = this.query.publish === 'true';
+  var noDep = this.query.nodeps === 'true';
+  var syncUpstreamFirst = this.query.sync_upstream === 'true';
+  var syncFromBackupFile = this.query.sync_from_backup === 'true';
+  if (!config.enableWebDataRemoteRegistry && !config.sourceNpmRegistryIsCNpm) {
+    syncUpstreamFirst = false;
+  }
+  debug('sync %s with query: %j, syncUpstreamFirst: %s', name, this.query, syncUpstreamFirst);
+  if (type === 'package' && publish && !this.user.isAdmin) {
+    this.status = 403;
+    const error = '[no_perms] Only admin can publish';
+    this.body = {
+      error,
+      reason: error,
+    };
+    return;
+  }
+
+  var options = {
+    type: type,
+    publish: publish,
+    noDep: noDep,
+    syncUpstreamFirst: syncUpstreamFirst,
+    syncFromBackupFile: syncFromBackupFile,
+  };
+
+  var logId = yield SyncModuleWorker.sync(name, username, options);
+  debug('sync %s got log id %j', name, logId);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logId: logId
+  };
 };
 
-exports.getSyncLog = function (req, res, next) {
-  var logId = req.params.id;
-  var name = req.params.name;
-  var offset = Number(req.query.offset) || 0;
-  Log.get(logId, function (err, row) {
-    if (err || !row) {
-      return next(err);
-    }
-    var log = row.log.trim();
-    if (offset > 0) {
-      log = log.split('\n').slice(offset).join('\n');
-    }
-    res.json(200, {ok: true, log: log});
-  });
+exports.scopeSync = function* () {
+  var scope = this.params.scope;
+
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    this.status = 404;
+    this.body = {
+      error: 'no_scope',
+      reason: 'only has syncScopeConfig config can use this feature'
+    };
+    return;
+  }
+
+  var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
+  var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
+  var packages = yield npmService.getScopePackagesShort(scope, scopeCnpmWeb)
+
+  debug('scopeSync %s with query: %j', scope, this.query);
+
+  var packageSyncWorkers = []
+
+  for (let i = 0; i < packages.length; i++) {
+    packageSyncWorkers.push(function* () {
+      var name = packages[i]
+      var logId = yield SyncModuleWorker.sync(name, 'admin', {
+        type: 'package',
+        publish: true,
+        noDep: true,
+        syncUpstreamFirst: false,
+        syncPrivatePackage: { [scope]: scopeCnpmRegistry }
+      })
+      return { name: name, logId: logId }
+    })
+  }
+
+  var logIds = yield packageSyncWorkers
+
+  debug('scopeSync %s got log id %j', scope, logIds);
+
+  this.status = 201;
+  this.body = {
+    ok: true,
+    logIds: logIds
+  };
+};
+
+exports.getSyncLog = function* (next) {
+  var logId = Number(this.params.id || this.params[1]);
+  var offset = Number(this.query.offset) || 0;
+
+  if (!logId) { // NaN
+    this.status = 404;
+    return;
+  }
+  var row = yield Log.get(logId);
+  if (!row) {
+    return yield next;
+  }
+
+  var log = row.log.trim();
+  var syncDone = row.log.indexOf('[done] Sync') >= 0;
+  if (offset > 0) {
+    log = log.split('\n').slice(offset).join('\n');
+    if (!log && syncDone) {
+      // append the last 1k string
+      // the cnpm client sync need the `[done] Sync {name}` string to detect when sync task finished
+      log = '... ignore long logs ...\n' + row.log.substring(row.log.length - 1024);
+    }
+  }
+  this.body = { ok: true, syncDone: syncDone, log: log };
 };

controllers/total.js

@@ -1,46 +1,60 @@
-/**!
- * cnpmjs.org - controllers/total.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
+const Total = require('../services/total');
+const version = require('../package.json').version;
+const config = require('../config');
+const getDownloadTotal = require('./utils').getDownloadTotal;
+const cacheClient = require('../common/cache');
+const logger = require('../common/logger');
 
-var microtime = require('microtime');
-var eventproxy = require('eventproxy');
-var Total = require('../proxy/total');
-var down = require('./download');
-var version = require('../package.json').version;
-var config = require('../config');
+const startTime = '' + Date.now();
+let cache = null;
 
-var startTime = '' + microtime.now();
+module.exports = function* showTotal() {
+  if (cache && Date.now() - cache.cache_time < 120000) {
+    // cache 120 seconds
+    this.body = cache;
+    return;
+  }
 
-exports.show = function (req, res, next) {
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  Total.get(ep.done('total'));
-  down.total(null, ep.done('download'));
-  ep.all('total', 'download', function (total, download) {
-    total.download = download;
-    total.db_name = 'registry';
-    total.instance_start_time = startTime;
-    total.node_version = process.version;
-    total.app_version = version;
-    total.donate = 'https://me.alipay.com/imk2';
-    total.sync_model = config.syncModel;
-    if (req.query.callback) {
-      return res.jsonp(total, req.query.callback);
+  const cacheKey = 'registry_total';
+  if (cacheClient) {
+    const result = yield cacheClient.get(cacheKey);
+    if (result) {
+      this.body = JSON.parse(result);
+      return;
     }
-    res.json(total);
-  });
+  }
+
+  if (cache) {
+    // set cache_time fisrt, avoid query in next time
+    cache.cache_time = Date.now();
+  }
+
+  const r = yield [ Total.get(), getDownloadTotal() ];
+  const total = r[0];
+  const download = r[1];
+
+  total.download = download;
+  total.db_name = 'registry';
+  total.instance_start_time = startTime;
+  total.node_version = process.version;
+  total.app_version = version;
+  total.donate = 'https://www.gittip.com/fengmk2';
+  total.sync_model = config.syncModel;
+
+  cache = total;
+  cache.cache_time = Date.now();
+
+  this.body = total;
+  if (cacheClient) {
+    cacheClient.pipeline()
+      .set(cacheKey, JSON.stringify(total))
+      // cache 12h
+      .expire(cacheKey, 3600 * 12)
+      .exec()
+      .catch(err => {
+        logger.error(err);
+      });
+  }
 };

controllers/utils.js

@@ -0,0 +1,168 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:utils');
+var path = require('path');
+var fs = require('fs');
+var utility = require('utility');
+var ms = require('humanize-ms');
+var moment = require('moment');
+var rimraf = require('rimraf');
+var downloadTotalService = require('../services/download_total');
+var nfs = require('../common/nfs');
+var config = require('../config');
+
+var DOWNLOAD_TIMEOUT = ms('10m');
+
+exports.downloadAsReadStream = function* (key) {
+  var options = { timeout: DOWNLOAD_TIMEOUT };
+  if (nfs.createDownloadStream) {
+    return yield nfs.createDownloadStream(key, options);
+  }
+
+  var tmpPath = path.join(config.uploadDir,
+    utility.randomString() + key.replace(/\//g, '-'));
+  var tarball;
+  function cleanup() {
+    debug('cleanup %s', tmpPath);
+    rimraf(tmpPath, utility.noop);
+    if (tarball) {
+      tarball.destroy();
+    }
+  }
+  debug('downloadAsReadStream() %s to %s', key, tmpPath);
+  try {
+    yield nfs.download(key, tmpPath, options);
+  } catch (err) {
+    debug('downloadAsReadStream() %s to %s error: %s', key, tmpPath, err.stack);
+    cleanup();
+    throw err;
+  }
+  tarball = fs.createReadStream(tmpPath);
+  tarball.once('error', cleanup);
+  tarball.once('end', cleanup);
+  return tarball;
+};
+
+exports.getDownloadTotal = function* (name) {
+  var end = moment();
+  var start = end.clone().subtract(1, 'months').startOf('month');
+  var lastday = end.clone().subtract(1, 'days').format('YYYY-MM-DD');
+  var lastweekStart = end.clone().subtract(1, 'weeks').startOf('isoweek');
+  var lastweekEnd = lastweekStart.clone().endOf('isoweek').format('YYYY-MM-DD');
+  var lastmonthEnd = start.clone().endOf('month').format('YYYY-MM-DD');
+  var thismonthStart = end.clone().startOf('month').format('YYYY-MM-DD');
+  var thisweekStart = end.clone().startOf('isoweek').format('YYYY-MM-DD');
+  start = start.format('YYYY-MM-DD');
+  end = end.format('YYYY-MM-DD');
+  lastweekStart = lastweekStart.format('YYYY-MM-DD');
+  var method = name ? 'getModuleTotal' : 'getTotal';
+  var args = [start, end];
+  if (name) {
+    args.unshift(name);
+  }
+
+  var rows = yield downloadTotalService[method].apply(downloadTotalService, args);
+  var download = {
+    today: 0,
+    thisweek: 0,
+    thismonth: 0,
+    lastday: 0,
+    lastweek: 0,
+    lastmonth: 0,
+    total: 0,
+  };
+
+  for (var i = 0; i < rows.length; i++) {
+    var r = rows[i];
+    if (r.date === end) {
+      download.today += r.count;
+    }
+    if (r.date >= thismonthStart) {
+      download.thismonth += r.count;
+    }
+    if (r.date >= thisweekStart) {
+      download.thisweek += r.count;
+    }
+
+    if (r.date === lastday) {
+      download.lastday += r.count;
+    }
+    if (r.date >= lastweekStart && r.date <= lastweekEnd) {
+      download.lastweek += r.count;
+    }
+    if (r.date >= start && r.date <= lastmonthEnd) {
+      download.lastmonth += r.count;
+    }
+  }
+  if (name) {
+    download.total = yield downloadTotalService.getTotalByName(name);
+  }
+  return download;
+};
+
+exports.setLicense = function (pkg) {
+  var license;
+  license = pkg.license || pkg.licenses || pkg.licence || pkg.licences;
+  if (!license) {
+    return ;
+  }
+
+  if (Array.isArray(license)) {
+    license = license[0];
+  }
+
+  if (typeof license === 'object') {
+    pkg.license = {
+      name: license.name || license.type,
+      url: license.url
+    };
+  }
+
+  if (typeof license === 'string') {
+    if (license.match(/(http|https)(:\/\/)/ig)) {
+      pkg.license = {
+        name: license,
+        url: license
+      };
+    } else {
+      pkg.license = {
+        url: exports.getOssLicenseUrlFromName(license),
+        name: license
+      };
+    }
+  }
+};
+
+exports.getOssLicenseUrlFromName = function (name) {
+  var base = 'http://opensource.org/licenses/';
+
+  var licenseMap = {
+    'bsd': 'BSD-2-Clause',
+    'mit': 'MIT',
+    'x11': 'MIT',
+    'mit/x11': 'MIT',
+    'apache 2.0': 'Apache-2.0',
+    'apache2': 'Apache-2.0',
+    'apache 2': 'Apache-2.0',
+    'apache-2': 'Apache-2.0',
+    'apache': 'Apache-2.0',
+    'gpl': 'GPL-3.0',
+    'gplv3': 'GPL-3.0',
+    'gplv2': 'GPL-2.0',
+    'gpl3': 'GPL-3.0',
+    'gpl2': 'GPL-2.0',
+    'lgpl': 'LGPL-2.1',
+    'lgplv2.1': 'LGPL-2.1',
+    'lgplv2': 'LGPL-2.1'
+  };
+
+  return licenseMap[name.toLowerCase()] ?
+    base + licenseMap[name.toLowerCase()] : base + name;
+};
+
+exports.ensureSinceIsDate = function(since) {
+  if (!(since instanceof Date)) {
+    return new Date(Number(since));
+  }
+  return since;
+}

controllers/web/badge.js

@@ -0,0 +1,51 @@
+'use strict';
+
+var config = require('../../config');
+var packageService = require('../../services/package');
+var DownloadTotal = require('../../services/download_total');
+
+exports.version = function* () {
+  var color = 'grey';
+  var name = this.params[0];
+  var tag = this.query.tag || 'latest';
+  var version = this.query.version;
+  let info;
+  if (version) {
+    info = yield packageService.getModule(name, version);
+  } else {
+    info = yield packageService.getModuleByTag(name, tag);
+  }
+  if (info) {
+    version = info.version;
+    if (/^0\.0\./.test(version)) {
+      // <0.1.0 & >=0.0.0
+      color = 'red';
+    } else if (/^0\./.test(version)) {
+      // <1.0.0 & >=0.1.0
+      color = 'green';
+    } else {
+      // >=1.0.0
+      color = 'blue';
+    }
+  }
+
+  var subject = config.badgeSubject;
+  if (this.query.subject) {
+    subject = this.query.subject;
+  }
+  if (!version) {
+    version = 'invalid';
+  }
+  var style = this.query.style || 'flat-square';
+  var url = config.badgeService.url(subject, version, { color, style });
+  this.redirect(url);
+};
+
+exports.downloads = function* () {
+  // https://dn-img-shields-io.qbox.me/badge/downloads-100k/month-brightgreen.svg?style=flat-square
+  var name = this.params[0];
+  var count = yield DownloadTotal.getTotalByName(name);
+  var style = this.query.style;
+  var url = config.badgeService.url('downloads', count, { style });
+  this.redirect(url);
+};

controllers/web/package.js

@@ -1,162 +0,0 @@
-/*!
- * cnpmjs.org - controllers/web/package.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-
-var moment = require('moment');
-var eventproxy = require('eventproxy');
-var semver = require('semver');
-var marked = require('marked');
-var gravatar = require('gravatar');
-var humanize = require('humanize-number');
-var Module = require('../../proxy/module');
-var down = require('../download');
-var sync = require('../sync');
-var Log = require('../../proxy/module_log');
-
-exports.display = function (req, res, next) {
-  var params = req.params;
-  var name = params.name;
-  var tag = params.version;
-  var ep = eventproxy.create();
-  ep.fail(next);
-
-  if (tag) {
-    var version = semver.valid(tag);
-    if (version) {
-      Module.get(name, version, ep.done('pkg'));
-    } else {
-      Module.getByTag(name, tag, ep.done('pkg'));
-    }
-  } else {
-    Module.getByTag(name, 'latest', ep.done('pkg'));
-  }
-
-  down.total(name, ep.done('download'));
-
-  ep.all('pkg', 'download', function (pkg, download) {
-    if (!pkg || !pkg.package) {
-      return next();
-    }
-
-    pkg.package.fromNow = moment(pkg.publish_time).fromNow();
-    pkg = pkg.package;
-    pkg.readme = marked(pkg.readme || '');
-    if (pkg.maintainers) {
-      for (var i = 0; i < pkg.maintainers.length; i++) {
-        var maintainer = pkg.maintainers[i];
-        if (maintainer.email) {
-          maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
-        }
-      }
-    }
-
-    setLicense(pkg);
-
-    for (var k in download) {
-      download[k] = humanize(download[k]);
-    }
-
-    res.render('package', {
-      title: 'Package - ' + pkg.name,
-      package: pkg,
-      download: download
-    });
-  });
-};
-
-exports.search = function (req, res, next) {
-  var params = req.params;
-  var word = req.params.word;
-  Module.search(word, function (err, packages) {
-    if (err) {
-      return next(err);
-    }
-
-    res.render('search', {
-      title: 'Keyword - ' + word,
-      keyword: word,
-      packages: packages || []
-    });
-  });
-};
-
-exports.displaySync = function (req, res, next) {
-  var name = req.params.name;
-  res.render('sync', {
-    name: name,
-    title: 'Sync - ' + name
-  });
-};
-
-function setLicense(pkg) {
-  var license;
-  license = pkg.license || pkg.licenses || pkg.licence || pkg.licences;
-  if (!license) {
-    return ;
-  }
-
-  if (Array.isArray(license)) {
-    license = license[0];
-  }
-
-  if (typeof license === 'object') {
-    pkg.license = {
-      name: license.name || license.type,
-      url: license.url
-    };
-  }
-
-  if (typeof license === 'string') {
-    if (license.match(/(http|https)(:\/\/)/ig)) {
-      pkg.license = {
-        name: license,
-        url: license
-      };
-    } else {
-      pkg.license = {
-        url: getOssLicenseUrlFromName(license),
-        name: license
-      };
-    }
-  }
-}
-exports.setLicense = setLicense;
-
-function getOssLicenseUrlFromName(name) {
-  var base = 'http://opensource.org/licenses/';
-
-  var licenseMap = {
-    'bsd': 'BSD-2-Clause',
-    'mit': 'MIT',
-    'x11': 'MIT',
-    'mit/x11': 'MIT',
-    'apache 2.0': 'Apache-2.0',
-    'apache2': 'Apache-2.0',
-    'apache 2': 'Apache-2.0',
-    'apache-2': 'Apache-2.0',
-    'apache': 'Apache-2.0',
-    'gpl': 'GPL-3.0',
-    'gplv3': 'GPL-3.0',
-    'gplv2': 'GPL-2.0',
-    'gpl3': 'GPL-3.0',
-    'gpl2': 'GPL-2.0',
-    'lgpl': 'LGPL-2.1',
-    'lgplv2.1': 'LGPL-2.1',
-    'lgplv2': 'LGPL-2.1'
-  };
-
-  return licenseMap[name.toLowerCase()] ?
-    base + licenseMap[name.toLowerCase()] : base + name;
-}

controllers/web/package/list_privates.js

@@ -0,0 +1,22 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+var config = require('../../../config');
+
+module.exports = function* listPrivates() {
+  var tasks = {};
+  for (var i = 0; i < config.scopes.length; i++) {
+    var scope = config.scopes[i];
+    tasks[scope] = packageService.listPrivateModulesByScope(scope);
+  }
+
+  if (config.privatePackages && config.privatePackages.length > 0) {
+    tasks['no scoped'] = packageService.listModules(config.privatePackages);
+  }
+
+  var scopes = yield tasks;
+  yield this.render('private', {
+    title: 'private packages',
+    scopes: scopes
+  });
+};

controllers/web/package/search.js

@@ -0,0 +1,51 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:web:package:search');
+var packageService = require('../../../services/package');
+var config = require('../../../config');
+
+module.exports = function* search() {
+  var params = this.params;
+  var word = params.word || params[0];
+  var limit = Number(this.query.limit) || 100;
+
+  if (limit > 10000) {
+    limit = 10000;
+  }
+
+  if (config.disableSearch) {
+    return this.redirect(`/package/${encodeURIComponent(word)}`);
+  }
+
+  debug('search %j', word);
+  var result = yield packageService.search(word, {
+    limit: limit
+  });
+
+  var match = null;
+  for (var i = 0; i < result.searchMatchs.length; i++) {
+    var p = result.searchMatchs[i];
+    if (p.name === word) {
+      match = p;
+      break;
+    }
+  }
+
+  // return a json result
+  if (this.query && this.query.type === 'json') {
+    this.jsonp = {
+      keyword: word,
+      match: match,
+      packages: result.searchMatchs,
+      keywords: result.keywordMatchs,
+    };
+    return;
+  }
+  yield this.render('search', {
+    title: 'Keyword - ' + word,
+    keyword: word,
+    match: match,
+    packages: result.searchMatchs,
+    keywords: result.keywordMatchs,
+  });
+};

controllers/web/package/search_range.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var packageService = require('../../../services/package');
+
+module.exports = function* searchRange() {
+  var startKey = this.query.startkey || '';
+  if (startKey[0] === '"') {
+    startKey = startKey.substring(1);
+  }
+  if (startKey[startKey.length - 1] === '"') {
+    startKey = startKey.substring(0, startKey.length - 1);
+  }
+  var limit = Number(this.query.limit) || 20;
+  var result = yield packageService.search(startKey, {limit: limit});
+
+  var packages = result.searchMatchs.concat(result.keywordMatchs);
+
+  var rows = [];
+  for (var i = 0; i < packages.length; i++) {
+    var p = packages[i];
+    var row = {
+      key: p.name,
+      count: 1,
+      value: {
+        name: p.name,
+        description: p.description,
+      }
+    };
+    rows.push(row);
+  }
+  this.body = {
+    rows: rows
+  };
+};

controllers/web/package/show.js

@@ -0,0 +1,258 @@
+'use strict';
+
+var debug = require('debug')('cnpmjs.org:controllers:web:package:show');
+var bytes = require('bytes');
+var giturl = require('giturl');
+var moment = require('moment');
+var semver = require('semver');
+var gravatar = require('gravatar');
+var humanize = require('humanize-number');
+var config = require('../../../config');
+var utils = require('../../utils');
+var setDownloadURL = require('../../../lib/common').setDownloadURL;
+var renderMarkdown = require('../../../common/markdown').render;
+var packageService = require('../../../services/package');
+var blocklistService = require('../../../services/blocklist');
+var downloadTotalService = require('../../../services/download_total');
+var showWithRemote = require('./showWithRemote');
+
+module.exports = function* show(next) {
+  if (config.enableWebDataRemoteRegistry) {
+    return yield showWithRemote(this, next);
+  }
+  var params = this.params;
+  // normal: {name: $name, version: $version}
+  // scope: [$name, $version]
+  var orginalName = params.name || params[0];
+  var name = orginalName;
+  var tag = params.version || params[1];
+  debug('display %s with %j', name, params);
+
+  var getPackageMethod;
+  var getPackageArgs;
+  var version = semver.valid(tag || '');
+  if (version) {
+    getPackageMethod = 'getModule';
+    getPackageArgs = [name, version];
+  } else {
+    getPackageMethod = 'getModuleByTag';
+    getPackageArgs = [name, tag || 'latest'];
+  }
+
+  var pkg = yield packageService[getPackageMethod].apply(packageService, getPackageArgs);
+  if ((!pkg || !pkg.package) && tag) {
+    // + if we can't find it by tag, it may be a non-semver version, check it then
+    // + if we can't find it by version, it may be a tag, check it then
+    getPackageMethod = version ? 'getModuleByTag' : 'getModule';
+    pkg = yield packageService[getPackageMethod](name, tag);
+  }
+
+  // if it's still not found
+  if (!pkg || !pkg.package) {
+    // check if unpublished
+    var unpublishedInfo = yield packageService.getUnpublishedModule(name);
+    debug('show unpublished %j', unpublishedInfo);
+    if (unpublishedInfo) {
+      var data = {
+        name: name,
+        unpublished: unpublishedInfo.package
+      };
+      data.unpublished.time = new Date(data.unpublished.time);
+      if (data.unpublished.maintainers) {
+        for (var i = 0; i < data.unpublished.maintainers.length; i++) {
+          var maintainer = data.unpublished.maintainers[i];
+          if (maintainer.email) {
+            maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
+          }
+        }
+      }
+      yield this.render('package_unpublished', {
+        package: data,
+        title: 'Package - ' + name
+      });
+      return;
+    }
+
+    return yield next;
+  }
+
+  var blocks = yield blocklistService.findBlockPackageVersions(name);
+  if (blocks) {
+    var block = blocks['*'] || blocks[pkg.version];
+    if (block) {
+      this.status = 451;
+      this.body = `[block] package@${pkg.version} was blocked, reason: ${block.reason}`;
+      return;
+    }
+  }
+
+  var r = yield [
+    utils.getDownloadTotal(name),
+    packageService.listDependents(name),
+    packageService.listStarUserNames(name),
+    packageService.listMaintainers(name),
+    packageService.listModulesByName(name),
+    packageService.listModuleTags(name),
+    downloadTotalService.getTotalByName(name),
+  ];
+  var download = r[0];
+  var dependents = r[1];
+  var users = r[2];
+  var maintainers = r[3];
+  var rows = r[4];
+  var tags = r[5];
+  var downloadTotal = r[6];
+
+  const versionsMap = {};
+  const versions = [];
+  for (const row of rows) {
+    var versionPkg = row.package;
+    // pkg is string ... ignore it
+    if (typeof versionPkg === 'string') {
+      continue;
+    }
+    versionPkg.fromNow = moment(versionPkg.publish_time || row.publish_time).fromNow();
+    versions.push(versionPkg);
+    versionsMap[versionPkg.version] = versionPkg;
+  }
+
+  for (const row of tags) {
+    row.fromNow = versionsMap[row.version] && versionsMap[row.version].fromNow;
+  }
+
+  pkg.package.fromNow = moment(pkg.publish_time).fromNow();
+  pkg = pkg.package;
+  pkg.users = users;
+  pkg.versions = versions;
+  pkg.tags = tags;
+
+  if (!pkg.readme && config.enableAbbreviatedMetadata) {
+    var packageReadme = yield packageService.getPackageReadme(name);
+    if (packageReadme) {
+      pkg.readme = packageReadme.readme;
+    }
+  }
+  if (pkg.readme && typeof pkg.readme !== 'string') {
+    pkg.readme = 'readme is not string: ' + JSON.stringify(pkg.readme);
+  } else {
+    pkg.readme = renderMarkdown(pkg.readme || '');
+  }
+
+  if (!pkg.readme) {
+    pkg.readme = pkg.description || '';
+  }
+
+  if (maintainers.length > 0) {
+    pkg.maintainers = maintainers;
+  }
+
+  if (pkg.maintainers) {
+    for (var i = 0; i < pkg.maintainers.length; i++) {
+      var maintainer = pkg.maintainers[i];
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
+      }
+    }
+  }
+
+  if (pkg._npmUser) {
+    pkg.lastPublishedUser = pkg._npmUser;
+    if (pkg.lastPublishedUser.email) {
+      pkg.lastPublishedUser.gravatar = gravatar.url(pkg.lastPublishedUser.email, {s: '50', d: 'retro'}, true);
+    }
+  }
+
+  if (pkg.repository === 'undefined') {
+    pkg.repository = null;
+  }
+  if (pkg.repository && pkg.repository.url) {
+    if (!pkg.repository.weburl) {
+      pkg.repository.weburl = /^https?:\/\//.test(pkg.repository.url) ? pkg.repository.url : (giturl.parse(pkg.repository.url) || pkg.repository.url);
+    }
+  }
+  if (!pkg.bugs) {
+    pkg.bugs = {};
+  }
+
+  utils.setLicense(pkg);
+
+  for (var k in download) {
+    download[k] = humanize(download[k]);
+  }
+  setDownloadURL(pkg, this, config.registryHost);
+
+  pkg.dependents = dependents;
+
+  if (pkg.dist) {
+    pkg.dist.size = bytes(pkg.dist.size || 0);
+  }
+
+  if (pkg.name !== orginalName) {
+    pkg.name = orginalName;
+  }
+
+  pkg.registryUrl = '//' + config.registryHost + '/' + pkg.name;
+  pkg.registryPackageUrl = '//' + config.registryHost + '/' + pkg.name + '/' + pkg.version;
+
+  // pkg.engines = {
+  //   "python": ">= 0.11.9",
+  //   "node": ">= 0.11.9",
+  //   "node1": ">= 0.8.9",
+  //   "node2": ">= 0.10.9",
+  //   "node3": ">= 0.6.9",
+  // };
+  // "engines": "0.10.24",
+  // invalid engines
+  if (pkg.engines && typeof pkg.engines !== 'object') {
+    pkg.engines = {};
+  }
+
+  for (var k in pkg.engines) {
+    var engine = String(pkg.engines[k] || '').trim();
+    var color = 'blue';
+    if (k.indexOf('node') === 0 || k.indexOf('install-') === 0) {
+      color = 'green';
+      var version = /(\d+\.)/.exec(engine);
+      if (version) {
+        version = version[0];
+        if (/^[0123]\./.test(version)) {
+          color = 'red';
+        }
+      }
+      if (engine === '*') {
+        color = 'red';
+      }
+    }
+    pkg.engines[k] = {
+      version: engine,
+      title: k + ': ' + engine,
+      badgeURL: config.badgeService.url(k, engine, { color }),
+    };
+  }
+
+  let packagephobiaSupport = downloadTotal >= config.packagephobiaMinDownloadCount;
+  if (pkg._publish_on_cnpm) {
+    pkg.isPrivate = true;
+    // need download total >= 1000
+    packagephobiaSupport = downloadTotal >= config.packagephobiaMinDownloadCount && config.packagephobiaSupportPrivatePackage;
+  } else {
+    pkg.isPrivate = false;
+    // add security check badge
+    pkg.snyk = {
+      badge: `${config.snykUrl}/test/npm/${pkg.name}/badge.svg`,
+      url: `${config.snykUrl}/test/npm/${pkg.name}`,
+    };
+  }
+  if (packagephobiaSupport) {
+    pkg.packagephobia = {
+      badge: `${config.packagephobiaURL}/badge?p=${pkg.name}@${pkg.version}`,
+      url: `${config.packagephobiaURL}/result?p=${pkg.name}@${pkg.version}`,
+    };
+  }
+
+  yield this.render('package', {
+    title: 'Package - ' + pkg.name,
+    package: pkg,
+    download: download,
+  });
+};

controllers/web/package/showWithRemote.js

@@ -0,0 +1,97 @@
+'use strict';
+
+const debug = require('debug')('cnpmjs.org:controllers:web:package:showWithRemote');
+const moment = require('moment');
+const gravatar = require('gravatar');
+const giturl = require('giturl');
+const urllib = require('../../../common/urllib');
+const config = require('../../../config');
+const renderMarkdown = require('../../../common/markdown').render;
+
+module.exports = function* showWithRemote(ctx, next) {
+  const params = ctx.params;
+  const fullname = params.name || params[0];
+  const versionOrTag = params.version || params[1] || 'latest';
+  debug('display %s with %j', fullname, params);
+
+  const url = `${config.webDataRemoteRegistry}/${fullname}`;
+  const result = yield urllib.request(url, {
+    dataType: 'json',
+    timeout: 20000,
+    followRedirect: true,
+    gzip: true,
+  });
+  if (result.status !== 200) {
+    return yield next;
+  }
+  const manifest = result.data;
+
+  const distTags = manifest['dist-tags'] || {};
+  const realVersion = distTags[versionOrTag] || versionOrTag;
+  const versionsMap = manifest.versions || {};
+  const pkg = versionsMap[realVersion];
+  if (!pkg) {
+    return yield next;
+  }
+
+  const maintainers = manifest.maintainers;
+  if (maintainers) {
+    for (const maintainer of maintainers) {
+      if (maintainer.email) {
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
+      }
+    }
+    pkg.maintainers = maintainers;
+  }
+
+  const timeMap = manifest.time || {};
+  
+  pkg.readme = manifest.readme || '';
+  if (typeof pkg.readme !== 'string') {
+    pkg.readme = 'readme is not string: ' + JSON.stringify(pkg.readme);
+  } else {
+    pkg.readme = renderMarkdown(pkg.readme);
+  }
+  
+  pkg.fromNow = moment(new Date(timeMap[pkg.version])).fromNow();
+  // [ {tag, version, fromNow} ]
+  const tags = [];
+  for (const tag in distTags) {
+    const version = distTags[tag];
+    const time = timeMap[version];
+    if (time) {
+      const fromNow = moment(new Date(time)).fromNow();
+      tags.push({ tag, version, fromNow });
+    }
+  }
+  pkg.tags = tags;
+  // [ {version, deprecated, fromNow} ]
+  const versions = [];
+  for (const version in versionsMap) {
+    const item = versionsMap[version];
+    versions.push({
+      version,
+      deprecated: item.deprecated,
+      fromNow: moment(new Date(timeMap[version])).fromNow(),
+    });
+  }
+  pkg.versions = versions;
+
+  pkg.registryUrl = '//' + config.registryHost + '/' + pkg.name;
+  pkg.registryPackageUrl = '//' + config.registryHost + '/' + pkg.name + '/' + pkg.version;
+
+  if (pkg.repository === 'undefined') {
+    pkg.repository = null;
+  }
+  if (pkg.repository && pkg.repository.url) {
+    if (!pkg.repository.weburl) {
+      pkg.repository.weburl = /^https?:\/\//.test(pkg.repository.url) ? pkg.repository.url : (giturl.parse(pkg.repository.url) || pkg.repository.url);
+    }
+  }
+
+  yield ctx.render('package', {
+    title: 'Package - ' + manifest.name,
+    package: pkg,
+    download: null,
+  });
+};

controllers/web/show_scope_sync.js

@@ -0,0 +1,23 @@
+'use strict';
+var config = require('../../config');
+var npmService = require('../../services/npm');
+
+module.exports = function* showScopeSync () {
+  var scope = this.params.scope;
+  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
+    return item.scope === scope
+  })
+
+  if (!scopeConfig) {
+    return this.redirect('/');
+  }
+
+  var packages = yield npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
+
+  yield this.render('scope_sync', {
+    packages: packages,
+    scope: scopeConfig.scope,
+    sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
+    title: 'Sync Scope Packages',
+  });
+};

controllers/web/show_sync.js

@@ -0,0 +1,23 @@
+'use strict';
+
+var config = require('../../config');
+
+module.exports = function* showSync() {
+  var name = this.params.name || this.params[0] || this.query.name;
+  if (!name) {
+    return this.redirect('/');
+  }
+  var type = 'package';
+  if (name.indexOf(':') > 0) {
+    var splits = name.split(':');
+    name = splits[1];
+    type = splits[0];
+  }
+  var syncTaskUrl = config.enableWebDataRemoteRegistry ? `${config.webDataRemoteRegistry}/${name}/sync` : null;
+  yield this.render('sync', {
+    type: type,
+    name: name,
+    title: 'Sync ' + type + ' - ' + name,
+    syncTaskUrl,
+  });
+};

controllers/web/user.js

@@ -1,45 +0,0 @@
-/*!
- * cnpmjs.org - controllers/web/package.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com> (http://deadhorse.me)
- */
-
-'use strict';
-
-/**
- * Module dependencies.
- */
-var Module = require('../../proxy/module');
-var User = require('../../proxy/user');
-var eventproxy = require('eventproxy');
-
-exports.display = function (req, res, next) {
-  var name = req.params.name;
-
-  var ep = eventproxy.create();
-  ep.fail(next);
-  Module.listByAuthor(name, ep.done('packages'));
-  User.get(name, ep.done('user'));
-
-  ep.all('packages', 'user', function (packages, user) {
-    //because of sync, maybe no this user in database,
-    //but his packages in this registry
-    if (!user && !packages.length) {
-      return next();
-    }
-    user = {
-      name: name,
-      email: user && user.email
-    };
-
-    return res.render('profile', {
-      title: 'User - ' + name,
-      packages: packages || [],
-      user: user
-    });
-  });
-};

controllers/web/user/show.js

@@ -0,0 +1,57 @@
+'use strict';
+
+var config = require('../../../config');
+var packageService = require('../../../services/package');
+var userService = require('../../../services/user');
+var common = require('../../../lib/common');
+var he = require('he');
+
+module.exports = function* showUser(next) {
+  var name = this.params.name;
+  var isAdmin = common.isAdmin(name);
+  var scopes = config.scopes || [];
+  var user;
+  var r = yield [packageService.listModulesByUser(name), userService.getAndSave(name)];
+  var packages = r[0];
+  var user = r[1];
+  if (!user && !packages.length) {
+    return yield next;
+  }
+
+  user = user || {};
+
+  var data = {
+    name: name,
+    email: user.email ? he.encode(user.email, { encodeEverything: true }) : user.email,
+    json: user.json || {},
+    isNpmUser: user.isNpmUser,
+  };
+
+  if (data.json.login) {
+    // custom user format
+    // convert to npm user format
+    var json = data.json;
+    data.json = {
+      _id: 'org.couchdb.user:' + user.name,
+      _rev: user.rev,
+      name: user.name,
+      email: user.email,
+      type: 'user',
+      roles: [],
+      date: user.gmt_modified,
+      avatar: json.avatar_url,
+      fullname: json.name || json.login,
+      homepage: json.html_url,
+      im: json.im_url
+    };
+  }
+
+  yield this.render('profile', {
+    title: 'User - ' + name,
+    packages: packages,
+    user: data,
+    lastModified: user.gmt_modified,
+    isAdmin: isAdmin,
+    scopes: scopes
+  });
+};

dispatch.js

@@ -1,59 +1,67 @@
-/*!
- * cnpmjs.org - dispatch.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  dead_horse <dead_horse@qq.com>
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
+var childProcess = require('child_process');
 var path = require('path');
 var util = require('util');
-var fs = require('fs');
-var cluster = require('cluster');
+var cfork = require('cfork');
 var config = require('./config');
 var workerPath = path.join(__dirname, 'worker.js');
-var childProcess = require('child_process');
 var syncPath = path.join(__dirname, 'sync');
+var scopeSyncPath = path.join(__dirname, 'sync/sync_scope');
+
+console.log('Starting cnpmjs.org ...\ncluster: %s\nadmins: %j\nscopes: %j\nsourceNpmRegistry: %s\nsyncModel: %s',
+  config.enableCluster, config.admins, config.scopes, config.sourceNpmRegistry, config.syncModel);
 
 if (config.enableCluster) {
-  cluster.setupMaster({
-    exec: workerPath
-  });
-
-  cluster.on('fork', function (worker) {
-    console.log('[%s] [worker:%d] new worker start', new Date(), worker.process.pid);
-  });
-
-  cluster.on('disconnect', function (worker) {
-    var w = cluster.fork();
-    console.error('[%s] [master:%s] wroker:%s disconnect! new worker:%s fork',
-      new Date(), process.pid, worker.process.pid, w.process.pid);
-  });
-
-  cluster.on('exit', function (worker, code, signal) {
-    var exitCode = worker.process.exitCode;
-    var err = new Error(util.format('worker %s died (code: %s, signal: %s)', worker.process.pid, exitCode, signal));
-    err.name = 'WorkerDiedError';
-    console.error(err);
-  });
-
-  var numCPUs = require('os').cpus().length;
-  // Fork workers.
-  for (var i = 0; i < numCPUs; i++) {
-    cluster.fork();
-  }
-
-  childProcess.fork(syncPath);
+  forkWorker();
+  config.syncModel !== 'none' && forkSyncer();
+  // sync assign private scope package
+  config.syncScope && forkScopeSyncer();
 } else {
   require(workerPath);
-  require(syncPath);
+  config.syncModel !== 'none' && require(syncPath);
+  // sync assign private scope package
+  config.syncScope && require(scopeSyncPath);
+}
+
+function forkWorker() {
+  cfork({
+    exec: workerPath,
+    count: config.numCPUs,
+  }).on('fork', function (worker) {
+    console.log('[%s] [worker:%d] new worker start', Date(), worker.process.pid);
+  }).on('disconnect', function (worker) {
+    console.error('[%s] [master:%s] wroker:%s disconnect, suicide: %s, state: %s.',
+      Date(), process.pid, worker.process.pid, worker.suicide, worker.state);
+  }).on('exit', function (worker, code, signal) {
+    var exitCode = worker.process.exitCode;
+    var err = new Error(util.format('worker %s died (code: %s, signal: %s, suicide: %s, state: %s)',
+      worker.process.pid, exitCode, signal, worker.suicide, worker.state));
+    err.name = 'WorkerDiedError';
+    console.error('[%s] [master:%s] wroker exit: %s', Date(), process.pid, err.stack);
+  });
+}
+
+function forkSyncer() {
+  var syncer = childProcess.fork(syncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkSyncer, 1000);
+  });
+}
+
+function forkScopeSyncer() {
+  var syncer = childProcess.fork(scopeSyncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkScopeSyncer, 1000);
+  });
 }

docker-compose.yml

@@ -0,0 +1,30 @@
+version: '3'
+services:
+  web:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: cnpmjs.org
+    depends_on:
+      - mysql-db
+    volumes:
+      - cnpm-files-volume:/var/data/cnpm_data
+    ports:
+      - "7001:7001"
+      - "7002:7002"
+
+  mysql-db:
+    image: mysql:5.6
+    restart: always
+    environment :
+      MYSQL_USER: root      
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"      
+      MYSQL_DATABASE: cnpmjs_test
+      MYSQL_ROOT_HOST: "%"
+    volumes:
+      - ./docs/db.sql:/docker-entrypoint-initdb.d/cnpm-init.sql           
+      - cnpm-db-volume:/var/lib/mysql 
+
+volumes:
+  cnpm-files-volume:
+  cnpm-db-volume:    

docs/Migrating-from-1.x-to-2.x.md

@@ -0,0 +1,143 @@
+# Migrating from 1.x to 2.x
+
+2.x using [Sequelize] ORM to supports MySQL, MariaDB, SQLite or PostgreSQL databases.
+
+## New download total table structure
+
+### Create `downloads` table SQL
+
+You should create `downloads` table first:
+
+```sql
+CREATE TABLE IF NOT EXISTS `downloads` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+  `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
+  `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
+  `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
+  `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
+  `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
+  `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
+  `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
+  `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
+  `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
+  `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
+  `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
+  `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
+  `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
+  `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
+  `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
+  `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
+  `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
+  `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
+  `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
+  `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
+  `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
+  `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
+  `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
+  `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
+  `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
+  `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
+  `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
+  `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
+  `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
+  `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
+  `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
+  `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_date` (`name`, `date`),
+  KEY `date` (`date`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+```
+
+### Sync `download_total` to `downloads`
+
+Then use [sync_download_total.js](../tools/sync_download_total.js) scrpt to sync datas from `download_total`:
+
+```bash
+$ node --harmony tools/sync_download_total.js
+```
+
+# `config.js` changes in 2.x
+
+## New database config
+
+```js
+/**
+* database config
+*/
+
+database: {
+  db: 'cnpmjs_test',
+  username: 'root',
+  password: '',
+
+  // the sql dialect of the database
+  // - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
+  dialect: 'sqlite',
+
+  // custom host; default: 127.0.0.1
+  host: '127.0.0.1',
+
+  // custom port; default: 3306
+  port: 3306,
+
+  // use pooling in order to reduce db connection overload and to increase speed
+  // currently only for mysql and postgresql (since v1.5.0)
+  pool: {
+    maxConnections: 10,
+    minConnections: 0,
+    maxIdleTime: 30000
+  },
+
+  // the storage engine for 'sqlite'
+  // default store into ~/cnpmjs.org.sqlite
+  storage: path.join(process.env.HOME || root, 'cnpmjs.org.sqlite'),
+
+  logging: !!process.env.SQL_DEBUG,
+},
+```
+
+If you're still using MySQL and old config.js `mysqlServers: []` from 1.x:
+
+```js
+mysqlServers: [
+{
+  host: '127.0.0.1',
+  port: 3306,
+  user: 'root',
+  password: ''
+}
+],
+mysqlDatabase: 'cnpmjs_test',
+mysqlMaxConnections: 4,
+mysqlQueryTimeout: 5000,
+```
+
+We will do forward compat, and auto change old style config.js to:
+
+```js
+database: {
+  db: 'cnpmjs_test',
+  username: 'root',
+  password: '',
+  dialect: 'mysql',
+  host: '127.0.0.1',
+  port: 3306,
+  pool: {
+    maxConnections: 10,
+    minConnections: 0,
+    maxIdleTime: 30000
+    },
+  logging: !!process.env.SQL_DEBUG,
+},
+```
+
+## remove `adaptScope`
+
+`adaptScope: true | false` feature was removed.
+
+
+[Sequelize]: http://sequelizejs.com/

docs/db.sql

@@ -1,70 +1,197 @@
-CREATE TABLE `user` (
+CREATE TABLE IF NOT EXISTS `user` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `name` varchar(100) NOT NULL COMMENT 'user name',
- `salt` varchar(100) NOT NULL,
+ `salt` varchar(100) NOT NULL COMMENT 'user salt',
  `password_sha` varchar(100) NOT NULL COMMENT 'user password hash',
  `ip` varchar(64) NOT NULL COMMENT 'user last request ip',
- `roles` varchar(200) NOT NULL DEFAULT '[]',
- `rev` varchar(40) NOT NULL,
- `email` varchar(400) NOT NULL,
+ `roles` varchar(200) NOT NULL DEFAULT '[]' COMMENT 'user roles',
+ `rev` varchar(40) NOT NULL COMMENT 'user rev',
+ `email` varchar(400) NOT NULL COMMENT 'user email',
+ `json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details',
+ `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false',
  PRIMARY KEY (`id`),
- UNIQUE KEY `name` (`name`),
- KEY `gmt_modified` (`gmt_modified`)
+ UNIQUE KEY `uk_name` (`name`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='user base info';
+-- ALTER TABLE `user`
+--   ADD `json` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'json details',
+--   ADD `npm_user` tinyint(1) DEFAULT '0' COMMENT 'user sync from npm or not, 1: true, other: false';
+-- ALTER TABLE `user` CHANGE `json` `json` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'json details';
+-- ALTER TABLE `user`
+--   CHANGE `salt` `salt` varchar(100) NOT NULL COMMENT 'user salt',
+--   CHANGE `roles` `roles` varchar(200) NOT NULL DEFAULT '[]' COMMENT 'user roles',
+--   CHANGE `rev` `rev` varchar(40) NOT NULL COMMENT 'user rev',
+--   CHANGE `email` `email` varchar(400) NOT NULL COMMENT 'user email';
 
-CREATE TABLE `module` (
+CREATE TABLE IF NOT EXISTS `module_keyword` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `keyword` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'keyword',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `description` longtext COMMENT 'module description',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_keyword_module_name` (`keyword`,`name`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module keyword';
+-- ALTER TABLE `module_keyword`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--   CHANGE `description` `description` longtext COMMENT 'module description';
+
+CREATE TABLE IF NOT EXISTS `module_star` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_user_module_name` (`user`,`name`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
+-- ALTER TABLE `module_star`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_user_module_name` (`user`,`name`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='private module maintainers';
+-- ALTER TABLE `module_maintainer`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `npm_module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_user_module_name` (`user`,`name`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='npm original module maintainers';
+-- ALTER TABLE `npm_module_maintainer`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `author` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `author` varchar(100) NOT NULL COMMENT 'module author',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
- `description` longtext,
- `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
- `dist_shasum` varchar(100) DEFAULT NULL,
- `dist_tarball` varchar(2048) DEFAULT NULL,
- `dist_size` int(10) unsigned NOT NULL DEFAULT '0',
- `publish_time` bigint(20) unsigned,
+ `description` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'module description',
+ `package` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'package.json',
+ `dist_shasum` varchar(100) DEFAULT NULL COMMENT 'module dist SHASUM',
+ `dist_tarball` varchar(2048) DEFAULT NULL COMMENT 'module dist tarball',
+ `dist_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'module dist size',
+ `publish_time` bigint(20) unsigned COMMENT 'module publish time',
  PRIMARY KEY (`id`),
- UNIQUE KEY `name` (`name`,`version`),
- KEY `gmt_modified` (`gmt_modified`),
- KEY `publish_time` (`publish_time`),
- KEY `author` (`author`)
+ UNIQUE KEY `uk_name` (`name`,`version`),
+ KEY `idx_gmt_modified` (`gmt_modified`),
+ KEY `idx_publish_time` (`publish_time`),
+ KEY `idx_author` (`author`),
+ KEY `idx_name_gmt_modified` (`name`,`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module info';
 -- ALTER TABLE `module` ADD `description` longtext;
 -- ALTER TABLE `module` ADD `publish_time` bigint(20) unsigned, ADD KEY `publish_time` (`publish_time`);
 -- ALTER TABLE `module` CHANGE `package` `package` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- ALTER TABLE `module` CHANGE `description` `description` LONGTEXT CHARACTER SET utf8 COLLATE utf8_general_ci;
 -- show create table module\G
+-- ALTER TABLE  `module` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE `module`
+--   CHANGE `author` `author` varchar(100) NOT NULL COMMENT 'module author',
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--   CHANGE `version` `version` varchar(30) NOT NULL COMMENT 'module version',
+--   CHANGE `description` `description` longtext COMMENT 'module description',
+--   CHANGE `package` `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'package.json',
+--   CHANGE `dist_shasum` `dist_shasum` varchar(100) DEFAULT NULL COMMENT 'module dist SHASUM',
+--   CHANGE `dist_tarball` `dist_tarball` varchar(2048) DEFAULT NULL COMMENT 'module dist tarball',
+--   CHANGE `dist_size` `dist_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'module dist size',
+--   CHANGE `publish_time` `publish_time` bigint(20) unsigned COMMENT 'module publish time';
 
-CREATE TABLE `module_log` (
+CREATE TABLE IF NOT EXISTS `module_abbreviated` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `username` varchar(100) NOT NULL,
- `name` varchar(100) NOT NULL COMMENT 'module name',
- `log` longtext,
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `version` varchar(30) NOT NULL COMMENT 'module version',
+ `package` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the abbreviated metadata',
+ `publish_time` bigint(20) unsigned COMMENT 'the publish time',
  PRIMARY KEY (`id`),
- KEY `name` (`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+ UNIQUE KEY `uk_name` (`name`,`version`),
+ KEY `idx_gmt_modified` (`gmt_modified`),
+ KEY `idx_publish_time` (`publish_time`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module abbreviated info';
+-- ALTER TABLE `module_abbreviated`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--   CHANGE `publish_time` `publish_time` bigint(20) unsigned COMMENT 'the publish time';
 
-CREATE TABLE `tag` (
+CREATE TABLE IF NOT EXISTS `package_readme` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `name` varchar(100) NOT NULL COMMENT 'module name',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `readme` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT 'the latest version readme',
+ `version` varchar(30) NOT NULL COMMENT 'module version',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_name` (`name`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='package latest readme';
+-- ALTER TABLE `package_readme`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `module_log` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `username` varchar(100) NOT NULL COMMENT 'which username',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `log` longtext COMMENT 'the raw log',
+ PRIMARY KEY (`id`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module sync log';
+-- ALTER TABLE  `module_log` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE `module_log`
+--   CHANGE `username` `username` varchar(100) NOT NULL COMMENT 'which username',
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--   CHANGE `log` `log` longtext COMMENT 'the raw log';
+
+CREATE TABLE IF NOT EXISTS `tag` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
  `tag` varchar(30) NOT NULL COMMENT 'tag name',
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
  PRIMARY KEY (`id`),
- UNIQUE KEY `name` (`name`, `tag`)
+ UNIQUE KEY `uk_name` (`name`, `tag`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
 -- ALTER TABLE  `tag` ADD  `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
+-- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE `tag` ADD KEY `gmt_modified` (`gmt_modified`);
+-- ALTER TABLE `tag`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
 
-CREATE TABLE `total` (
- `name` varchar(100) NOT NULL COMMENT 'total name',
+CREATE TABLE IF NOT EXISTS `module_unpublished` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'base info: tags, time, maintainers, description, versions',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_name` (`name`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module unpublished info';
+-- ALTER TABLE `module_unpublished`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `total` (
+ `name` varchar(214) NOT NULL COMMENT 'total name',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
  `module_delete` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'module delete count',
  `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry',
@@ -77,7 +204,9 @@ CREATE TABLE `total` (
  `last_sync_module` varchar(100) COMMENT 'last sync success module name',
  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='total info';
-INSERT INTO total(name, gmt_modified, module_delete) VALUES('total', now(), 0, 0);
+-- init `total` count
+INSERT INTO total(name, gmt_modified) VALUES('total', now())
+  ON DUPLICATE KEY UPDATE gmt_modified=now();
 -- ALTER TABLE `total` ADD `last_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync from official registry'
 -- ALTER TABLE `total` ADD `last_exist_sync_time` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'last timestamp sync exist packages from official registry'
 -- ALTER TABLE `total` ADD `sync_status` tinyint unsigned NOT NULL DEFAULT '0' COMMENT 'system sync from official registry status'
@@ -86,14 +215,132 @@ INSERT INTO total(name, gmt_modified, module_delete) VALUES('total', now(), 0, 0
 -- ALTER TABLE `total` ADD `fail_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages sync fail at this time'
 -- ALTER TABLE `total` ADD `left_sync_num` int unsigned NOT NULL DEFAULT '0' COMMENT 'how many packages left to be sync'
 -- ALTER TABLE `total` ADD `last_sync_module` varchar(100) NOT NULL COMMENT 'last sync success module name';
+-- ALTER TABLE `total` CHANGE `name` `name` varchar(214) NOT NULL COMMENT 'total name';
 
-CREATE TABLE `download_total` (
+-- CREATE TABLE IF NOT EXISTS `download_total` (
+--  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+--  `gmt_create` datetime NOT NULL COMMENT 'create time',
+--  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+--  `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format',
+--  `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--  `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
+--  PRIMARY KEY (`id`),
+--  UNIQUE KEY `date_name` (`date`, `name`)
+-- ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE  `download_total` CHANGE  `name`  `name` VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE  `download_total` CHANGE `date` `date` datetime NOT NULL COMMENT 'YYYY-MM-DD format';
+
+CREATE TABLE IF NOT EXISTS `downloads` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+  `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+  `date` int unsigned NOT NULL COMMENT 'YYYYMM format',
+  `d01` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '01 download count',
+  `d02` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '02 download count',
+  `d03` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '03 download count',
+  `d04` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '04 download count',
+  `d05` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '05 download count',
+  `d06` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '06 download count',
+  `d07` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '07 download count',
+  `d08` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '08 download count',
+  `d09` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '09 download count',
+  `d10` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '10 download count',
+  `d11` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '11 download count',
+  `d12` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '12 download count',
+  `d13` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '13 download count',
+  `d14` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '14 download count',
+  `d15` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '15 download count',
+  `d16` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '16 download count',
+  `d17` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '17 download count',
+  `d18` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '18 download count',
+  `d19` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '19 download count',
+  `d20` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '20 download count',
+  `d21` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '21 download count',
+  `d22` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '22 download count',
+  `d23` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '23 download count',
+  `d24` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '24 download count',
+  `d25` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '25 download count',
+  `d26` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '26 download count',
+  `d27` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '27 download count',
+  `d28` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '28 download count',
+  `d29` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '29 download count',
+  `d30` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '30 download count',
+  `d31` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT '31 download count',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_date` (`name`, `date`),
+  KEY `idx_date` (`date`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+-- ALTER TABLE `downloads`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name';
+
+CREATE TABLE IF NOT EXISTS `module_deps` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `deps` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_name_deps` (`name`,`deps`),
+ KEY `idx_name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
+-- ALTER TABLE `module_deps`
+--   CHANGE `name` `name` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+--   CHANGE `deps` `deps` varchar(214) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'which module depend on this module';
+
+CREATE TABLE IF NOT EXISTS `dist_dir` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
  `gmt_modified` datetime NOT NULL COMMENT 'modified time',
- `date` varchar(10) NOT NULL COMMENT 'YYYY-MM-DD format',
- `name` varchar(100) NOT NULL COMMENT 'module name',
- `count` bigint(20) unsigned NOT NULL DEFAULT '0' COMMENT 'download count',
+ `name` varchar(214) NOT NULL COMMENT 'dir name',
+ `parent` varchar(214) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
  PRIMARY KEY (`id`),
- UNIQUE KEY `date_name` (`date`, `name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module download total info';
+ UNIQUE KEY `uk_name` (`parent`, `name`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
+-- ALTER TABLE `dist_dir`
+--   CHANGE `name` `name` varchar(214) NOT NULL COMMENT 'dir name',
+--   CHANGE `parent` `parent` varchar(214) NOT NULL COMMENT 'parent dir' DEFAULT '/';
+
+CREATE TABLE IF NOT EXISTS `dist_file` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(214) NOT NULL COMMENT 'file name',
+ `parent` varchar(214) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
+ `sha1` varchar(40) COMMENT 'sha1 hex value',
+ `url` varchar(2048),
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_fullname` (`parent`, `name`),
+ KEY `idx_gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';
+-- ALTER TABLE `dist_file`
+--   CHANGE `name` `name` varchar(214) NOT NULL COMMENT 'file name',
+--   CHANGE `parent` `parent` varchar(214) NOT NULL COMMENT 'parent dir' DEFAULT '/';
+
+CREATE TABLE IF NOT EXISTS `token` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `token` varchar(100) NOT NULL COMMENT 'token',
+ `user_id` varchar(100) NOT NULL COMMENT 'user name',
+ `readonly` tinyint NOT NULL DEFAULT 0 COMMENT 'readonly or not, 1: true, other: false',
+ `token_key` varchar(200) NOT NULL COMMENT 'token sha512 hash',
+ `cidr_whitelist` varchar(500) NOT NULL COMMENT 'ip list, ["127.0.0.1"]',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `uk_token` (`token`),
+ KEY `idx_user` (`user_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='token info';
+
+CREATE TABLE IF NOT EXISTS `package_version_blocklist` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+  `gmt_create` datetime(3) NOT NULL COMMENT 'create time',
+  `gmt_modified` datetime(3) NOT NULL COMMENT 'modified time',
+  `name` varchar(214) NOT NULL COMMENT 'package name',
+  `version` varchar(30) NOT NULL COMMENT 'package version, "*" meaning all versions',
+  `reason` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL COMMENT 'block reason',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_version` (`name`, `version`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci COMMENT='package version block list';

docs/dockerize/config.js

@@ -0,0 +1,259 @@
+'use strict';
+
+var mkdirp = require('mkdirp');
+var copy = require('copy-to');
+var path = require('path');
+var fs = require('fs');
+var os = require('os');
+
+var version = require('../package.json').version;
+var Nfs = require('fs-cnpm');
+var root = path.dirname(__dirname);
+var dataDir = process.env.CNPM_DATA_DIR ;
+
+var config = {
+  version: version,
+  dataDir: dataDir,
+
+  /**
+   * Cluster mode
+   */
+  enableCluster: false,
+  numCPUs: os.cpus().length,
+
+  /*
+   * server configure
+   */
+
+  registryPort: 7001,
+  webPort: 7002,
+  bindingHost: '0.0.0.0', // binding on 0.0.0.0 for outside of container access
+
+  // debug mode
+  // if in debug mode, some middleware like limit wont load
+  // logger module will print to stdout
+  debug: process.env.NODE_ENV === 'development',
+  // page mode, enable on development env
+  pagemock: process.env.NODE_ENV === 'development',
+  // session secret
+  sessionSecret: 'cnpmjs.org test session secret',
+  // max request json body size
+  jsonLimit: '10mb',
+  // log dir name
+  logdir: path.join(dataDir, 'logs'),
+  // update file template dir
+  uploadDir: path.join(dataDir, 'downloads'),
+  // web page viewCache
+  viewCache: false,
+
+  // config for koa-limit middleware
+  // for limit download rates
+  limit: {
+    enable: false,
+    token: 'koa-limit:download',
+    limit: 1000,
+    interval: 1000 * 60 * 60 * 24,
+    whiteList: [],
+    blackList: [],
+    message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
+  },
+
+  enableCompress: false, // enable gzip response or not
+
+  // default system admins
+  admins: {
+    // name: email
+    fengmk2: 'fengmk2@gmail.com',
+    admin: 'admin@cnpmjs.org',
+    dead_horse: 'dead_horse@qq.com',
+  },
+
+  // email notification for errors
+  // check https://github.com/andris9/Nodemailer for more informations
+  mail: {
+    enable: false,
+    appname: 'cnpmjs.org',
+    from: 'cnpmjs.org mail sender <adderss@gmail.com>',
+    service: 'gmail',
+    auth: {
+      user: 'address@gmail.com',
+      pass: 'your password'
+    }
+  },
+
+  logoURL: 'https://os.alipayobjects.com/rmsportal/oygxuIUkkrRccUz.jpg', // cnpm logo image url
+  adBanner: '',
+  customReadmeFile: '', // you can use your custom readme file instead the cnpm one
+  customFooter: '', // you can add copyright and site total script html here
+  npmClientName: 'cnpm', // use `${name} install package`
+
+  // max handle number of package.json `dependencies` property
+  maxDependencies: 200,
+
+  /**
+   * database config
+   */
+
+  database: {
+    db: 'cnpmjs_test',
+    username: 'root',
+    password: '',
+
+    // the sql dialect of the database
+    // - currently supported: 'mysql', 'sqlite', 'postgres', 'mariadb'
+    dialect: 'mysql',
+
+    // the Docker container network hostname defined at docker-compose.yml
+    host: 'mysql-db',
+
+    // custom port; default: 3306
+    port: 3306,
+
+    // use pooling in order to reduce db connection overload and to increase speed
+    // currently only for mysql and postgresql (since v1.5.0)
+    pool: {
+      maxConnections: 10,
+      minConnections: 0,
+      maxIdleTime: 30000
+    },
+
+    // the storage engine for 'sqlite'
+    // default store into ~/.cnpmjs.org/data.sqlite
+    //storage: path.join(dataDir, 'data.sqlite'),
+
+    logging: !!process.env.SQL_DEBUG,
+  },
+
+  // package tarball store in local filesystem by default
+  nfs: new Nfs({
+    dir: path.join(dataDir, 'nfs')
+  }),
+  // if set true, will 302 redirect to `nfs.url(dist.key)`
+  downloadRedirectToNFS: false,
+
+  // registry url name
+  registryHost: '127.0.0.1:7001',
+
+  /**
+   * registry mode config
+   */
+
+  // enable private mode or not
+  // private mode: only admins can publish, other users just can sync package from source npm
+  // public mode: all users can publish
+  enablePrivate: false,
+
+  // registry scopes, if don't set, means do not support scopes
+  scopes: [ '@cnpm', '@cnpmtest', '@cnpm-test' ],
+
+  // some registry already have some private packages in global scope
+  // but we want to treat them as scoped private packages,
+  // so you can use this white list.
+  privatePackages: [],
+
+  /**
+   * sync configs
+   */
+
+  // the official npm registry
+  // cnpm wont directly sync from this one
+  // but sometimes will request it for some package infomations
+  // please don't change it if not necessary
+  officialNpmRegistry: 'https://registry.npmjs.com',
+  officialNpmReplicate: 'https://replicate.npmjs.com',
+
+  // sync source, upstream registry
+  // If you want to directly sync from official npm's registry
+  // please drop them an email first
+  sourceNpmRegistry: 'https://registry.npmmirror.com',
+
+  // upstream registry is base on cnpm/cnpmjs.org or not
+  // if your upstream is official npm registry, please turn it off
+  sourceNpmRegistryIsCNpm: true,
+
+  // if install return 404, try to sync from source registry
+  syncByInstall: true,
+
+  // sync mode select
+  // none: do not sync any module, proxy all public modules from sourceNpmRegistry
+  // exist: only sync exist modules
+  // all: sync all modules
+  syncModel: 'none', // 'none', 'all', 'exist'
+
+  syncConcurrency: 1,
+  // sync interval, default is 10 minutes
+  syncInterval: '10m',
+
+  // sync polular modules, default to false
+  // because cnpm can't auto sync tag change for now
+  // so we want to sync popular modules to ensure their tags
+  syncPopular: false,
+  syncPopularInterval: '1h',
+  // top 100
+  topPopular: 100,
+
+  // sync devDependencies or not, default is false
+  syncDevDependencies: false,
+
+  // changes streaming sync
+  syncChangesStream: false,
+  handleSyncRegistry: 'http://127.0.0.1:7001',
+
+  // badge subject on http://shields.io/
+  badgePrefixURL: 'https://img.shields.io/badge',
+  badgeSubject: 'cnpm',
+
+  // custom user service, @see https://github.com/cnpm/cnpmjs.org/wiki/Use-Your-Own-User-Authorization
+  // when you not intend to ingegrate with your company's user system, then use null, it would
+  // use the default cnpm user system
+  userService: null,
+
+  // always-auth https://docs.npmjs.com/misc/config#always-auth
+  // Force npm to always require authentication when accessing the registry, even for GET requests.
+  alwaysAuth: false,
+
+  // if you're behind firewall, need to request through http proxy, please set this
+  // e.g.: `httpProxy: 'http://proxy.mycompany.com:8080'`
+  httpProxy: null,
+
+  // snyk.io root url
+  snykUrl: 'https://snyk.io',
+
+  // https://github.com/cnpm/cnpmjs.org/issues/1149
+  // if enable this option, must create module_abbreviated and package_readme table in database
+  enableAbbreviatedMetadata: true,
+
+  // global hook function: function* (envelope) {}
+  // envelope format please see https://github.com/npm/registry/blob/master/docs/hooks/hooks-payload.md#payload
+  globalHook: null,
+  accelerateHostMap: {},
+};
+
+if (process.env.NODE_ENV !== 'test') {
+  var customConfig;
+  if (process.env.NODE_ENV === 'development') {
+    customConfig = path.join(root, 'config', 'config.js');
+  } else {
+    // 1. try to load `$dataDir/config.json` first, not exists then goto 2.
+    // 2. load config/config.js, everything in config.js will cover the same key in index.js
+    customConfig = path.join(dataDir, 'config.json');
+    if (!fs.existsSync(customConfig)) {
+      customConfig = path.join(root, 'config', 'config.js');
+    }
+  }
+  if (fs.existsSync(customConfig)) {
+    copy(require(customConfig)).override(config);
+  }
+}
+
+mkdirp.sync(config.logdir);
+mkdirp.sync(config.uploadDir);
+
+module.exports = config;
+
+config.loadConfig = function (customConfig) {
+  if (!customConfig) {
+    return;
+  }
+  copy(customConfig).override(config);
+};

docs/flow/publish.md

@@ -1,5 +1,24 @@
 # npm publish flow
 
+## Flows
+
+1. try to put package.json and tgz, maybe base64 tgz body
+2. if new version not exists, publish success
+3. if new version exists, 409, try to get full package info with ?write=true
+4. if new version had publish, show: "Update the 'version' field in package.json and try again."
+
+```bash
+$ cnpm publish
+npm http PUT http://r.cnpmjs.org/cnpmjs.org
+npm http 409 http://r.cnpmjs.org/cnpmjs.org
+npm http GET http://r.cnpmjs.org/cnpmjs.org?write=true
+npm http 200 http://r.cnpmjs.org/cnpmjs.org?write=true
+npm ERR! publish fail Cannot publish over existing version.
+npm ERR! publish fail Update the 'version' field in package.json and try again.
+```
+
+## Details
+
 code: https://github.com/isaacs/npm-registry-client/blob/master/lib/publish.js
 
 * couch login if token not exists: [couch-login](https://github.com/isaacs/couch-login)

docs/network.puml

@@ -0,0 +1,98 @@
+@startuml
+
+title cnpmjs.org, npmmirror.com Network
+
+node "China User" {
+  [cnpm cli]
+}
+
+node "OSS: aliyuncs.com" {
+  [tnpm-hz.oss-cn-hangzhou]
+}
+
+node "SLB: 114.55.80.225 Hangzhou" {
+  [npmmirror.com]
+  [registry.npmmirror.com]
+}
+
+node "npmtaobao3: 121.41.*" {
+  [nginx 5]
+  [registry 5]
+  [web 5]
+  [syncer 5]
+}
+
+node "npmtaobao4: 120.26.*" {
+  [nginx 6]
+  [registry 6]
+  [web 6]
+}
+
+node "cnpm6: 47.88.189.193 Singapore" {
+  [cnpmjs.org]
+  [r.cnpmjs.org]
+  [nginx 7]
+  [registry 7]
+  [web 7]
+  [syncer 7]
+}
+
+node "Aliyun CDN" {
+  [oss.npmmirror.com]
+}
+
+node "qiniu CDN" {
+  [cnpmjs.up.qiniu.com]
+  [dn-cnpm.qbox.me]
+}
+
+database "taobaonpm DB" {
+  [rds2860*.mysql.rds.aliyuncs.com]
+}
+
+database "cnpm DB" {
+  [rdsqiz*.mysql.rds.aliyuncs.com]
+}
+
+node "npmjs.com" {
+  [registry.npmjs.com]
+  [www.npmjs.com]
+}
+
+[cnpm cli] -down-> [registry.npmmirror.com]: Read, Write
+[cnpm cli] -down-> [oss.npmmirror.com]: Read tgz
+[cnpm cli] -down-> [npmmirror.com]: "Read /mirrors"
+
+[registry.npmmirror.com] -down-> [nginx 5]
+[nginx 5] -down-> [registry 5]
+[npmmirror.com] -down-> [nginx 5]
+[nginx 5] -down-> [web 5]
+[registry.npmmirror.com] -down-> [nginx 6]
+[nginx 6] -down-> [registry 6]
+[npmmirror.com] -down-> [nginx 6]
+[nginx 6] -down-> [web 6]
+
+[registry 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
+[web 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read
+[syncer 5] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
+[syncer 5] -down-> [tnpm-hz.oss-cn-hangzhou]: Write
+[syncer 5] -> [r.cnpmjs.org]: Read
+[syncer 5] -> [dn-cnpm.qbox.me]: Read tgz
+[registry 6] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read, Write
+[web 6] -down-> [rds2860*.mysql.rds.aliyuncs.com]: Read
+
+[cnpmjs.org] -down-> [nginx 7]
+[nginx 7] -down-> [registry 7]
+[r.cnpmjs.org] -down-> [nginx 7]
+[nginx 7] -down-> [web 7]
+
+[registry 7] -down-> [rdsqiz*.mysql.rds.aliyuncs.com]: Read, Write
+[web 7] -down-> [rdsqiz*.mysql.rds.aliyuncs.com]: Read
+[syncer 7] -down-> [rdsqiz*.mysql.rds.aliyuncs.com]: Read, Write
+[syncer 7] -down-> [cnpmjs.up.qiniu.com]: Write
+[syncer 7] -down-> [registry.npmjs.com]: Read
+
+[oss.npmmirror.com] -down-> [tnpm-hz.oss-cn-hangzhou]: Read
+[dn-cnpm.qbox.me] -down-> [cnpmjs.up.qiniu.com]: Read
+
+@enduml

docs/update_sqls/v1.6.x-upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS `npm_module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='npm original module maintainers';

docs/web/install.md

@@ -1,113 +0,0 @@
-# Install & Get Started
-
-## Deps
-
-* MySQL Server: http://db4free.net/
-* qiniu CDN: http://www.qiniu.com/
-* redis session: https://garantiadata.com Support 24MB free spaces.
-* node: >=0.10.21
-
-## Clone
-
-```bash
-$ git clone git://github.com/fengmk2/cnpmjs.org.git $HOME/cnpmjs.org
-$ cd $HOME/cnpmjs.org
-```
-
-## Create your `config.js`
-
-```bash
-$ vim config/config.js
-```
-
-`config.js` content sample:
-
-```js
-module.exports = {
-  debug: false,
-  enableCluster: true, // enable cluster mode
-  logdir: 'your application log dir',
-  mysqlServers: [
-    {
-      host: 'your mysql host',
-      port: 3306,
-      user: 'yourname',
-      password: 'your password'
-    }
-  ],
-  mysqlDatabase: 'cnpmjs',
-  redis: {
-    host: 'your redist host',
-    port: 6379,
-  },
-  qn: {
-    accessKey: "your qiniu appkey",
-    secretKey: "your secret key",
-    bucket: "foobucket",
-    domain: "http://foobucket.u.qiniudn.com"
-  },
-  nfs: null, // you can set a nfs to replace qiniu cdn
-  enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
-  admins: {
-    admin: 'admin@cnpmjs.org',
-  },
-  syncModel: 'exist', //`all` sync all packages, `exist` only update  exist packages, `none` do nothing
-};
-```
-
-## Create MySQL Database and Tables
-
-```bash
-$ mysql -u yourname -p
-
-mysql> use cnpmjs;
-mysql> source docs/db.sql
-```
-
-## Use your own CDN
-If you wan to use your own CDN instead of qiniu. Just look at `common/qnfs.js` and implement the interface like it, then pass it by set `config.nfs`.
-
-## npm install
-
-```bash
-$ npm install
-```
-
-## start
-
-```bash
-$ ./bin/nodejsctl start
-
-Starting cnpmjs.org ...
-Start nodejs success. PID=27175
-```
-
-## open registry and web
-
-```bash
-# registry
-$ open http://localhost:7001
-# web
-$ open http://localhost:7002
-```
-
-## use cnpm cli with your own registry  
-You do not need to write another command line tool with your own registry, 
-just alias [cnpm](http://github.com/fengmk2/cnpm), then you can get a npm client for you own registry.
-
-```
-# install cnpm first
-npm install -g cnpm 
-
-# then alias lnpm to cnpm, but change config to your own registry
-alias lnpm='cnpm --registry=http://localhost:7001\
- --registryweb=http://localhost:7002\
- --cache=$HOME/.npm/.cache/lnpm\
- --userconfig=$HOME/.lnpmrc'
-
- #or put this in .zshrc or .bashrc
- echo "#lnpm alias\nalias lnpm='cnpm --registry=http://localhost:7001\
- --registryweb=http://localhost:7002\
- --cache=$HOME/.npm/.cache/lnpm\
- --userconfig=$HOME/.lnpmrc'" >> $HOME/.zshrc && source $HOME/.zshrc
-```

docs/web/readme.md

@@ -1,24 +1,16 @@
-# cnpmjs.org: Private npm registry and web for Enterprise
+# cnpmjs.org: Private npm registry and web for Company
 
-[![Build Status](https://secure.travis-ci.org/fengmk2/cnpmjs.org.png)](http://travis-ci.org/fengmk2/cnpmjs.org) [![Coverage Status](https://coveralls.io/repos/fengmk2/cnpmjs.org/badge.png)](https://coveralls.io/r/fengmk2/cnpmjs.org) [![Dependency Status](https://gemnasium.com/fengmk2/cnpmjs.org.png)](https://gemnasium.com/fengmk2/cnpmjs.org)
-
-[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
-
-## What is this?
-
-> Private npm registry and web for Enterprise, base on MySQL and Simple File Store.
-
-@[JacksonTian](https://github.com/JacksonTian/) had a talk about [private npm](https://speakerdeck.com/jacksontian/qi-ye-ji-node-dot-jskai-fa).
-
-## Install your private npm registry
-
-@see [Install and Get Started](/install).
+So `cnpm` is meaning: **Company npm**.
 
 ## Registry
 
-* Our public registry: [registry.cnpmjs.org](http://registry.cnpmjs.org)
-* Current [cnpmjs.org](/) version: <span id="app-version"></span>
+- Our public registry: [r.cnpmjs.org](https://r.cnpmjs.org), syncing from [registry.npmjs.com](https://registry.npmjs.com)
+- [cnpmjs.org](/) version: <span id="app-version"></span>
+- [Node.js](https://nodejs.org) version: <span id="node-version"></span>
+- For developers in China, please visit [the China mirror](https://npmmirror.com). 中国用户请访问[国内镜像站点](https://npmmirror.com)。
+- Use the private npm service provided by Alibaba Cloud DevOps which build with cnpm. [https://packages.aliyun.com/](https://packages.aliyun.com/?channel=pd_cnpm_github)
 
+<div class="ant-table">
 <table class="downloads">
   <tbody>
     <tr>
@@ -38,12 +30,17 @@
     </tr>
   </tbody>
 </table>
+</div>
 
 <div class="sync" style="display:none;">
   <h3>Sync Status</h3>
   <p id="sync-model"></p>
   <p>Last sync time is <span id="last-sync-time"></span>. </p>
-  <p class="syncing alert alert-info">The sync worker is working in the backend now. </p>
+  <div class="ant-alert ant-alert-info syncing">
+    <span class="anticon ant-alert-icon anticon-info-circle"></span>
+    <span class="ant-alert-description">The sync worker is working in the backend now. </span>
+  </div>
+  <div class="ant-table">
   <table class="sync-status">
     <tbody>
       <tr>
@@ -58,88 +55,55 @@
       </tr>
     </tbody>
   </table>
+  </div>
 </div>
 
-Running on [Node.js](http://nodejs.org), version <span id="node-version"></span>.
+<script src="/js/readme.js"></script>
 
-<script>
-$(function () {
-  function humanize(n, options) {
-    options = options || {};
-    var d = options.delimiter || ',';
-    var s = options.separator || '.';
-    n = n.toString().split('.');
-    n[0] = n[0].replace(/(\d)(?=(\d\d\d)+(?!\d))/g, '$1' + d);
-    return n.join(s);
-  }
+## Badges
 
-  $.getJSON('/total', function (data) {
-    $('#total-packages').html(humanize(data.doc_count));
-    $('#total-versions').html(humanize(data.doc_version_count));
-    $('#total-deletes').html(humanize(data.doc_del_count));
+Default style is `flat-square`.
 
-    var downloads = $('table.downloads');
-    downloads.find('td.count:eq(3)').html(humanize(data.download.today));
-    downloads.find('td.count:eq(4)').html(humanize(data.download.thisweek));
-    downloads.find('td.count:eq(5)').html(humanize(data.download.thismonth));
-    downloads.find('td.count:eq(6)').html(humanize(data.download.lastday));
-    downloads.find('td.count:eq(7)').html(humanize(data.download.lastweek));
-    downloads.find('td.count:eq(8)').html(humanize(data.download.lastmonth));
+### Version
 
-    $('#node-version').html(data.node_version || 'v0.10.22');
-    $('#app-version').html(data.app_version || '0.0.0');
+Badge URL: `https://cnpmjs.org/badge/v/cnpmjs.org.svg` ![cnpmjs.org-version-badge](//cnpmjs.org/badge/v/cnpmjs.org.svg)
 
-    if (data.sync_model === 'all') {
-      $('#sync-model').html('This registry will sync all packages from official registry.');
-      $('#last-sync-time').html(new Date(data.last_sync_time));
-    } else if (data.sync_model === 'exist') {
-      $('#sync-model').html('This registry will only update exist packages from official registry.');
-      $('#last-sync-time').html(new Date(data.last_exist_sync_time));
-    }
+* `<0.1.0 & >=0.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.0.1-red.svg?style=flat-square)
+* `<1.0.0 & >=0.1.0`: ![red-badge](https://img.shields.io/badge/cnpm-0.1.0-green.svg?style=flat-square)
+* `>=1.0.0`: ![red-badge](https://img.shields.io/badge/cnpm-1.0.0-blue.svg?style=flat-square)
 
-    $('#need-sync').html(data.need_sync_num);
-    $('#success-sync').html(data.success_sync_num);
-    $('#fail-sync').html(data.fail_sync_num);
-    $('#left-sync').html(data.left_sync_num);
-    $('#percent-sync').html(Math.floor(data.success_sync_num / data.need_sync_num * 100));
-    $('#last-success-name').html('<a target="_blank" href="/package/' + data.last_sync_module + '">' +
-      data.last_sync_module + '</a>');
+### Downloads
 
-    if (!data.sync_status) {
-      $('.syncing').html('');
-    }
+Badge URL: `https://cnpmjs.org/badge/d/cnpmjs.org.svg` ![cnpmjs.org-download-badge](//cnpmjs.org/badge/d/cnpmjs.org.svg)
 
-    $('.sync').show();
-  });
-});
-</script>
+## Usage
 
-## cnpm cli
-
-alias it:
+use our npm client [cnpm](https://github.com/cnpm/cnpm)(More suitable with cnpmjs.org and gzip support), you can get our client through npm:
 
 ```bash
-alias cnpm="npm --registry=http://registry.cnpmjs.org \
+$ npm install -g cnpm --registry=https://registry.npmmirror.com
+```
+
+Or you can alias NPM to use it:
+
+```bash
+alias cnpm="npm --registry=https://registry.npmmirror.com \
 --cache=$HOME/.npm/.cache/cnpm \
+--disturl=https://npmmirror.com/mirrors/node \
 --userconfig=$HOME/.cnpmrc"
 
 #Or alias it in .bashrc or .zshrc
-$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=http://registry.cnpmjs.org \
+$ echo '\n#alias for cnpm\nalias cnpm="npm --registry=https://registry.npmmirror.com \
   --cache=$HOME/.npm/.cache/cnpm \
+  --disturl=https://npmmirror.com/mirrors/node \
   --userconfig=$HOME/.cnpmrc"' >> ~/.zshrc && source ~/.zshrc
 ```
 
-Or you can just use our `cnpm` cli:
-
-```bash
-$ npm install cnpm -g
-```
-
 ### install
 
-Install package from [registry.cnpmjs.org](http://registry.cnpmjs.org). When isntall a  package or version not exist, it will try to install from official registry([registry.npmjs.org](http://registry.npmjs.org)), and sync this package to cnpm in the backend.
+Install package from [r.cnpmjs.org](//r.cnpmjs.org). When installing a package or version does not exist, it will try to install from the official registry([registry.npmjs.org](https://registry.npmjs.org)), and sync this package to cnpm in the backend.
 
-```
+```bash
 $ cnpm install [name]
 ```
 
@@ -151,10 +115,10 @@ Only `cnpm` cli has this command. Meaning sync package from source npm.
 $ cnpm sync connect
 ```
 
-sync package on web: [cnpmjs.org/sync/connect](http://cnpmjs.org/sync/connect)
+sync package on web: [sync/connect](/sync/connect)
 
 ```bash
-$ open http://cnpmjs.org/sync/connect
+$ open http://registry.npmmirror.com/sync/connect
 ```
 
 ### publish / unpublish
@@ -166,39 +130,27 @@ $ cnpm publish [name]
 $ cnpm unpublish [name]
 ```
 
-### Orther command
-Support all the other npm command.
+### Other commands
+
+Support all the other npm commands. e.g.:
+
+```bash
+$ cnpm info cnpm
+```
 
 ## TODO list
 
-@see Github [Issues](https://github.com/fengmk2/cnpmjs.org/issues)
+@see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
 
-## Authors
+## Histories
 
 Release [History](/history).
 
-```bash
-$ git summary
+## npmjs.org, cnpmjs.org and npmmirror.com relation
 
- project  : cnpmjs.org
- repo age : 2 weeks
- active   : 81 days
- commits  : 205
- files    : 83
- authors  :
-   120  fengmk2                 58.5%
-    84  dead_horse              41.0%
-     1  Alsotang                0.5%
-```
+![npm&cnpm](https://cloud.githubusercontent.com/assets/543405/21505401/fd0b6220-cca1-11e6-86ed-599cc81bb03b.png)
 
-## npm and cnpm relation
+## Sponsors
 
-![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
-
-## 捐赠 Donate
-如果您觉得 [cnpmjs.org] 对您有帮助，欢迎请作者一杯咖啡.
-
-[![Donate](https://img.alipay.com/sys/personalprod/style/mc/btn-index.png)](https://me.alipay.com/imk2)
-
- [cnpmjs.org]: http://cnpmjs.org/
- [registry.cnpmjs.org]: http://registry.cnpmjs.org/
+- [![阿里云](https://static.aliyun.com/images/www-summerwind/logo.gif)](http://click.aliyun.com/m/4288/) [![阿里云云效](https://img.alicdn.com/tfs/TB116yt3fb2gK0jSZK9XXaEgFXa-106-20.png)](https://devops.aliyun.com/?channel=pd_cnpm_github) (2016.2 - now)
+- [![UCloud云计算](https://www.ucloud.cn/static/style/images/about/logo.png)](http://www.ucloud.cn?sem=sdk-CNPMJS) (2015.3 - 2016.3)

index.js

@@ -0,0 +1,16 @@
+'use strict';
+
+var config = require('./config');
+
+exports.loadConfig = config.loadConfig;
+exports.config = config;
+
+exports.startWorker = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./worker');
+};
+
+exports.startSync = function (customConfig) {
+  config.loadConfig(customConfig);
+  require('./sync');
+};

lib/common.js

@@ -1,35 +1,146 @@
-/**!
- * cnpmjs.org - lib/common.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
-/**
- * Module dependencies.
- */
-
 var crypto = require('crypto');
 var path = require('path');
+var utility = require('utility');
+var util = require('util');
 var config = require('../config');
+var BASIC_PREFIX = /basic /i;
+var BEARER_PREFIX = /bearer /i;
 
-exports.getTarballFilepath = function (filename) {
+exports.getTarballFilepath = function (packageName, packageVersion, filename) {
   // ensure download file path unique
-  var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex') + '.tgz');
+  // TODO: not only .tgz, and also other extname
+  var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex'));
+  // use filename string md5 instead, fix "ENAMETOOLONG: name too long" error
+  name = packageName.replace(/\//g, '-').replace(/\@/g, '') + '-' + packageVersion.substring(0, 20) + '.' + utility.md5(name) + '.tgz';
   return path.join(config.uploadDir, name);
 };
 
 exports.getCDNKey = function (name, filename) {
+  // if name is scope package name, need to auto fix filename as a scope package file name
+  // e.g.: @scope/foo, filename: foo-1.0.0.tgz => filename: @scope/foo-1.0.0.tgz
+  if (name[0] === '@' && filename[0] !== '@') {
+    filename = name.split('/')[0] + '/' + filename;
+  }
   return '/' + name + '/-/' + filename;
 };
 
-exports.downloadURL = function (pkg, req) {
-  if (pkg.dist && pkg.dist.tarball) {
-    pkg.dist.tarball = 'http://' + req.headers.host + '/' + pkg.name + '/download/' + path.basename(pkg.dist.tarball);
+exports.getUnpublishFileKey = function (name) {
+  return `/${name}/sync/unpublish/unpublish-package.json`;
+};
+
+exports.getPackageFileCDNKey = function (name, version) {
+  return `/${name}/sync/packages/package-${version}.json`;
+};
+
+exports.getDistTagCDNKey = function (name, tag) {
+  return `/${name}/sync/tags/tag-${tag}.json`;
+};
+
+exports.getSyncTagDir = function (name) {
+  return `${name}/sync/tags/`;
+};
+
+exports.getSyncPackageDir = function (name) {
+  return `${name}/sync/packages/`;
+};
+
+const TAG_NAME_REG = /^tag-(.+)\.json$/;
+exports.getTagNameFromFileName = function (fileName) {
+  const res = fileName.match(TAG_NAME_REG);
+  return res && res[1];
+};
+
+exports.isBackupTagFile = function (fileName) {
+  return TAG_NAME_REG.test(fileName);
+};
+
+const PACKAGE_NAME_REG = /^package-(.+)\.json$/;
+exports.getVersionFromFileName = function (fileName) {
+  const res = fileName.match(PACKAGE_NAME_REG);
+  return res && res[1];
+};
+
+exports.isBackupPkgFile = function (fileName) {
+  return PACKAGE_NAME_REG.test(fileName);
+};
+
+exports.setDownloadURL = function (pkg, ctx, host) {
+  if (pkg.dist) {
+    host = host || config.registryHost || ctx.host;
+    var protocol = config.protocol || ctx.protocol;
+    pkg.dist.tarball = util.format('%s://%s/%s/download/%s-%s.tgz',
+      protocol,
+      host, pkg.name, pkg.name, pkg.version);
   }
 };
+
+exports.isAdmin = function (username) {
+  return typeof config.admins[username] === 'string';
+};
+
+exports.isMaintainer = function (user, maintainers) {
+  if (user.isAdmin) {
+    return true;
+  }
+
+  var username = user.name;
+  maintainers = maintainers || [];
+  var match = maintainers.filter(function (item) {
+    return item.name === username;
+  });
+
+  return match.length > 0;
+};
+
+exports.isLocalModule = function (mods) {
+  for (var i = 0; i < mods.length; i++) {
+    var r = mods[i];
+    if (config.privatePackages.includes(r.name)) {
+      return true;
+    }
+    if (r.package && r.package._publish_on_cnpm) {
+      return true;
+    }
+  }
+  return false;
+};
+
+exports.isPrivateScopedPackage = function (name) {
+  if (!name) {
+    return false;
+  }
+
+  if (name[0] !== '@') {
+    return false;
+  }
+  return config.scopes.indexOf(name.split('/')[0]) >= 0;
+};
+
+var AuthorizeType = exports.AuthorizeType = {
+  BASIC: 'BASIC',
+  BEARER: 'BEARER',
+};
+
+exports.getAuthorizeType = function (ctx) {
+  var authorization = (ctx.get('authorization') || '').trim();
+  if (BASIC_PREFIX.test(authorization)) {
+    return AuthorizeType.BASIC;
+  } else if (BEARER_PREFIX.test(authorization)) {
+    return AuthorizeType.BEARER;
+  }
+};
+
+exports.isSyncWorkerRequest = function (ctx) {
+  // sync request will contain this query params
+  let isSyncWorkerRequest = ctx.query.cache === '0';
+  if (!isSyncWorkerRequest) {
+    const ua = ctx.headers['user-agent'] || '';
+    // old sync client will request with these user-agent
+    if (ua.indexOf('npm_service.cnpmjs.org/') !== -1) {
+      isSyncWorkerRequest = true;
+    }
+  }
+  return isSyncWorkerRequest;
+};

middleware/auth.js

@@ -1,63 +1,93 @@
-/*!
- * cnpmjs.org - middleware/auth.js
- *
- * Copyright(c) cnpmjs.org and other contributors.
- * MIT Licensed
- *
- * Authors:
- *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
- */
-
 'use strict';
 
+var debug = require('debug')('cnpmjs.org:middleware:auth');
+var UserService = require('../services/user');
+var TokenService = require('../services/token');
+var config = require('../config');
+var common = require('../lib/common');
+
 /**
- * Module dependencies.
+ * Parse the request authorization
+ * get the real user
  */
 
-var debug = require('debug')('cnpmjs.org:middleware:auth');
-var User = require('../proxy/user');
-var config = require('../config');
+module.exports = function () {
+  return function* auth(next) {
+    this.user = {};
 
-module.exports = function (options) {
-  return function auth(req, res, next) {
-    req.session.onlySync = config.enablePrivate ? true : false;
-    if (req.session.name) {
-      if (config.admins[req.session.name]) {
-        req.session.isAdmin = true;
-      }
-      debug('auth exists user: %s, onlySync: %s, isAdmin: %s, headers: %j',
-        req.session.name, req.session.onlySync, req.session.isAdmin, req.headers);
-      return next();
-    }
-    var authorization = (req.headers.authorization || '').split(' ')[1] || '';
-    authorization = authorization.trim();
+    var authorization = (this.get('authorization') || '').trim();
+    debug('%s %s with %j', this.method, this.url, authorization);
     if (!authorization) {
-      return next();
+      return yield unauthorized.call(this, next);
     }
-    authorization = new Buffer(authorization, 'base64').toString().split(':');
-    var username = authorization[0];
-    var password = authorization[1];
 
-    User.auth(username, password, function (err, row) {
-      if (err) {
-        return next(err);
-      }
+    var row;
+    try {
+      var authorizeType = common.getAuthorizeType(this);
 
-      if (!row) {
-        debug('auth fail user: %j, headers: %j', row, req.headers);
-        return res.json(401, {
-          error: 'unauthorized',
-          reason: 'Name or password is incorrect.'
-        });
+      if (authorizeType === common.AuthorizeType.BASIC) {
+        row = yield basicAuth(authorization);
+      } else if (authorizeType === common.AuthorizeType.BEARER) {
+        row = yield bearerAuth(authorization, this.method, this.ip);
+      } else {
+        return yield unauthorized.call(this, next);
       }
+    } catch (err) {
+      // do not response error here
+      // many request do not need login
+      this.user.error = err;
+    }
 
-      req.session.name = row.name;
-      if (config.admins.hasOwnProperty(req.session.name)) {
-        req.session.isAdmin = true;
-      }
-      debug('auth pass user: %j, onlySync: %s, isAdmin: %s, headers: %j',
-        row, req.session.onlySync, req.session.isAdmin, req.headers);
-      next();
-    });
+    if (!row) {
+      debug('auth fail user: %j, headers: %j', row, this.header);
+      return yield unauthorized.call(this, next);
+    }
+
+    this.user.name = row.login;
+    this.user.isAdmin = row.site_admin;
+    this.user.scopes = row.scopes;
+    debug('auth pass user: %j, headers: %j', this.user, this.header);
+    yield next;
   };
 };
+
+function* basicAuth(authorization) {
+  authorization = authorization.split(' ')[1];
+  authorization = Buffer.from(authorization, 'base64').toString();
+
+  var pos = authorization.indexOf(':');
+  if (pos === -1) {
+    return null;
+  }
+
+  var username = authorization.slice(0, pos);
+  var password = authorization.slice(pos + 1);
+
+  return yield UserService.auth(username, password);
+}
+
+function* bearerAuth(authorization, method, ip) {
+  var token = authorization.split(' ')[1];
+  var isReadOperation = method === 'HEAD' || method === 'GET';
+  return yield TokenService.validateToken(token, {
+    isReadOperation: isReadOperation,
+    accessIp: ip,
+  });
+}
+
+function* unauthorized(next) {
+  if (!config.alwaysAuth || this.method !== 'GET') {
+    return yield next;
+  }
+  this.status = 401;
+  this.set('WWW-Authenticate', 'Basic realm="sample"');
+  if (this.accepts(['html', 'json']) === 'json') {
+    const error = '[unauthorized] login first';
+    this.body = {
+      error,
+      reason: error,
+    };
+  } else {
+    this.body = 'login first';
+  }
+}

middleware/block.js

@@ -0,0 +1,14 @@
+'use strict';
+
+module.exports = function () {
+  return function* block(next) {
+    var ua = String(this.get('user-agent')).toLowerCase();
+    if (ua === 'ruby') {
+      this.status = 403;
+      return this.body = {
+        message: 'forbidden Ruby user-agent, ip: ' + this.ip
+      };
+    }
+    yield next;
+  };
+};

middleware/editable.js

@@ -0,0 +1,29 @@
+'use strict';
+
+var packageService = require('../services/package');
+
+// admin or module's maintainer can modified the module
+module.exports = function* editable(next) {
+  var username = this.user && this.user.name;
+  var moduleName = this.params.name || this.params[0];
+  if (username && moduleName) {
+    if (this.user.isAdmin) {
+      return yield next;
+    }
+    var isMaintainer = yield packageService.isMaintainer(moduleName, username);
+    if (isMaintainer) {
+      return yield next;
+    }
+  }
+
+  this.status = 403;
+  var message = 'not authorized to modify ' + moduleName;
+  if (username) {
+    message = username + ' ' + message;
+  }
+  message = '[forbidden] ' + message;
+  this.body = {
+    error: message,
+    reason: message,
+  };
+};

middleware/exists_package.js

@@ -0,0 +1,17 @@
+'use strict';
+
+var packageService = require('../services/package');
+
+module.exports = function* (next) {
+  var name = this.params.name || this.params[0];
+  var pkg = yield packageService.getLatestModule(name);
+  if (pkg) {
+    return yield next;
+  }
+  this.status = 404;
+  const error = '[not_found] document not found';
+  this.body = {
+    error,
+    reason: error,
+  };
+};

middleware/limit.js

@@ -0,0 +1,14 @@
+'use strict';
+
+var config = require('../config');
+var limit = require('koa-limit');
+
+var limitConfig = config.limit;
+
+if (!limitConfig.enable) {
+  module.exports = function* ignoreLimit(next) {
+    yield next;
+  };
+} else {
+  module.exports = limit(limitConfig);
+}