Release 0.7.0

Maintainer logic improve

.gitignore

@@ -16,11 +16,14 @@ results
 node_modules
 npm-debug.log
 public/dist/
+.dist
 config/config.js
 backup/*.json
 backup/*.gz
 docs/web/history.md
+docs/web/_readme.md
 view/web/_layout.html
 bin/mysql.js
 bin/test.sql
 coverage/
+config/web_readme.md

.jshintrc

@@ -58,7 +58,7 @@
     "shadow"        : true,      // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
     "sub"           : false,     // true: Tolerate using `[]` notation when it can still be expressed in dot notation
     "supernew"      : false,     // true: Tolerate `new function () { ... };` and `new Object;`
-    "validthis"     : false,     // true: Tolerate using this in a non-constructor function
+    "validthis"     : true,      // true: Tolerate using this in a non-constructor function
 
     // Environments
     "browser"       : true,     // Web Browser (window, document, etc)

.npmignore

@@ -8,6 +8,8 @@ logo.png
 public/dist/
 backup/*.json
 backup/*.gz
+docs/web/history.md
+docs/web/_readme.md
 view/web/_layout.html
 bin/mysql.js
 bin/test.sql
@@ -15,3 +17,5 @@ coverage/
 .jshintrc
 .jshintignore
 .DS_Store
+config/web_readme.md
+.dist/

CONTRIBUTING.md

@@ -0,0 +1,39 @@
+# How to contribute
+
+Third-party patches are essential for keeping `cnpmjs.org` great.
+We want to keep it as easy as possible to contribute changes that
+get things working in your environment. There are a few guidelines that we
+need contributors to follow so that we can have a chance of keeping on
+top of things.
+
+## Getting Started
+
+* Make sure you have a [GitHub account](https://github.com/signup/free)
+* Fork the repository on GitHub
+
+## Making Changes
+
+* Create a topic branch from where you want to base your work.
+  * This is usually the master branch.
+  * Only target release branches if you are certain your fix must be on that
+    branch.
+  * To quickly create a topic branch based on master.
+    Please avoid working directly on the `master` branch.
+* Make commits of logical units and including unit tests.
+* Check for unnecessary whitespace with `git diff --check` before committing.
+* Make sure your commit messages are in the proper format.
+* Make sure you have added the necessary tests for your changes.
+* Run _all_ the tests to assure nothing else was accidentally broken.
+* Follow [node style guide](https://github.com/felixge/node-style-guide)
+
+## Submitting Changes
+
+* Push your changes to a topic branch in your fork of the repository.
+* Submit a pull request.
+* Make sure travis-ci test pass.
+
+# Additional Resources
+
+* [General GitHub documentation](http://help.github.com/)
+* [GitHub pull request documentation](http://help.github.com/send-pull-requests/)
+* [cnpmjs.org](http://cnpmjs.org)

History.md

@@ -1,11 +1,114 @@
 
+0.7.0 / 2014-07-07
+==================
+
+ * use module_maintainers on GET /pakcage/:name page
+ * use new module_maintainers on GET /:name
+ * admin user should never publish to other user's packages. fix #363
+ * Add a new table for module-maintainers.
+ * gravatar use https
+ * support https
+
+0.6.1 / 2014-06-18
+==================
+
+ * hot fix removeTagsByNames()
+ * fix _rev not exists
+ * sync unpublished on GET /sync/:name
+
+0.6.0 / 2014-06-16
+==================
+
+ * sync unpublished info. close #353
+ * Delete not exists versions on sync worker. #353
+
+0.5.3 / 2014-06-13
+==================
+
+  * fix sync response 204
+  * add links in History.md
+  * bump koa
+  * fix test-cov
+  * bump koa and should
+
+0.5.2 / 2014-06-04
+==================
+
+ * sync hotfix
+ * sync phantomjs downloads pkg. close [#348](https://github.com/cnpm/cnpmjs.org/issues/348)
+ * add restart, fixed [#346](https://github.com/cnpm/cnpmjs.org/issues/346)
+
+0.5.1 / 2014-05-28
+==================
+
+ * fix attack on /-/all/since?stale=update_after&startkey=2 close [#336](https://github.com/cnpm/cnpmjs.org/issues/336)
+ * bump thunkify-wrap
+ * bump koa-middlewares
+ * remove outputError
+ * bump dependencies
+ * use svg badge
+ * add package/notfound page
+ * add dist mirror link to home page
+ * fix sync listdiff and add more test cases
+
+0.5.0 / 2014-05-13
+==================
+
+ * filter /nightlies/*
+ * use koa setter instead of set()
+ * add more info on error email
+ * add sync dist to sync/index.js
+ * show dist page
+ * sync dist file and save it to database
+ * disable gzip before [#335](https://github.com/cnpm/cnpmjs.org/issues/335) has fix
+
+0.4.3 / 2014-04-18
+==================
+
+  * Merge pull request [#334](https://github.com/cnpm/cnpmjs.org/issues/334) from cnpm/fix-permission
+  * add permission check to /:name/:tag
+  * Merge pull request [#333](https://github.com/cnpm/cnpmjs.org/issues/333) from cnpm/issue332-tag
+  * fix space
+  * add put /:name/:tag, close [#332](https://github.com/cnpm/cnpmjs.org/issues/332)
+
+0.4.2 / 2014-04-17
+==================
+
+ * sync interval config
+ * fix fav ico and show pkg size on pkg info page. fix [#318](https://github.com/cnpm/cnpmjs.org/issues/318)
+ * sync work sync one done must wait for a defer.setImmediate. fix [#328](https://github.com/cnpm/cnpmjs.org/issues/328)
+ * bump dep versions
+ * if download tarball 404, throw err better than ignore it. fixed [#325](https://github.com/cnpm/cnpmjs.org/issues/325)
+ * refator sync
+ * hotfix, close [#321](https://github.com/cnpm/cnpmjs.org/issues/321)
+ * hotfix, close [#319](https://github.com/cnpm/cnpmjs.org/issues/319)
+ * support custom web home page
+ * npm get short only can read from cnpm now
+ * if using reverted proxy like nginx, only binding on local host
+ * fix redis detect logic
+
+0.4.1 / 2014-04-10
+==================
+
+ * fix sync status code error
+
+0.4.0 / 2014-04-09
+==================
+
+ * fix test cases to run on local machine
+ * add contribute guidelines
+ * use local mysql for dev env. fix [#308](https://github.com/cnpm/cnpmjs.org/issues/308)
+ * use copy to
+ * use koa-compress and koa-conditional-get
+ * maintainers is string, fix [#301](https://github.com/cnpm/cnpmjs.org/issues/301)
+
 0.3.13 / 2014-03-27
 ==================
 
  * fix npm adduser update 409 bug
  * fix multiline coverage
- * show package engines. fixed #280
+ * show package engines. fixed [#280](https://github.com/cnpm/cnpmjs.org/issues/280)
- * dont sync local package field. fix #295
+ * dont sync local package field. fix [#295](https://github.com/cnpm/cnpmjs.org/issues/295)
 
 0.3.12 / 2014-03-26
 ==================
@@ -13,53 +116,53 @@
  * fix result.successes not exist error
  * fix search list
  * add simple request for listall
- * only return package name in /-/all and /-/all/since, fixed #291
+ * only return package name in /-/all and /-/all/since, fixed [#291](https://github.com/cnpm/cnpmjs.org/issues/291)
  * refine docs foloder
- * use module gmt_modified as etag. fix #288
+ * use module gmt_modified as etag. fix [#288](https://github.com/cnpm/cnpmjs.org/issues/288)
  * fix typo, remove unused config in package.json
  * web page only list cnpm registry related info
  * use generator in qnfs
 
-0.3.11 / 2014-03-20 
+0.3.11 / 2014-03-20
 ==================
 
-  * use common.isMaintainer, fixed #283
+  * use common.isMaintainer, fixed [#283](https://github.com/cnpm/cnpmjs.org/issues/283)
   * update dependencies
-  * use co-mocha for test, fixed #279
+  * use co-mocha for test, fixed [#279](https://github.com/cnpm/cnpmjs.org/issues/279)
   * update thunkify-wrap, breaking change in thunkify-wrap
   * refactor SQLs by using multiline
   * use multiline to refactor sqls
   * ignore contributors
 
-0.3.10 / 2014-03-16 
+0.3.10 / 2014-03-16
 ==================
 
-  * Only /_session request send the authSession. fixed #223
+  * Only /_session request send the authSession. fixed [#223](https://github.com/cnpm/cnpmjs.org/issues/223)
-  * sync npm user info when maintainers and contributors not exists. fixed #82
+  * sync npm user info when maintainers and contributors not exists. fixed [#82](https://github.com/cnpm/cnpmjs.org/issues/82)
   * save npm user to mysql
   * password salt always be randoms
-  * remove session access in /name and /name/version, fixed #274
+  * remove session access in /name and /name/version, fixed [#274](https://github.com/cnpm/cnpmjs.org/issues/274)
   * fix update maintainer session error
   * update koa-middlewares
   * fix test, fix sync_by_install
   * use defer session
-  * Support npm owner|author add [name] [pkg]. fixed #271
+  * Support npm owner|author add [name] [pkg]. fixed [#271](https://github.com/cnpm/cnpmjs.org/issues/271)
 
-0.3.9 / 2014-03-14 
+0.3.9 / 2014-03-14
 ==================
 
   * custom user-agent
   * use co-urllib instead of thunkify urllib; fix mock http.request test cases
   * request limit custom message
   * add config.redis check
-  * add koa-limit, fixed #267
+  * add koa-limit, fixed [#267](https://github.com/cnpm/cnpmjs.org/issues/267)
 
-0.3.8 / 2014-03-11 
+0.3.8 / 2014-03-11
 ==================
 
-  * update middlewares, fixed missing charset bug #264
+  * update middlewares, fixed missing charset bug [#264](https://github.com/cnpm/cnpmjs.org/issues/264)
 
-0.3.7 / 2014-03-11 
+0.3.7 / 2014-03-11
 ==================
 
   * show worker die date time
@@ -69,63 +172,63 @@
   * fix return versions
   * fix makefile, remove eventproxy
   * refactor sync_module_worker
-  * add make test-dev, fixed #259
+  * add make test-dev, fixed [#259](https://github.com/cnpm/cnpmjs.org/issues/259)
   * change npm.js to generator
   * update urllib, proxy/npm.js use generator
   * sync_all and sync_exist to generator
   * change function to generator
   * need node >= v0.11.9
 
-0.3.6 / 2014-03-06 
+0.3.6 / 2014-03-06
 ==================
 
-  * install missing package should sync it from source npm. fixed #252
+  * install missing package should sync it from source npm. fixed [#252](https://github.com/cnpm/cnpmjs.org/issues/252)
   * npm publish dont contains .jshint*
   * npm test run jshint
   * Add jshint check: $ make jshint
   * use `yield* next` instead of `yield next`
   * replace dist.u.qiniudn.com with cnpmjs.org/dist
 
-0.3.5 / 2014-03-05 
+0.3.5 / 2014-03-05
 ==================
 
-  * redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed #249
+  * redirect /dist/xxx.tgz => http://dist.u.qiniudn.com/xxx.tgz fixed [#249](https://github.com/cnpm/cnpmjs.org/issues/249)
-  * redirect /name to /package/name when /name is 404. fixed #245
+  * redirect /name to /package/name when /name is 404. fixed [#245](https://github.com/cnpm/cnpmjs.org/issues/245)
-  * Add missing properies and sync missing star users. fixed #235
+  * Add missing properies and sync missing star users. fixed [#235](https://github.com/cnpm/cnpmjs.org/issues/235)
 
-0.3.4 / 2014-03-04 
+0.3.4 / 2014-03-04
 ==================
 
   * add cov
   * use istanbul run test coverage
-  * gzip support. fix #241
+  * gzip support. fix [#241](https://github.com/cnpm/cnpmjs.org/issues/241)
   * readme spelling patch (@stanzheng)
-  * default readme to null, fixed #233
+  * default readme to null, fixed [#233](https://github.com/cnpm/cnpmjs.org/issues/233)
   * remove readme in versions
 
-0.3.3 / 2014-02-28 
+0.3.3 / 2014-02-28
 ==================
 
-  * Merge pull request #232 from cnpm/host-hotfix
+  * Merge pull request [#232](https://github.com/cnpm/cnpmjs.org/issues/232) from cnpm/host-hotfix
   * get request host from request.headers
-  * Merge pull request #231 from cnpm/bug-fix
+  * Merge pull request [#231](https://github.com/cnpm/cnpmjs.org/issues/231) from cnpm/bug-fix
-  * fix deps display bug#230 and nsf.url TypeError#229
+  * fix deps display bug[#230](https://github.com/cnpm/cnpmjs.org/issues/230) and nsf.url TypeError[#229](https://github.com/cnpm/cnpmjs.org/issues/229)
 
-0.3.2 / 2014-02-28 
+0.3.2 / 2014-02-28
 ==================
 
   * update koa-sess and koa-redis
   * fix sync all test
   * remove nfs.downloadStream first, fix tmppath error
-  * fix fengmk2/giturl#1 bug
+  * fix fengmk2/giturl[#1](https://github.com/cnpm/cnpmjs.org/issues/1) bug
 
-0.3.1 / 2014-02-27 
+0.3.1 / 2014-02-27
 ==================
 
-  * add etag fixed #224
+  * add etag fixed [#224](https://github.com/cnpm/cnpmjs.org/issues/224)
   * travis ci install on source npm
 
-0.3.0 / 2014-02-27 
+0.3.0 / 2014-02-27
 ==================
 
   * fix typo and dont sync not exists pkgs
@@ -146,162 +249,162 @@
   * fix all the test of registry module.test.js
   * convert registry/module.js to koa type
   * fix auth middleware
-  * finish registry user controller koa and update mm to support thunkify. fixed #196
+  * finish registry user controller koa and update mm to support thunkify. fixed [#196](https://github.com/cnpm/cnpmjs.org/issues/196)
   * change controllers/user.js to koa
   * thunkify all proxy
   * convert all middlewares to koa type
   * change regsitry sync to koa
   * addd koa-jsonp, koa-bodyparser, fix / controller
   * first koa run registry home page /
-  * Merge pull request #212 from cnpm/fix-sync-404
+  * Merge pull request [#212](https://github.com/cnpm/cnpmjs.org/issues/212) from cnpm/fix-sync-404
   * return friendly 404 reason
-  * Merge pull request #211 from cnpm/bug-fix
+  * Merge pull request [#211](https://github.com/cnpm/cnpmjs.org/issues/211) from cnpm/bug-fix
-  * override json limit to default 10mb. fixed #209
+  * override json limit to default 10mb. fixed [#209](https://github.com/cnpm/cnpmjs.org/issues/209)
-  * fix #210 addPackageAndDist package version detect bug
+  * fix [#210](https://github.com/cnpm/cnpmjs.org/issues/210) addPackageAndDist package version detect bug
 
-0.2.27 / 2014-02-19 
+0.2.27 / 2014-02-19
 ==================
 
-  * support json result in search, fixed #189
+  * support json result in search, fixed [#189](https://github.com/cnpm/cnpmjs.org/issues/189)
 
-0.2.26 / 2014-02-19 
+0.2.26 / 2014-02-19
 ==================
 
   * npm publish also need to add deps
 
-0.2.25 / 2014-02-19 
+0.2.25 / 2014-02-19
 ==================
 
   * max handle number of package.json `dependencies` property
-  * Dependents support. fixed #190
+  * Dependents support. fixed [#190](https://github.com/cnpm/cnpmjs.org/issues/190)
 
-0.2.24 / 2014-02-13 
+0.2.24 / 2014-02-13
 ==================
 
   * fix if delete all the versions
-  * refactor remove module, fixed #186
+  * refactor remove module, fixed [#186](https://github.com/cnpm/cnpmjs.org/issues/186)
 
-0.2.23 / 2014-01-26 
+0.2.23 / 2014-01-26
 ==================
 
-  * system admin can add, publish, remove the packages. fixed #176
+  * system admin can add, publish, remove the packages. fixed [#176](https://github.com/cnpm/cnpmjs.org/issues/176)
 
-0.2.22 / 2014-01-26 
+0.2.22 / 2014-01-26
 ==================
 
-  * add keyword and search support keyword. #181
+  * add keyword and search support keyword. [#181](https://github.com/cnpm/cnpmjs.org/issues/181)
 
-0.2.21 / 2014-01-24 
+0.2.21 / 2014-01-24
 ==================
 
   * refactor code styles on package.html
   * nav-tabs e.preventDefault
-  * Show registry server error response. fixed #178
+  * Show registry server error response. fixed [#178](https://github.com/cnpm/cnpmjs.org/issues/178)
   * nav-tabs for package.html (@4simple)
 
-0.2.20 / 2014-01-23 
+0.2.20 / 2014-01-23
 ==================
 
   * hotfix sync missing dependencies and readmes
-  * fix sync readme error, fixed #174
+  * fix sync readme error, fixed [#174](https://github.com/cnpm/cnpmjs.org/issues/174)
   * add updateReadme in module
 
-0.2.19 / 2014-01-22 
+0.2.19 / 2014-01-22
 ==================
 
-  * npm install no need to check authorization header. fixed #171
+  * npm install no need to check authorization header. fixed [#171](https://github.com/cnpm/cnpmjs.org/issues/171)
 
-0.2.18 / 2014-01-20 
+0.2.18 / 2014-01-20
 ==================
 
-  * Support gitlab git url to display and click. fixed #160
+  * Support gitlab git url to display and click. fixed [#160](https://github.com/cnpm/cnpmjs.org/issues/160)
   * fix redis crash
 
-0.2.17 / 2014-01-17 
+0.2.17 / 2014-01-17
 ==================
 
   * custom logo url
   * hotfix layout bug
 
-0.2.16 / 2014-01-16 
+0.2.16 / 2014-01-16
 ==================
 
   * fix publish-time bug
 
-0.2.15 / 2014-01-16 
+0.2.15 / 2014-01-16
 ==================
 
   * add publish_time to debug
 
-0.2.14 / 2014-01-16 
+0.2.14 / 2014-01-16
 ==================
 
   * add make autod
-  * update publish_time, fixed #163
+  * update publish_time, fixed [#163](https://github.com/cnpm/cnpmjs.org/issues/163)
 
-0.2.13 / 2014-01-15 
+0.2.13 / 2014-01-15
 ==================
 
   * markdown tmpl not support footer, need to wrap on app start
 
-0.2.12 / 2014-01-15 
+0.2.12 / 2014-01-15
 ==================
 
   * add footer and npm client name customable
 
-0.2.11 / 2014-01-15 
+0.2.11 / 2014-01-15
 ==================
 
   * package page contributor link to search, default is true
 
-0.2.10 / 2014-01-14 
+0.2.10 / 2014-01-14
 ==================
 
-  * fix #155 Content-Disposition wrong.
+  * fix [#155](https://github.com/cnpm/cnpmjs.org/issues/155) Content-Disposition wrong.
 
-0.2.9 / 2014-01-14 
+0.2.9 / 2014-01-14
 ==================
 
   * support startkey=c and startkey="c"
-  * support couch db search api. fixed #153
+  * support couch db search api. fixed [#153](https://github.com/cnpm/cnpmjs.org/issues/153)
   * fix fork me image link
   * support sync by query.name
 
-0.2.8 / 2014-01-14 
+0.2.8 / 2014-01-14
 ==================
 
   * dont show err stack on test env
   * add download link for package page
 
-0.2.7 / 2014-01-13 
+0.2.7 / 2014-01-13
 ==================
 
-  * add shasum when nfs.upload and hfs.uploadBuffer, fixed #148
+  * add shasum when nfs.upload and hfs.uploadBuffer, fixed [#148](https://github.com/cnpm/cnpmjs.org/issues/148)
 
-0.2.6 / 2014-01-13 
+0.2.6 / 2014-01-13
 ==================
 
-  * support custom session store, fixed #146
+  * support custom session store, fixed [#146](https://github.com/cnpm/cnpmjs.org/issues/146)
 
-0.2.5 / 2014-01-13 
+0.2.5 / 2014-01-13
 ==================
 
   * add download timeout and unit test
   * use downloadStream() first
   * nfs download to a writeable stream.
 
-0.2.4 / 2014-01-10 
+0.2.4 / 2014-01-10
 ==================
 
-  * set main script to  index.js, fixed #142
+  * set main script to  index.js, fixed [#142](https://github.com/cnpm/cnpmjs.org/issues/142)
 
-0.2.3 / 2014-01-10 
+0.2.3 / 2014-01-10
 ==================
 
   * Dont show sync button on private package
-  * Sync package as publish with no deps. fixed #138
+  * Sync package as publish with no deps. fixed [#138](https://github.com/cnpm/cnpmjs.org/issues/138)
 
-0.2.2 / 2014-01-10 
+0.2.2 / 2014-01-10
 ==================
 
   * keep compatibility
@@ -311,23 +414,23 @@
   * new npm publish in one request, add _publish_in_cnpm
   * support unsure name ufs
   * contributors maybe a object
-  * Object #<Object> has no method 'forEach' fixed #134
+  * Object #<Object> has no method 'forEach' fixed [#134](https://github.com/cnpm/cnpmjs.org/issues/134)
-  * support custom config as a module, fixed issue #132
+  * support custom config as a module, fixed issue [#132](https://github.com/cnpm/cnpmjs.org/issues/132)
-  * support npm new publish flow. fixed #129
+  * support npm new publish flow. fixed [#129](https://github.com/cnpm/cnpmjs.org/issues/129)
   * add toString and constructor to test admin
-  * fix #119 hasOwnProperty check admin bug.
+  * fix [#119](https://github.com/cnpm/cnpmjs.org/issues/119) hasOwnProperty check admin bug.
 
-0.2.0 / 2013-12-27 
+0.2.0 / 2013-12-27
 ==================
 
   * remove to lower case
-  * fix #127 execSync and execsync.
+  * fix [#127](https://github.com/cnpm/cnpmjs.org/issues/127) execSync and execsync.
   * add contributors list on package page
   * mv blanket to config
   * sync typeerror fix #statusCode
   * add disturl
-  * fix #122 admin security bug
+  * fix [#122](https://github.com/cnpm/cnpmjs.org/issues/122) admin security bug
-  * fixed #121, let pkg 404 as success
+  * fixed [#121](https://github.com/cnpm/cnpmjs.org/issues/121), let pkg 404 as success
   * fix sql insert error
   * fix typos
 
@@ -339,78 +442,78 @@
   * make sure all packages name are lower case
   * select ids from tag
   * fix nodejsctl
-  * fix #112 missing versions and time no sync
+  * fix [#112](https://github.com/cnpm/cnpmjs.org/issues/112) missing versions and time no sync
   * remove restart command
   * fix sync missing packages error
   * fix web/readme.md, add install
-  * fix #109 pkg no times and no versions bug.
+  * fix [#109](https://github.com/cnpm/cnpmjs.org/issues/109) pkg no times and no versions bug.
 
 0.1.2 / 2013-12-19
 ==================
 
-  * fix times not exists canot sync bug. fixed #101
+  * fix times not exists canot sync bug. fixed [#101](https://github.com/cnpm/cnpmjs.org/issues/101)
   * support npm run command
-  * remove before_install and install in travis, fixed #102
+  * remove before_install and install in travis, fixed [#102](https://github.com/cnpm/cnpmjs.org/issues/102)
-  * split all sub queries, fixed #104
+  * split all sub queries, fixed [#104](https://github.com/cnpm/cnpmjs.org/issues/104)
-  * fix doc, fixed #103
+  * fix doc, fixed [#103](https://github.com/cnpm/cnpmjs.org/issues/103)
   * fix search too slow.
   * dont email sync log level info
   * only sync missing packages at first time
   * update dependencies
-  * sync all will sync all the missing packages, fixed #97
+  * sync all will sync all the missing packages, fixed [#97](https://github.com/cnpm/cnpmjs.org/issues/97)
 
 0.1.0 / 2013-12-12
 ==================
 
   * add sync title
-  * add favicon. fixed #69
+  * add favicon. fixed [#69](https://github.com/cnpm/cnpmjs.org/issues/69)
-  * refine sync page, fiexd #70
+  * refine sync page, fiexd [#70](https://github.com/cnpm/cnpmjs.org/issues/70)
   * add app version
   * add test for sync
   * refine sync page
   * registry and web all use controllers/sync.js
-  * sync from web, fixed #58
+  * sync from web, fixed [#58](https://github.com/cnpm/cnpmjs.org/issues/58)
   * saving missing descriptions
-  * add package download info. fixed #63
+  * add package download info. fixed [#63](https://github.com/cnpm/cnpmjs.org/issues/63)
   * add avatar
   * use dependecies, fixed #issue62
-  * support open search, fixed #60
+  * support open search, fixed [#60](https://github.com/cnpm/cnpmjs.org/issues/60)
-  * make sure publish_time and author is same to source npm registry. fixed #56
+  * make sure publish_time and author is same to source npm registry. fixed [#56](https://github.com/cnpm/cnpmjs.org/issues/56)
   * add test for search
   * add a simple search by mysql like
-  * fix This version of MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery. fixed #54
+  * fix This version of MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery. fixed [#54](https://github.com/cnpm/cnpmjs.org/issues/54)
   * update install doc, use nodejsctl to start
   * must add limit on list by author sql
-  * fix sql, change test to fit my local database, fixed #46
+  * fix sql, change test to fit my local database, fixed [#46](https://github.com/cnpm/cnpmjs.org/issues/46)
   * use registry.cnpmjs.org
-  * add install document and total package info on home page. fix #42
+  * add install document and total package info on home page. fix [#42](https://github.com/cnpm/cnpmjs.org/issues/42)
-  * add module_id to tag table. #46
+  * add module_id to tag table. [#46](https://github.com/cnpm/cnpmjs.org/issues/46)
-  * skip error version. fixed #43
+  * skip error version. fixed [#43](https://github.com/cnpm/cnpmjs.org/issues/43)
   * sync may make a user do not exist in database, but have modules in registry
   * add user page
   * fix set license
-  * ignore 404 on sync. fixed #39
+  * ignore 404 on sync. fixed [#39](https://github.com/cnpm/cnpmjs.org/issues/39)
   * fix module page, add test
   * update urllib to 0.5.5
   * version and tag
   * add module page
   * fix download url
   * first get tag, then try version
-  * support sync triggle by install, finish #31
+  * support sync triggle by install, finish [#31](https://github.com/cnpm/cnpmjs.org/issues/31)
   * addTag error return 500
   * just one download field
   * add download total info on home page
   * add download count
   * versions empty and also check missing tags
   * remove tags on unpublish
-  * add module tag. fix #6
+  * add module tag. fix [#6](https://github.com/cnpm/cnpmjs.org/issues/6)
   * add [done] flag to check sync done on client
-  * get sync log #29
+  * get sync log [#29](https://github.com/cnpm/cnpmjs.org/issues/29)
   * fix test in module
   * rm tmp file on down request error
   * add time for debug str
   * fix pkg not exists null bug
-  * use sync module woker to handle sync process. fixed #19
+  * use sync module woker to handle sync process. fixed [#19](https://github.com/cnpm/cnpmjs.org/issues/19)
   * if private mode enable, only admin can publish module
   * add alias in readme
   * fix sql, add sort by name
@@ -419,15 +522,15 @@
   * add npm and cnpm image
   * add registry total info on home page
   * fix mods bug in module.removeAll, change module.update => module.removeWithVersions
-  * add test, fix bug. fixed #18
+  * add test, fix bug. fixed [#18](https://github.com/cnpm/cnpmjs.org/issues/18)
   * spoort unpublish
   * add web page index readme
-  * switchable nfs #21
+  * switchable nfs [#21](https://github.com/cnpm/cnpmjs.org/issues/21)
   * change file path to match npm file path
-  * use qn cdn to store tarball file fixed #16
+  * use qn cdn to store tarball file fixed [#16](https://github.com/cnpm/cnpmjs.org/issues/16)
-  * add GET /:name/:version, fixed #3
+  * add GET /:name/:version, fixed [#3](https://github.com/cnpm/cnpmjs.org/issues/3)
   * add module controller test cases; fix next module not exists logic bug.
-  * publish module flow finish #11
+  * publish module flow finish [#11](https://github.com/cnpm/cnpmjs.org/issues/11)
   * add test for controllers/registry/user.js
   * add test for middleware/auth
   * add test for proxy/user
@@ -438,9 +541,9 @@
   * add start time
   * add home page
   * remove session controller
-  * adduser() finish fixed #5
+  * adduser() finish fixed [#5](https://github.com/cnpm/cnpmjs.org/issues/5)
   * rm app.js and routes.js
-  * Mock npm adduser server response, fixing #5
+  * Mock npm adduser server response, fixing [#5](https://github.com/cnpm/cnpmjs.org/issues/5)
   * adjust project dir, separate registry and web server
   * Init rest frame for cnpmjs.org
   * init

Makefile

@@ -1,26 +1,35 @@
 TESTS = $(shell ls -S `find test -type f -name "*.test.js" -print`)
-REPORTER = tap
+REPORTER = spec
 TIMEOUT = 30000
 MOCHA_OPTS =
+REGISTRY = --registry=http://registry.npm.taobao.org
 
 install:
-	@npm install --registry=http://r.cnpmjs.org \
+	@npm install $(REGISTRY) \
 		--disturl=http://dist.cnpmjs.org
 
 jshint: install
 	@-./node_modules/.bin/jshint ./
 
-test: install
+pretest:
+	@mysql -uroot -e 'DROP DATABASE IF EXISTS cnpmjs_test;'
+	@mysql -uroot -e 'CREATE DATABASE cnpmjs_test;'
+	@mysql -uroot 'cnpmjs_test' < ./docs/db.sql
+	@mysql -uroot 'cnpmjs_test' -e 'show tables;'
+
+test: install pretest
 	@NODE_ENV=test ./node_modules/.bin/mocha \
 		--harmony-generators \
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require co-mocha\
+		--require should-http \
+		--require co-mocha \
+		--require ./test/init.js \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 
-test-cov cov: install
+test-cov cov: install pretest
 	@NODE_ENV=test node --harmony \
 		node_modules/.bin/istanbul cover --preserve-comments \
 		./node_modules/.bin/_mocha \
@@ -28,7 +37,9 @@ test-cov cov: install
 		--reporter $(REPORTER) \
 		--timeout $(TIMEOUT) \
 		--require should \
-		--require co-mocha\
+		--require should-http \
+		--require co-mocha \
+		--require ./test/init.js \
 		$(MOCHA_OPTS) \
 		$(TESTS)
 	@./node_modules/.bin/cov coverage

README.md

@@ -1,9 +1,9 @@
 cnpmjs.org
 =======
 
-[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.png)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.png)](https://gemnasium.com/cnpm/cnpmjs.org)
+[![Build Status](https://secure.travis-ci.org/cnpm/cnpmjs.org.svg)](http://travis-ci.org/cnpm/cnpmjs.org) [![Dependency Status](https://gemnasium.com/cnpm/cnpmjs.org.svg)](https://gemnasium.com/cnpm/cnpmjs.org)
 
-[![NPM](https://nodei.co/npm/cnpmjs.org.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
+[![NPM](https://nodei.co/npm/cnpmjs.org.svg?downloads=true&stars=true)](https://nodei.co/npm/cnpmjs.org/)
 
 ![logo](https://raw.github.com/cnpm/cnpmjs.org/master/logo.png)
 
@@ -46,15 +46,26 @@ only need to change the registry in config. Even include manual synchronization
 
 ![cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=480&h=360)
 
-## Develop
+## Develop on your local machine
 
-**Notice**: need node version >=0.11.9
+### Dependencies
+
+* [node](http://nodejs.org) >=0.11.9
+* [mysql](http://dev.mysql.com/downloads/) >= 0.5.0, include `mysqld` and `mysql cli`. I test on `mysql@5.6.16`.
+
+### Start MySQL
+
+```bash
+$ nohup mysqld &
+```
+
+### Clone codes and run test
 
 ```bash
 # clone from git
-$ git clone git@github.com:cnpm/cnpmjs.org.git
+$ git clone https://github.com/cnpm/cnpmjs.org.git
 
-# install dependencise
+# install dependencies
 $ make install
 
 # test
@@ -67,17 +78,17 @@ $ make test-cov
 $ make autod
 
 # start server
-$ node --harmony-generators dispatch.js
+$ node --harmony_generators dispatch.js
 ```
 
-## Contribute
+## How to contribute
 
-* clone the project
+* Clone the project
-* checkout a new branch
+* Checkout a new branch
-* add new features or fix bugs in the new branch
+* Add new features or fix bugs in the new branch
-* make a pull request and we will review it ASAP
+* Make a pull request and we will review it ASAP
 
-Tips: make sure your code is follow the [node-style-guide](https://github.com/felixge/node-style-guide).
+Tips: make sure your code is following the [node-style-guide](https://github.com/felixge/node-style-guide).
 
 ## Authors
 

bin/nodejsctl

@@ -7,7 +7,7 @@ export NODE_ENV='production'
 ulimit -c unlimited
 
 cd `dirname $0`/..
-NODEJS='node --harmony-generators'
+NODEJS='node --harmony'
 BASE_HOME=`pwd`
 PROJECT_NAME=`basename ${BASE_HOME}`
 STDOUT_LOG=`$NODEJS -e "console.log(require('path').join(require('$BASE_HOME/config').logdir, 'nodejs_stdout.log'));\

common/qnfs.js

@@ -16,10 +16,11 @@
 
 var thunkify = require('thunkify-wrap');
 var qn = require('qn');
+var fs = require('fs');
 var config = require('../config');
 var client = qn.create(config.qn);
 
-thunkify(client, ['delete', 'uploadFile', 'upload']);
+thunkify(client, ['delete', 'uploadFile', 'upload', 'download']);
 
 exports._client = client;
 
@@ -62,6 +63,14 @@ exports.url = function (key) {
   return client.resourceURL(key);
 };
 
+exports.download = function* (key, filepath, options) {
+  var writeStream = fs.createWriteStream(filepath);
+  yield client.download(key, {
+    timeout: options.timeout,
+    writeStream: writeStream
+  });
+};
+
 exports.remove = function *(key) {
   try {
     return yield client.delete(key);

common/redis.js

@@ -17,7 +17,7 @@
 var config = require('../config');
 
 // close redis by set config.redis to `null` or `{}`
-if (!config.redis || !config.redis.host || !config.redis.port) {
+if (config.redis && config.redis.host && config.redis.port) {
 
   var redis = require('redis');
   var wrapper = require('co-redis');

config/index.js

@@ -19,6 +19,7 @@ var path = require('path');
 var fs = require('fs');
 var os = require('os');
 var mkdirp = require('mkdirp');
+var copy = require('copy-to');
 
 fs.existsSync = fs.existsSync || path.existsSync;
 var version = require('../package.json').version;
@@ -29,6 +30,7 @@ var config = {
   version: version,
   registryPort: 7001,
   webPort: 7002,
+  bindingHost: '127.0.0.1', // only binding on 127.0.0.1 for local access
   enableCluster: false,
   numCPUs: os.cpus().length,
   debug: true, // if debug
@@ -37,30 +39,34 @@ var config = {
   // mysql config
   mysqlServers: [
     {
-      host: 'keydiary.mysql.rds.aliyuncs.com', // 'db4free.net'
+      host: '127.0.0.1',
       port: 3306,
-      user: 'cnpmjs',
+      user: 'root',
-      password: 'cnpmjs123'
+      password: ''
     }
   ],
-  mysqlDatabase: 'cnpmjstest',
+  mysqlDatabase: 'cnpmjs_test',
   mysqlMaxConnections: 4,
   mysqlQueryTimeout: 5000,
 
   sessionSecret: 'cnpmjs.org test session secret',
   redis: {
-    host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
+    // host: 'pub-redis-19533.us-east-1-4.3.ec2.garantiadata.com',
-    port: 19533,
+    // port: 19533,
-    pass: 'cnpmjs_dev'
+    // pass: 'cnpmjs_dev'
   },
   jsonLimit: '10mb', // max request json body size
-  uploadDir: path.join(root, 'public', 'dist'),
+  uploadDir: path.join(root, '.dist'),
   // qiniu cdn: http://www.qiniu.com/, it free for dev.
   qn: {
-    accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
+    // accessKey: "iN7NgwM31j4-BZacMjPrOQBs34UG1maYCAQmhdCV",
-    secretKey: "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j",
+    // secretKey: "6QTOr2Jg1gcZEWDQXKOGZh5PziC2MCV5KsntT70j",
-    bucket: "qtestbucket",
+    // bucket: "qtestbucket",
-    domain: "http://qtestbucket.qiniudn.com"
+    // domain: "http://qtestbucket.qiniudn.com",
+    accessKey: "5UyUq-l6jsWqZMU6tuQ85Msehrs3Dr58G-mCZ9rE",
+    secretKey: "YaRsPKiYm4nGUt8mdz2QxeV5Q_yaUzVxagRuWTfM",
+    bucket: "qiniu-sdk-test",
+    domain: "http://qiniu-sdk-test.qiniudn.com",
   },
 
   mail: {
@@ -74,15 +80,20 @@ var config = {
     debug: false
   },
 
-  disturl: 'http://dist.u.qiniudn.com',
+  noticeSyncDistError: true,
-  logoURL: 'http://ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
+  disturl: 'http://nodejs.org/dist',
+  syncDist: false,
+  logoURL: '//ww4.sinaimg.cn/large/69c1d4acgw1ebfly5kjlij208202oglr.jpg',
   registryHost: 'r.cnpmjs.org',
+  // customReadmeFile: __dirname + '/web_readme.md',
+  customReadmeFile: '', // you can use your custom readme file instead the cnpm one
   customFooter: '', // you can add copyright and site total script html here
   npmClientName: 'cnpm', // use `${name} install package`
   packagePageContributorSearch: true, // package page contributor link to search, default is true
   sourceNpmRegistry: 'http://registry.npmjs.org',
   enablePrivate: true, // enable private mode, only admin can publish, other use just can sync package from source npm
   admins: {
+    // name: email
     fengmk2: 'fengmk2@gmail.com',
     admin: 'admin@cnpmjs.org',
     dead_horse: 'dead_horse@qq.com',
@@ -92,6 +103,7 @@ var config = {
   backupFilePrefix: '/cnpm/backup/', // backup filepath prefix
   syncModel: 'none', // 'none', 'all', 'exist'
   syncConcurrency: 1,
+  syncInterval: '10m', // sync interval, default is 10 minutes
   maxDependencies: 200, // max handle number of package.json `dependencies` property
 
   limit: {
@@ -102,16 +114,14 @@ var config = {
     whiteList: [],
     blackList: [],
     message: 'request frequency limited, any question, please contact fengmk2@gmail.com',
-  }
+  },
+  enableCompress: false, // enable gzip response or not
 };
 
 // load config/config.js, everything in config.js will cover the same key in index.js
 var customConfig = path.join(root, 'config/config.js');
 if (fs.existsSync(customConfig)) {
-  var options = require(customConfig);
+  copy(require(customConfig)).override(config);
-  for (var k in options) {
-    config[k] = options[k];
-  }
 }
 
 mkdirp.sync(config.logdir);
@@ -123,7 +133,5 @@ config.loadConfig = function (customConfig) {
   if (!customConfig) {
     return;
   }
-  for (var key in customConfig) {
+  copy(customConfig).override(config);
-    config[key] = customConfig[key];
-  }
 };

controllers/registry/module.js

@@ -18,6 +18,7 @@
 var debug = require('debug')('cnpmjs.org:controllers:registry:module');
 var path = require('path');
 var fs = require('fs');
+var util = require('util');
 var crypto = require('crypto');
 var utility = require('utility');
 var coRead = require('co-read');
@@ -36,13 +37,17 @@ var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var logger = require('../../common/logger');
 var ModuleDeps = require('../../proxy/module_deps');
 var ModuleStar = require('../../proxy/module_star');
+var ModuleUnpublished = require('../../proxy/module_unpublished');
+var packageService = require('../../services/package');
+var downloadAsReadStream = require('../utils').downloadAsReadStream;
 
 /**
  * show all version of a module
+ * GET /:name
  */
-exports.show = function *(next) {
+exports.show = function* (next) {
   var name = this.params.name;
-  var modifiedTime = yield *Module.getLastModified(name);
+  var modifiedTime = yield* Module.getLastModified(name);
   debug('show %s, last modified: %s', name, modifiedTime);
   if (modifiedTime) {
     // use modifiedTime as etag
@@ -60,16 +65,42 @@ exports.show = function *(next) {
   var r = yield [
     Module.listTags(name),
     Module.listByName(name),
-    ModuleStar.listUsers(name)
+    ModuleStar.listUsers(name),
+    packageService.listMaintainers(name),
   ];
   var tags = r[0];
   var rows = r[1];
   var users = r[2];
+  var maintainers = r[3];
+
+  debug('show %s got %d rows, %d tags, %d star users, maintainers: %j',
+    name, rows.length, tags.length, users.length, maintainers);
+
   var userMap = {};
   for (var i = 0; i < users.length; i++) {
     userMap[users[i]] = true;
   }
   users = userMap;
+
+  if (rows.length === 0) {
+    // check if unpublished
+    var unpublishedInfo = yield* ModuleUnpublished.get(name);
+    debug('show unpublished %j', unpublishedInfo);
+    if (unpublishedInfo) {
+      this.status = 404;
+      this.body = {
+        _id: name,
+        name: name,
+        time: {
+          modified: unpublishedInfo.package.time,
+          unpublished: unpublishedInfo.package,
+        },
+        _attachments: {}
+      };
+      return;
+    }
+  }
+
   // if module not exist in this registry,
   // sync the module backend and return package info from official registry
   if (rows.length === 0) {
@@ -82,8 +113,8 @@ exports.show = function *(next) {
       return;
     }
     var result = yield SyncModuleWorker.sync(name, 'sync-by-install');
-    this.status = result.ok ? 200 : result.statusCode;
     this.body = result.pkg;
+    this.status = result.ok ? 200 : (result.statusCode || 500);
     return;
   }
 
@@ -118,6 +149,10 @@ exports.show = function *(next) {
       readme = pkg.readme;
     }
     delete pkg.readme;
+    if (maintainers.length > 0) {
+      // TODO: need to use newer maintainers
+      pkg.maintainers = maintainers;
+    }
 
     if (!createdTime || t < createdTime) {
       createdTime = t;
@@ -150,6 +185,12 @@ exports.show = function *(next) {
 
   var pkg = latestMod.package;
 
+  if (tags.length === 0 && pkg.version !== 'next') {
+    // some sync error reason, will cause tags missing
+    // set latest tag at least
+    distTags.latest = pkg.version;
+  }
+
   var info = {
     _id: name,
     _rev: rev,
@@ -177,6 +218,9 @@ exports.show = function *(next) {
 
 /**
  * get the special version or tag of a module
+ *
+ * GET /:name/:version
+ * GET /:name/:tag
  */
 exports.get = function *(next) {
   var name = this.params.name;
@@ -185,10 +229,18 @@ exports.get = function *(next) {
   var method = version ? 'get' : 'getByTag';
   var queryLabel = version ? version : tag;
 
-  var mod = yield Module[method](name, queryLabel);
+  var rs = yield [
+    Module[method](name, queryLabel),
+    packageService.listMaintainers(name),
+  ];
+  var mod = rs[0];
   if (mod) {
     common.setDownloadURL(mod.package, this);
     mod.package._cnpm_publish_time = mod.publish_time;
+    var maintainers = rs[1];
+    if (maintainers.length > 0) {
+      mod.package.maintainers = maintainers;
+    }
     this.body = mod.package;
     return;
   }
@@ -217,8 +269,6 @@ exports.get = function *(next) {
 
 var _downloads = {};
 
-var DOWNLOAD_TIMEOUT = ms('10m');
-
 exports.download = function *(next) {
   var name = this.params.name;
   var filename = this.params.filename;
@@ -258,28 +308,13 @@ exports.download = function *(next) {
   _downloads[name] = (_downloads[name] || 0) + 1;
 
   if (typeof dist.size === 'number') {
-    this.set('Content-Length', dist.size);
+    this.length = dist.size;
   }
-  this.set('Content-Type', mime.lookup(dist.key));
+  this.type = mime.lookup(dist.key);
-  this.set('Content-Disposition', 'attachment; filename="' + filename + '"');
+  this.attachment = filename;
-  this.set('ETag', dist.shasum);
+  this.etag = dist.shasum;
 
-  // use download file api
+  this.body = yield* downloadAsReadStream(dist.key);
-  var tmpPath = path.join(config.uploadDir,
-    utility.randomString() + dist.key.replace(/\//g, '-'));
-  function cleanup() {
-    fs.unlink(tmpPath, utility.noop);
-  }
-  try {
-    yield nfs.download(dist.key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
-  } catch (err) {
-    cleanup();
-    this.throw(err);
-  }
-  var tarball = fs.createReadStream(tmpPath);
-  tarball.once('error', cleanup);
-  tarball.once('end', cleanup);
-  this.body = tarball;
 };
 
 setInterval(function () {
@@ -342,15 +377,18 @@ exports.upload = function *(next) {
 
   debug('%s: upload %s, file size: %d', username, this.url, length);
   var mod = yield Module.getById(id);
-  if (!mod) {
+  if (!mod || mod.name !== name) {
     debug('can not get this module');
     return yield* next;
   }
-  if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
+
+  var isMaintainer = yield* packageService.isMaintainer(name, username);
+
+  if (!isMaintainer) {
     this.status = 403;
     this.body = {
-      error: 'no_perms',
+      error: 'forbidden user',
-      reason: 'Current user can not publish this module'
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -454,11 +492,13 @@ exports.updateLatest = function *(next) {
     debug('can not get nextMod');
     return yield* next;
   }
+  // TODO: old publish flow, we will delete it in the next version
+  // so dont change the old logic
   if (!common.isMaintainer(this.user, nextMod.package.maintainers)) {
-    this.status = 401;
+    this.status = 403;
     this.body = {
-      error: 'noperms',
+      error: 'forbidden user',
-      reason: 'Current user can not publish this module'
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -538,16 +578,27 @@ exports.addPackageAndDist = function *(next) {
     tags.push([t, distTags[t]]);
   }
 
-  debug('addPackageAndDist %s:%s, attachment size: %s', name, version, attachment.length);
+  debug('%s addPackageAndDist %s:%s, attachment size: %s, maintainers: %j',
-
+    username, name, version, attachment.length, versionPackage.maintainers);
 
   var exists = yield Module.get(name, version);
   var shasum;
   if (exists) {
-    this.status = 409;
+    this.status = 403;
     this.body = {
-      error: 'conflict',
+      error: 'forbidden',
-      reason: 'Document update conflict.'
+      reason: 'cannot modify pre-existing version: ' + version
+    };
+    return;
+  }
+
+  // check maintainers
+  var isMaintainer = yield* packageService.isMaintainer(name, username);
+  if (!isMaintainer) {
+    this.status = 403;
+    this.body = {
+      error: 'forbidden user',
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -560,7 +611,8 @@ exports.addPackageAndDist = function *(next) {
     this.status = 403;
     this.body = {
       error: 'size_wrong',
-      reason: 'Attachment size ' + attachment.length + ' not match download size ' + tarballBuffer.length,
+      reason: 'Attachment size ' + attachment.length
+        + ' not match download size ' + tarballBuffer.length,
     };
     return;
   }
@@ -616,6 +668,10 @@ exports.addPackageAndDist = function *(next) {
   };
 };
 
+// old flows: NEED TO be deleted
+// 1. add()
+// 2. upload()
+// 3. updateLatest()
 exports.add = function *(next) {
   var username = this.user.name;
   var name = this.params.name;
@@ -688,69 +744,82 @@ exports.add = function *(next) {
   };
 };
 
-exports.updateOrRemove = function *(next) {
+// PUT /:name/-rev/:rev
-  debug('updateOrRemove module %s, %j', this.params.name, this.request.body);
+exports.updateOrRemove = function* (next) {
+  debug('updateOrRemove module %s, %s, %j', this.url, this.params.name, this.request.body);
   var body = this.request.body;
   if (body.versions) {
-    yield *exports.removeWithVersions.call(this, next);
+    yield* exports.removeWithVersions.call(this, next);
-  } else if (body.maintainers && body.maintainers.length > 0) {
+  } else if (body.maintainers) {
-    yield *exports.updateMaintainers.call(this, next);
+    yield* exports.updateMaintainers.call(this, next);
   } else {
-    yield *next;
+    yield* next;
   }
 };
 
-exports.updateMaintainers = function *(next) {
+exports.updateMaintainers = function* (next) {
   var name = this.params.name;
   var body = this.request.body;
   debug('updateMaintainers module %s, %j', name, body);
 
-  var latestMod = yield Module.getLatest(name);
+  var isMaintainer = yield* packageService.isMaintainer(name, this.user.name);
 
-  if (!latestMod || !latestMod.package) {
+  if (!isMaintainer && !this.user.isAdmin) {
-    return yield *next;
-  }
-  if (!common.isMaintainer(this.user, latestMod.package.maintainers)) {
     this.status = 403;
     this.body = {
-      error: 'no_perms',
+      error: 'forbidden user',
-      reason: 'Current user can not publish this module'
+      reason: this.user.name + ' not authorized to modify ' + name
     };
     return;
   }
 
-  var r = yield *Module.updateMaintainers(latestMod.id, body.maintainers);
+  var usernames = body.maintainers.map(function (user) {
+    return user.name;
+  });
+
+  if (usernames.length === 0) {
+    this.status = 403;
+    this.body = {
+      error: 'invalid operation',
+      reason: 'Can not remove all maintainers'
+    };
+    return;
+  }
+
+  var r = yield *packageService.updateMaintainers(name, usernames);
   debug('result: %j', r);
 
   this.status = 201;
   this.body = {
     ok: true,
     id: name,
-    rev: String(latestMod.id),
+    rev: this.params.rev,
   };
 };
 
-exports.removeWithVersions = function *(next) {
+exports.removeWithVersions = function* (next) {
-  debug('removeWithVersions module %s, with info %j', this.params.name, this.request.body);
+  // debug('removeWithVersions module %s, with info %j', this.params.name, this.request.body);
   var username = this.user.name;
   var name = this.params.name;
+  // left versions
   var versions = this.request.body.versions || {};
 
-  debug('removeWithVersions module %s, with versions %j', name, Object.keys(versions));
+  debug('removeWithVersions module %s, left versions %j', name, Object.keys(versions));
 
   // step1: list all the versions
   var mods = yield Module.listByName(name);
   if (!mods || !mods.length) {
-    return yield *next;
+    return yield* next;
   }
 
   // step2: check permission
-  var firstMod = mods[0];
+  var isMaintainer = yield* packageService.isMaintainer(name, username);
-  if (!common.isMaintainer(this.user, firstMod.package.maintainers) || firstMod.name !== name) {
+  // admin can delete the module
+  if (!isMaintainer && !this.user.isAdmin) {
     this.status = 403;
     this.body = {
-      error: 'no_perms',
+      error: 'forbidden user',
-      reason: 'Current user can not update this module'
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -810,11 +879,14 @@ exports.removeWithVersions = function *(next) {
   } else {
     debug('no tag need to be remove');
   }
+  // step 7: update last modified, make sure etag change
+  yield* Module.updateLastModified(name);
+
   this.status = 201;
   this.body = { ok: true };
 };
 
-exports.removeTar = function *(next) {
+exports.removeTar = function* (next) {
   debug('remove tarball with filename: %s, id: %s', this.params.filename, this.params.rev);
   var id = Number(this.params.rev);
   var filename = this.params.filename;
@@ -822,15 +894,17 @@ exports.removeTar = function *(next) {
   var username = this.user.name;
 
   var mod = yield Module.getById(id);
-  if (!mod) {
+  if (!mod || mod.name !== name) {
     return yield* next;
   }
 
-  if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
+  var isMaintainer = yield* packageService.isMaintainer(name, username);
+
+  if (!isMaintainer && !this.user.isAdmin) {
     this.status = 403;
     this.body = {
-      error: 'no_perms',
+      error: 'forbidden user',
-      reason: 'Current user can not delete this tarball'
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -842,8 +916,8 @@ exports.removeTar = function *(next) {
 
 exports.removeAll = function *(next) {
   debug('remove all the module with name: %s, id: %s', this.params.name, this.params.rev);
-  // var id = Number(this.params.rev);
   var name = this.params.name;
+  var username = this.user.name;
 
   var mods = yield Module.listByName(name);
   debug('removeAll module %s: %d', name, mods.length);
@@ -852,11 +926,13 @@ exports.removeAll = function *(next) {
     return yield* next;
   }
 
-  if (!common.isMaintainer(this.user, mod.package.maintainers) || mod.name !== name) {
+  var isMaintainer = yield* packageService.isMaintainer(name, username);
+  // admin can delete the module
+  if (!isMaintainer && !this.user.isAdmin) {
     this.status = 403;
     this.body = {
-      error: 'no_perms',
+      error: 'forbidden user',
-      reason: 'Current user can not delete this tarball'
+      reason: username + ' not authorized to modify ' + name
     };
     return;
   }
@@ -864,16 +940,24 @@ exports.removeAll = function *(next) {
   yield [Module.removeByName(name), Module.removeTags(name)];
   var keys = [];
   for (var i = 0; i < mods.length; i++) {
-    var key = urlparse(mods[i].dist_tarball).path;
+    var row = mods[i];
+    var dist = row.package.dist;
+    var key = dist.key;
+    if (!key) {
+      key = urlparse(dist.tarball).pathname;
+    }
     key && keys.push(key);
   }
-  try {
+  if (keys.length > 0) {
-    yield keys.map(function (key) {
+    try {
-      return nfs.remove(key);
+      yield keys.map(function (key) {
-    });
+        return nfs.remove(key);
-  } catch (err) {
+      });
-    // ignore error here
+    } catch (err) {
+      // ignore error here
+    }
   }
+
   this.body = { ok: true };
 };
 
@@ -938,3 +1022,60 @@ exports.listAllModuleNames = function *() {
     return m.name;
   });
 };
+
+// PUT /:name/:tag
+exports.updateTag = function* () {
+  var version = this.request.body;
+  var tag = this.params.tag;
+  var name = this.params.name;
+  debug('updateTag: %s %s to %s', name, version, tag);
+
+  if (!version) {
+    this.status = 400;
+    this.body = {
+      error: 'version_missed',
+      reason: 'version not found'
+    };
+    return;
+  }
+
+  if (!semver.valid(version)) {
+    this.status = 403;
+    var reason = util.format('setting tag %s to invalid version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error: 'forbidden',
+      reason: reason
+    };
+    return;
+  }
+
+  var mod = yield Module.get(name, version);
+  if (!mod) {
+    this.status = 403;
+    var reason = util.format('setting tag %s to unknown version: %s: %s/%s',
+      tag, version, name, tag);
+    this.body = {
+      error: 'forbidden',
+      reason: reason
+    };
+    return;
+  }
+
+  // check permission
+  var isMaintainer = yield* packageService.isMaintainer(name, this.user.name);
+  if (!isMaintainer) {
+    this.status = 403;
+    this.body = {
+      error: 'forbidden user',
+      reason: this.user.name + ' not authorized to modify ' + name
+    };
+    return;
+  }
+
+  yield Module.addTag(name, tag, version);
+  this.status = 201;
+  this.body = {
+    ok: true
+  };
+};

controllers/sync.js

@@ -39,7 +39,7 @@ exports.sync = function *() {
   var result = yield SyncModuleWorker.sync(name, username, options);
 
   // friendly 404 reason info
-  if (result.staticCache === 404) {
+  if (result.statusCode === 404) {
     this.status = 404;
     this.body = {
       ok: false,
@@ -48,7 +48,7 @@ exports.sync = function *() {
     return;
   }
   if (!result.ok) {
-    this.status = result.statusCode;
+    this.status = result.statusCode || 500;
     this.body = result.pkg;
     return;
   }

controllers/total.js

@@ -15,13 +15,12 @@
  * Module dependencies.
  */
 
-var microtime = require('microtime');
 var Total = require('../proxy/total');
 var Download = require('./download');
 var version = require('../package.json').version;
 var config = require('../config');
 
-var startTime = '' + microtime.now();
+var startTime = '' + Date.now();
 
 exports.show = function *() {
   var r = yield [Total.get(), Download.total()];

controllers/utils.js

@@ -0,0 +1,46 @@
+/**!
+ * cnpmjs.org - controllers/utils.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:controllers:utils');
+var path = require('path');
+var fs = require('fs');
+var utility = require('utility');
+var ms = require('ms');
+var nfs = require('../common/nfs');
+var config = require('../config');
+
+var DOWNLOAD_TIMEOUT = ms('10m');
+
+exports.downloadAsReadStream = function* (key) {
+  var tmpPath = path.join(config.uploadDir,
+    utility.randomString() + key.replace(/\//g, '-'));
+  function cleanup() {
+    debug('cleanup %s', tmpPath);
+    fs.unlink(tmpPath, utility.noop);
+  }
+  debug('downloadAsReadStream() %s to %s', key, tmpPath);
+  try {
+    yield nfs.download(key, tmpPath, {timeout: DOWNLOAD_TIMEOUT});
+  } catch (err) {
+    debug('downloadAsReadStream() %s to %s error: %s', key, tmpPath, err.stack);
+    cleanup();
+    throw err;
+  }
+  var tarball = fs.createReadStream(tmpPath);
+  tarball.once('error', cleanup);
+  tarball.once('end', cleanup);
+  return tarball;
+};

controllers/web/dist.js

@@ -14,10 +14,72 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:controllers:web:dist');
+var mime = require('mime');
+var Dist = require('../../proxy/dist');
 var config = require('../../config');
+var downloadAsReadStream = require('../utils').downloadAsReadStream;
 
-exports.redirect = function *(next) {
+function padding(max, current, pad) {
+  pad = pad || ' ';
+  var left = max - current;
+  var str = '';
+  for (var i = 0; i < left; i++) {
+    str += pad;
+  }
+  return str;
+}
+
+exports.list = function* (next) {
   var params = this.params;
-  var url = config.disturl + (params[0] || '/');
+  var url = params[0];
-  this.redirect(url);
+  if (!url) {
+    // GET /dist => /dist/
+    return this.redirect('/dist/');
+  }
+
+  var isDir = url[url.length - 1] === '/';
+  if (!isDir) {
+    return yield* download.call(this, next);
+  }
+
+  var items = yield* Dist.listdir(url);
+  if (url === '/') {
+    // phantomjs/
+    items.push({
+      name: 'phantomjs/',
+      date: '',
+    });
+  }
+
+  yield this.render('dist', {
+    title: 'Mirror index of ' + config.disturl + url,
+    disturl: config.disturl,
+    dirname: url,
+    items: items,
+    padding: padding
+  });
 };
+
+ function* download(next) {
+  var fullname = this.params[0];
+  var info = yield* Dist.getfile(fullname);
+  debug('download %s got %j', fullname, info);
+  if (!info || !info.url) {
+    return yield* next;
+  }
+
+  if (info.url.indexOf('http') === 0) {
+    return this.redirect(info.url);
+  }
+
+  // download it from nfs
+  if (typeof info.size === 'number') {
+    this.length = info.size;
+  }
+  this.type = mime.lookup(info.url);
+  this.attachment = info.name;
+  this.etag = info.sha1;
+
+  this.body = yield* downloadAsReadStream(info.url);
+}

controllers/web/package.js

@@ -14,6 +14,7 @@
  * Module dependencies.
  */
 
+var bytes = require('bytes');
 var giturl = require('giturl');
 var moment = require('moment');
 var eventproxy = require('eventproxy');
@@ -29,6 +30,7 @@ var Log = require('../../proxy/module_log');
 var ModuleDeps = require('../../proxy/module_deps');
 var setDownloadURL = require('../../lib/common').setDownloadURL;
 var ModuleStar = require('../../proxy/module_star');
+var packageService = require('../../services/package');
 
 exports.display = function *(next) {
   var params = this.params;
@@ -50,6 +52,7 @@ exports.display = function *(next) {
     down.total(name),
     ModuleDeps.list(name),
     ModuleStar.listUsers(name),
+    packageService.listMaintainers(name)
   ];
   var pkg = r[0];
   var download = r[1];
@@ -57,6 +60,7 @@ exports.display = function *(next) {
     return item.deps;
   });
   var users = r[3];
+  var maintainers = r[4];
 
   if (!pkg || !pkg.package) {
     return yield* next;
@@ -70,11 +74,15 @@ exports.display = function *(next) {
     pkg.readme = pkg.description || '';
   }
 
+  if (maintainers.length > 0) {
+    pkg.maintainers = maintainers;
+  }
+
   if (pkg.maintainers) {
     for (var i = 0; i < pkg.maintainers.length; i++) {
       var maintainer = pkg.maintainers[i];
       if (maintainer.email) {
-        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, false);
+        maintainer.gravatar = gravatar.url(maintainer.email, {s: '50', d: 'retro'}, true);
       }
     }
   }
@@ -87,7 +95,7 @@ exports.display = function *(next) {
     for (var i = 0; i < pkg.contributors.length; i++) {
       var contributor = pkg.contributors[i];
       if (contributor.email) {
-        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, false);
+        contributor.gravatar = gravatar.url(contributor.email, {s: '50', d: 'retro'}, true);
       }
       if (config.packagePageContributorSearch || !contributor.url) {
         contributor.url = '/~' + encodeURIComponent(contributor.name);
@@ -108,6 +116,10 @@ exports.display = function *(next) {
 
   pkg.dependents = dependents;
 
+  if (pkg.dist) {
+    pkg.dist.size = bytes(pkg.dist.size || 0);
+  }
+
   yield this.render('package', {
     title: 'Package - ' + pkg.name,
     package: pkg,

dispatch.js

@@ -24,6 +24,18 @@ var childProcess = require('child_process');
 var syncPath = path.join(__dirname, 'sync');
 
 if (config.enableCluster) {
+  forkWorker();
+  if (config.syncModel !== 'none') {
+    forkSyncer();
+  }
+} else {
+  require(workerPath);
+  if (config.syncModel !== 'none') {
+    require(syncPath);
+  }
+}
+
+function forkWorker() {
   cluster.setupMaster({
     exec: workerPath
   });
@@ -50,9 +62,16 @@ if (config.enableCluster) {
   for (var i = 0; i < config.numCPUs; i++) {
     cluster.fork();
   }
-
+}
-  childProcess.fork(syncPath);
+
-} else {
+function forkSyncer() {
-  require(workerPath);
+  var syncer = childProcess.fork(syncPath);
-  require(syncPath);
+  syncer.on('exit', function (code, signal) {
+    var err = new Error(util.format('syncer %s died (code: %s, signal: %s, stdout: %s, stderr: %s)',
+      syncer.pid, code, signal, syncer.stdout, syncer.stderr));
+    err.name = 'SyncerWorkerDiedError';
+    console.error('[%s] [master:%s] syncer exit: %s: %s',
+      Date(), process.pid, err.name, err.message);
+    setTimeout(forkSyncer, 1000);
+  });
 }

docs/db.sql

@@ -40,6 +40,16 @@ CREATE TABLE `module_star` (
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module star';
 
+CREATE TABLE `module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';
+
 CREATE TABLE `module` (
  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
  `gmt_create` datetime NOT NULL COMMENT 'create time',
@@ -87,10 +97,23 @@ CREATE TABLE `tag` (
  `version` varchar(30) NOT NULL COMMENT 'module version',
  `module_id` bigint(20) unsigned NOT NULL COMMENT 'module id',
  PRIMARY KEY (`id`),
- UNIQUE KEY `name` (`name`, `tag`)
+ UNIQUE KEY `name` (`name`, `tag`),
+ KEY `gmt_modified` (`gmt_modified`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module tag';
 -- ALTER TABLE  `tag` ADD  `module_id` BIGINT( 20 ) UNSIGNED NOT NULL;
 -- ALTER TABLE  `tag` CHANGE  `name`  `name` VARCHAR( 100 ) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT  'module name';
+-- ALTER TABLE `tag` ADD KEY `gmt_modified` (`gmt_modified`);
+
+CREATE TABLE `module_unpublished` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ `package` longtext CHARACTER SET utf8 COLLATE utf8_general_ci COMMENT 'base info: tags, time, maintainers, description, versions',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module unpublished info';
 
 CREATE TABLE `total` (
  `name` varchar(100) NOT NULL COMMENT 'total name',
@@ -137,3 +160,30 @@ CREATE TABLE `module_deps` (
  UNIQUE KEY `name_deps` (`name`,`deps`),
  KEY `name` (`name`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module deps';
+
+CREATE TABLE `dist_dir` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(200) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `name` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist dir info';
+
+CREATE TABLE `dist_file` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `gmt_modified` datetime NOT NULL COMMENT 'modified time',
+ `name` varchar(100) NOT NULL COMMENT 'user name',
+ `parent` varchar(200) NOT NULL COMMENT 'parent dir' DEFAULT '/',
+ `date` varchar(20) COMMENT '02-May-2014 01:06',
+ `size` int(10) unsigned NOT NULL COMMENT 'file size' DEFAULT '0',
+ `sha1` varchar(40) COMMENT 'sha1 hex value',
+ `url` varchar(2048),
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `fullname` (`parent`, `name`),
+ KEY `gmt_modified` (`gmt_modified`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='dist file info';

docs/new.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `module_maintainer` (
+ `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key',
+ `gmt_create` datetime NOT NULL COMMENT 'create time',
+ `user` varchar(100) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT 'user name',
+ `name` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'module name',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `user_module_name` (`user`,`name`),
+ KEY `name` (`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='module maintainers';

docs/web/readme.md

@@ -1,9 +1,13 @@
-# cnpmjs.org: Private npm registry and web for Enterprise
+# cnpmjs.org: Private npm registry and web for Company
+
+So `cnpm` is meaning: **Company npm**.
 
 ## Registry
 
 * Our public registry: [r.cnpmjs.org](http://r.cnpmjs.org), syncing from [registry.npmjs.org](http://registry.npmjs.org)
 * Current [cnpmjs.org](/) version: <span id="app-version"></span>
+* Mirror of [nodejs.org/dist](http://nodejs.org/dist): [/dist mirror](/dist)
+* Mirror of [phantomjs downloads](https://bitbucket.org/ariya/phantomjs/downloads): [phantomjs mirror](/dist/phantomjs/)
 
 <table class="downloads">
   <tbody>
@@ -166,34 +170,15 @@ $ cnpm info cnpm
 
 @see Github [Issues](https://github.com/cnpm/cnpmjs.org/issues)
 
-## Authors
+## Histories
 
 Release [History](/history).
 
-```bash
-$ git summary
-
- project  : cnpmjs.org
- repo age : 4 months ago
- commits  : 472
- active   : 167 days
- files    : 104
- authors  :
-   272  fengmk2                 57.6%
-   195  dead_horse              41.3%
-     2  4simple                 0.4%
-     2  Stanley Zheng           0.4%
-     1  Alsotang                0.2%
-```
-
 ## npm and cnpm relation
 
 ![npm&cnpm](https://docs.google.com/drawings/d/12QeQfGalqjsB77mRnf5Iq5oSXHCIUTvZTwECMonqCmw/pub?w=383&h=284)
 
 ## 捐赠 Donate
-如果您觉得 [cnpmjs.org] 对您有帮助，欢迎请作者一杯咖啡.
+如果您觉得 [cnpmjs.org](/) 对您有帮助，欢迎请作者一杯咖啡.
 
 [![Donate](https://img.alipay.com/sys/personalprod/style/mc/btn-index.png)](https://me.alipay.com/imk2)
-
- [cnpmjs.org]: http://cnpmjs.org/
- [registry.cnpmjs.org]: http://registry.cnpmjs.org/

lib/common.js

@@ -21,6 +21,7 @@ var util = require('util');
 
 exports.getTarballFilepath = function (filename) {
   // ensure download file path unique
+  // TODO: not only .tgz, and also other extname
   var name = filename.replace(/\.tgz$/, '.' + crypto.randomBytes(16).toString('hex') + '.tgz');
   return path.join(config.uploadDir, name);
 };

middleware/registry_not_found.js

@@ -17,9 +17,13 @@
 module.exports = function *notFound(next) {
   yield *next;
 
-  if (this.status) {
+  if (this.status && this.status !== 404) {
     return;
   }
+  if (this.body && this.body.name) {
+    return;
+  }
+
   this.status = 404;
   this.body = {
     error: 'not_found',

middleware/static.js

@@ -0,0 +1,29 @@
+/**!
+ * cnpmjs.org - middleware/static.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var path = require('path');
+var middlewares = require('koa-middlewares');
+var config = require('../config');
+
+var staticDir = path.join(path.dirname(__dirname), 'public');
+
+module.exports = middlewares.staticCache(staticDir, {
+  buffer: config.debug ? false : true,
+  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+  alas: {
+    '/favicon.ico': '/favicon.png'
+  }
+});

middleware/sync_by_install.js

@@ -20,21 +20,22 @@ var config = require('../config');
  * this.allowSync  -  allow sync triggle by cnpm install
  */
 
-module.exports = function *syncByInstall(next) {
+module.exports = function* syncByInstall(next) {
   if (!config.syncByInstall || !config.enablePrivate) {
     // only config.enablePrivate should enable sync on install
-    return yield *next;
+    return yield* next;
   }
   // request not by node, consider it request from web
   var ua = this.get('user-agent');
   if (!ua || ua.indexOf('node') < 0) {
-    return yield *next;
+    return yield* next;
   }
 
+  // if request with `/xxx?write=true`, meaning the read request using for write
   if (this.query.write) {
-    return yield *next;
+    return yield* next;
   }
 
   this.allowSync = true;
-  yield *next;
+  yield* next;
 };

middleware/web_not_found.js

@@ -14,20 +14,33 @@
  * Module dependencies.
  */
 
+var debug = require('debug')('cnpmjs.org:middleware:web_not_found');
+
 module.exports = function *notFound(next) {
   yield *next;
 
-  if (this.status) {
+  if (this.status && this.status !== 404) {
     return;
   }
 
   var m = /^\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  debug('%s match %j', this.url, m);
   if (m) {
     return this.redirect('/package/' + m[1]);
   }
 
+  // package not found
+  m = /\/package\/([\w\-\_\.]+)\/?$/.exec(this.url);
+  if (m) {
+    var name = m[1];
+    this.status = 404;
+    yield* this.render('404', {
+      title: 'Package - ' + name,
+      name: name
+    });
+    return;
+  }
+
   this.status = 404;
-  this.type = 'text/html';
-  this.charset = 'utf-8';
   this.body = 'Cannot GET ' + this.path;
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cnpmjs.org",
-  "version": "0.3.13",
+  "version": "0.7.0",
   "description": "Private npm registry and web for Enterprise, base on MySQL and Simple Store Service",
   "main": "index.js",
   "scripts": {
@@ -10,41 +10,44 @@
     "stop": "./bin/nodejsctl stop"
   },
   "dependencies": {
-    "co": "3.0.5",
+    "bytes": "1.0.0",
+    "cheerio": "0.17.0",
+    "co": "3.0.6",
+    "co-defer": "0.1.0",
     "co-gather": "0.0.1",
-    "co-read": "0.0.1",
+    "co-read": "0.0.2",
     "co-redis": "1.1.0",
-    "co-urllib": "0.1.3",
+    "co-urllib": "0.2.3",
     "co-write": "0.3.0",
-    "debug": "0.7.4",
+    "copy-to": "1.0.1",
-    "eventproxy": "0.3.0",
+    "debug": "1.0.2",
-    "giturl": "0.0.2",
+    "eventproxy": "0.3.1",
-    "graceful": "0.0.6",
+    "giturl": "0.0.3",
+    "graceful": "0.1.0",
     "gravatar": "1.0.6",
     "humanize-number": "0.0.2",
-    "koa": "0.5.1",
+    "koa": "0.8.1",
-    "koa-limit": "1.0.0",
+    "koa-limit": "1.0.2",
     "koa-markdown": "0.0.3",
-    "koa-middlewares": "0.0.12",
+    "koa-middlewares": "1.0.0",
     "logfilestream": "0.1.0",
     "marked": "0.3.2",
-    "microtime": "0.5.1",
     "mime": "1.2.11",
-    "mkdirp": "0.3.5",
+    "mkdirp": "0.5.0",
-    "moment": "2.5.1",
+    "moment": "2.7.0",
     "ms": "0.6.2",
-    "multiline": "0.2.0",
+    "multiline": "0.3.4",
-    "mysql": "2.1.1",
+    "mysql": "2.3.2",
-    "nodemailer": "0.6.1",
+    "nodemailer": "0.7.0",
-    "qn": "0.2.1",
+    "qn": "0.2.2",
     "ready": "0.1.1",
-    "redis": "0.10.1",
+    "redis": "0.10.3",
-    "semver": "2.2.1",
+    "semver": "2.3.1",
-    "thunkify-wrap": "0.1.1",
+    "thunkify-wrap": "0.1.2",
-    "utility": "0.1.11"
+    "utility": "0.1.14"
   },
   "devDependencies": {
-    "autod": ">=0.0.13",
+    "autod": "~0.2.0",
     "chunkstream": "0.0.1",
     "co-mocha": "0.0.2",
     "contributors": "*",
@@ -54,8 +57,9 @@
     "mm": "0.2.1",
     "mocha": "*",
     "pedding": "0.0.3",
-    "should": "3.1.3",
+    "should": "4.0.4",
-    "supertest": "0.9.2"
+    "should-http": "0.0.1",
+    "supertest": "0.13.0"
   },
   "homepage": "https://github.com/cnpm/cnpmjs.org",
   "repository": {

proxy/dist.js

@@ -0,0 +1,85 @@
+/**!
+ * cnpmjs.org - proxy/dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var path = require('path');
+var multiline = require('multiline');
+var mysql = require('../common/mysql');
+
+var SAVE_FILE_SQL = multiline(function () {;/*
+  INSERT INTO
+    dist_file(gmt_create, gmt_modified, name, parent, date, size, url, sha1)
+  VALUES
+    (now(), now(), ?, ?, ?, ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    name=VALUES(name),
+    parent=VALUES(parent),
+    date=VALUES(date),
+    size=VALUES(size),
+    url=VALUES(url),
+    sha1=VALUES(sha1);
+*/});
+
+exports.savefile = function* (info) {
+  return yield mysql.query(SAVE_FILE_SQL, [
+    info.name, info.parent, info.date, info.size, info.url, info.sha1
+  ]);
+};
+
+var SAVE_DIR_SQL = multiline(function () {;/*
+  INSERT INTO
+    dist_dir(gmt_create, gmt_modified, name, parent, date)
+  VALUES
+    (now(), now(), ?, ?, ?)
+  ON DUPLICATE KEY UPDATE
+    name=VALUES(name),
+    parent=VALUES(parent),
+    date=VALUES(date);
+*/});
+
+exports.savedir = function* (info) {
+  return yield mysql.query(SAVE_DIR_SQL, [
+    info.name, info.parent, info.date
+  ]);
+};
+
+var LIST_DIRS_SQL = multiline(function () {;/*
+  SELECT name, parent, date FROM dist_dir WHERE parent = ?;
+*/});
+var LIST_FILES_SQL = multiline(function () {;/*
+  SELECT name, parent, date, size, url, sha1
+  FROM dist_file WHERE parent = ?;
+*/});
+
+exports.listdir = function* (name) {
+  var rs = yield [
+    mysql.query(LIST_DIRS_SQL, [name]),
+    mysql.query(LIST_FILES_SQL, [name]),
+  ];
+  return rs[0].concat(rs[1]);
+};
+
+var GET_FILE_SQL = multiline(function () {;/*
+  SELECT name, parent, date, url, size, sha1 FROM dist_file WHERE parent = ? AND name = ?;
+*/});
+
+exports.getfile = function* (fullname) {
+  var name = path.basename(fullname);
+  var parent = path.dirname(fullname);
+  if (parent !== '/') {
+    parent += '/';
+  }
+  return yield mysql.queryOne(GET_FILE_SQL, [parent, name]);
+};

proxy/module.js

@@ -350,7 +350,7 @@ var DELETE_TAGS_BY_IDS_SQL = multiline(function () {;/*
   DELETE FROM
     tag
   WHERE
-    id in (?);
+    id IN (?);
 */});
 exports.removeTagsByIds = function (ids, callback) {
   mysql.query(DELETE_TAGS_BY_IDS_SQL, [ids], callback);
@@ -433,38 +433,16 @@ exports.listByName = function (name, callback) {
   });
 };
 
-var LIST_SINCE_SQLS = [];
+var LIST_SINCE_SQL = multiline(function () {;/*
-LIST_SINCE_SQLS.push(multiline(function () {;/*
-  SELECT
-    module_id
-  FROM
-    tag
-  WHERE
-    tag="latest" AND gmt_modified>?;
-*/}));
-LIST_SINCE_SQLS.push(multiline(function () {;/*
   SELECT
     distinct(name)
   FROM
-    module
+    tag
   WHERE
-    id IN (?);
+    gmt_modified > ?;
-*/}));
+*/});
 exports.listSince = function (start, callback) {
-  var ep = eventproxy.create();
+  mysql.query(LIST_SINCE_SQL, [new Date(start)], callback);
-  ep.fail(callback);
-  mysql.query(LIST_SINCE_SQLS[0], [new Date(start)], ep.done(function (rows) {
-    if (!rows || rows.length === 0) {
-      return callback(null, []);
-    }
-    ep.emit('ids', rows.map(function (r) {
-      return r.module_id;
-    }));
-  }));
-
-  ep.once('ids', function (ids) {
-    mysql.query(LIST_SINCE_SQLS[1], [ids], callback);
-  });
 };
 
 var LIST_ALL_NAME_SQL = multiline(function () {;/*
@@ -675,13 +653,6 @@ exports.search = function (word, options, callback) {
 
 thunkify(exports);
 
-exports.updateMaintainers = function *(id, maintainers) {
-  var mod = yield exports.getById(id);
-  mod.package.maintainers = maintainers;
-  var pkg = stringifyPackage(mod.package);
-  return yield mysql.query(UPDATE_PACKAGE_SQL, [pkg, id]);
-};
-
 var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
   SELECT
     id, gmt_modified
@@ -690,9 +661,23 @@ var GET_LAST_MODIFIED_MODULE_SQL = multiline(function () {;/*
   WHERE
     name=?
   ORDER BY
-    gmt_modified DESC;
+    gmt_modified DESC
+  LIMIT 1;
 */});
-exports.getLastModified = function *(name) {
+exports.getLastModified = function* (name) {
   var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
   return row && row.gmt_modified;
 };
+
+var UPDATE_LAST_MODIFIED_SQL = 'UPDATE module SET gmt_modified=now() WHERE id=?;';
+exports.updateLastModified = function* (name) {
+  var row = yield mysql.queryOne(GET_LAST_MODIFIED_MODULE_SQL, [name]);
+  if (row) {
+    yield mysql.query(UPDATE_LAST_MODIFIED_SQL, [row.id]);
+  }
+};
+
+var DELETE_TAGS_BY_NAMES_SQL = 'DELETE FROM tag WHERE name=? AND tag IN (?);';
+exports.removeTagsByNames = function* (moduleName, tagNames) {
+  return yield mysql.query(DELETE_TAGS_BY_NAMES_SQL, [moduleName, tagNames]);
+};

proxy/module_maintainer.js

@@ -0,0 +1,81 @@
+/**!
+ * cnpmjs.org - proxy/module_maintainer.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:proxy:module_maintainer');
+var mysql = require('../common/mysql');
+var Module = require('./module');
+
+var GET_MAINTANINERS_SQL = 'SELECT user FROM module_maintainer WHERE name = ?;';
+
+exports.get = function* (name) {
+  var users = yield mysql.query(GET_MAINTANINERS_SQL, [name]);
+  return users.map(function (row) {
+    return row.user;
+  });
+};
+
+var ADD_SQL = 'INSERT INTO module_maintainer(name, user, gmt_create) \
+  VALUES (?, ?, now());';
+function* add(name, username) {
+  try {
+    yield mysql.query(ADD_SQL, [name, username]);
+  } catch (err) {
+    if (err.code !== 'ER_DUP_ENTRY') {
+      throw err;
+    }
+  }
+}
+
+var REMOVE_SQL = 'DELETE FROM module_maintainer WHERE name = ? AND user IN (?);';
+function* remove(name, usernames) {
+  return yield mysql.query(REMOVE_SQL, [name, usernames]);
+}
+
+exports.update = function* (name, maintainers) {
+  // maintainers should be [name1, name2, ...] format
+  // find out the exists maintainers then remove the deletes and add the left
+  if (maintainers.length === 0) {
+    return {
+      add: [],
+      remove: []
+    };
+  }
+  var exists = yield* exports.get(name);
+  var addUsers = maintainers;
+  var removeUsers = [];
+  if (exists.length > 0) {
+    for (var i = 0; i < exists.length; i++) {
+      var username = exists[i];
+      if (addUsers.indexOf(username) === -1) {
+        removeUsers.push(username);
+      }
+    }
+  }
+  var tasks = [];
+  for (var i = 0; i < addUsers.length; i++) {
+    tasks.push(add(name, addUsers[i]));
+  }
+  yield tasks;
+  // make sure all add users success then remove users
+  if (removeUsers.length > 0) {
+    yield* remove(name, removeUsers);
+  }
+  debug('add %d users, remove %d users', addUsers.length, removeUsers.length);
+  return {
+    add: addUsers,
+    remove: removeUsers
+  };
+};

proxy/module_star.js

@@ -18,10 +18,10 @@ var mysql = require('../common/mysql');
 var multiline = require('multiline');
 
 var ADD_SQL = multiline(function () {;/*
-  INSERT iNTO
+  INSERT INTO
-    module_star(name, user)
+    module_star(name, user, gmt_create)
   VALUES
-    (?, ?);
+    (?, ?, now());
 */});
 exports.add = function *add(name, user) {
   try {

proxy/module_unpublished.js

@@ -0,0 +1,43 @@
+/**!
+ * cnpmjs.org - proxy/module_unpublished.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var multiline = require('multiline');
+var mysql = require('../common/mysql');
+
+var SAVE_SQL = multiline(function () {;/*
+  INSERT INTO
+    module_unpublished(gmt_create, gmt_modified, name, package)
+  VALUES
+    (now(), now(), ?, ?)
+  ON DUPLICATE KEY UPDATE
+    gmt_modified=now(),
+    name=VALUES(name),
+    package=VALUES(package);
+*/});
+
+exports.add = function* (name, pkg) {
+  return yield mysql.query(SAVE_SQL, [name, JSON.stringify(pkg)]);
+};
+
+var GET_SQL = 'SELECT gmt_modified, name, package FROM module_unpublished WHERE name=?;';
+
+exports.get = function* (name) {
+  var row = yield mysql.queryOne(GET_SQL, [name]);
+  if (row) {
+    row.package = JSON.parse(row.package);
+  }
+  return row;
+};

proxy/npm.js

@@ -19,17 +19,18 @@ var config = require('../config');
 
 var USER_AGENT = 'cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
-function *request(url, options) {
+function* request(url, options) {
   options = options || {};
   options.dataType = options.dataType || 'json';
   options.timeout = options.timeout || 120000;
   options.headers = {
     'user-agent': USER_AGENT
   };
-  url = config.sourceNpmRegistry + url;
+  var registry = options.registry || config.sourceNpmRegistry;
+  url = registry + url;
   var r;
   try {
-    r = yield *urllib.request(url, options);
+    r = yield* urllib.request(url, options);
   } catch (err) {
     var statusCode = err.status || -1;
     var data = err.data || '[empty]';
@@ -42,6 +43,8 @@ function *request(url, options) {
   return r;
 }
 
+exports.request = request;
+
 exports.getUser = function *(name) {
   var url = '/-/user/org.couchdb.user:' + name;
   var r = yield *request(url);
@@ -70,7 +73,8 @@ exports.getAllSince = function *(startkey) {
 
 exports.getShort = function *() {
   var r = yield *request('/-/short', {
-    timeout: 300000
+    timeout: 300000,
+    registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now.
   });
   return r.data;
 };

proxy/sync_module_worker.js

@@ -17,6 +17,7 @@
 
 var co = require('co');
 var gather = require('co-gather');
+var defer = require('co-defer');
 var thunkify = require('thunkify-wrap');
 var debug = require('debug')('cnpmjs.org:proxy:sync_module_worker');
 var EventEmitter = require('events').EventEmitter;
@@ -27,6 +28,7 @@ var crypto = require('crypto');
 var urllib = require('co-urllib');
 var utility = require('utility');
 var ms = require('ms');
+var urlparse = require('url').parse;
 var nfs = require('../common/nfs');
 var npm = require('./npm');
 var common = require('../lib/common');
@@ -36,22 +38,19 @@ var Log = require('./module_log');
 var config = require('../config');
 var ModuleStar = require('./module_star');
 var User = require('./user');
+var ModuleUnpublished = require('./module_unpublished');
 
 var USER_AGENT = 'sync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
 
 function SyncModuleWorker(options) {
   EventEmitter.call(this);
   this._logId = options.logId;
-  this.startName = options.name;
   if (!Array.isArray(options.name)) {
     options.name = [options.name];
   }
 
-  this.names = options.name;
+  this.names = options.name || [];
-  // for (var i = 0; i < this.names.length; i++) {
+  this.startName = this.names[0];
-  //   // ensure package name is lower case
-  //   this.names[i] = this.names[i].toLowerCase();
-  // }
 
   this.username = options.username;
   this.concurrency = options.concurrency || 1;
@@ -124,53 +123,89 @@ SyncModuleWorker.prototype.add = function (name) {
   this.log('    add dependencies: %s', name);
 };
 
+SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
+  if (success) {
+    this.pushSuccess(name);
+  } else {
+    this.pushFail(name);
+  }
+  delete this.syncingNames[name];
+  var that = this;
+  // relase the stack: https://github.com/cnpm/cnpmjs.org/issues/328
+  defer.setImmediate(function *() {
+    that.log('[c#%s] setImmediate after, %s done, start next...', concurrencyId, name);
+    yield *that.next(concurrencyId);
+  });
+};
+
 SyncModuleWorker.prototype.next = function *(concurrencyId) {
   var name = this.names.shift();
   if (!name) {
-    return process.nextTick(this.finish.bind(this));
+    return setImmediate(this.finish.bind(this));
   }
 
   var that = this;
   that.syncingNames[name] = true;
-  var pkg;
+  var pkg = null;
+  var status = 0;
   // get from npm
   try {
-    pkg = yield npm.get(name);
+    var result = yield npm.request('/' + name);
+    pkg = result.data;
+    status = result.status;
   } catch (err) {
     // if 404
-    if (err.res && err.res.statusCode === 404) {
+    if (!err.res || err.res.statusCode !== 404) {
-      that.pushSuccess(name);
+      var errMessage = err.name + ': ' + err.message;
-    } else {
+      that.log('[c#%s] [error] [%s] get package error: %s, status: %s',
-      that.pushFail(name);
+        concurrencyId, name, errMessage, status);
+      yield *that._doneOne(concurrencyId, name, false);
+      return;
     }
-    that.log('[error] [%s] get package error: %s', name, err.stack);
-    delete that.syncingNames[name];
-    yield *that.next(concurrencyId);
-    return;
   }
+
+  var unpublishedInfo = null;
+  if (status === 404) {
+    // check if it's unpublished
+    if (pkg.time && pkg.time.unpublished && pkg.time.unpublished.time) {
+      unpublishedInfo = pkg.time.unpublished;
+    } else {
+      pkg = null;
+    }
+  }
+
   if (!pkg) {
-    that.log('[error] [%s] get package error: package not exist', name);
+    that.log('[c#%s] [error] [%s] get package error: package not exists, status: %s',
-    delete that.syncingNames[name];
+      concurrencyId, name, status);
-    yield that.next(concurrencyId);
+    yield* that._doneOne(concurrencyId, name, true);
     return;
   }
 
-  that.log('[c#%d] [%s] start...', concurrencyId, name);
+  that.log('[c#%d] [%s] pkg status: %d, start...', concurrencyId, name, status);
+
+  if (unpublishedInfo) {
+    try {
+      yield* that._unpublished(name, unpublishedInfo);
+    } catch (err) {
+      that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
+      yield* that._doneOne(concurrencyId, name, false);
+      return;
+    }
+    return yield* that._doneOne(concurrencyId, name, true);
+  }
+
   var versions;
   try {
-    versions = yield that._sync(name, pkg);
+    versions = yield* that._sync(name, pkg);
   } catch (err) {
-    that.pushFail(name);
+    that.log('[c#%s] [error] [%s] sync error: %s', concurrencyId, name, err.stack);
-    that.log('[error] [%s] sync error: %s', name, err.stack);
+    yield* that._doneOne(concurrencyId, name, false);
-    delete that.syncingNames[name];
-    yield *that.next(concurrencyId);
     return;
   }
-  that.log('[%s] synced success, %d versions: %s',
+
-    name, versions.length, versions.join(', '));
+  that.log('[c#%d] [%s] synced success, %d versions: %s',
-  that.pushSuccess(name);
+    concurrencyId, name, versions.length, versions.join(', '));
-  delete that.syncingNames[name];
+  yield* that._doneOne(concurrencyId, name, true);
-  yield that.next(concurrencyId);
 };
 
 function *_listStarUsers(modName) {
@@ -194,7 +229,61 @@ function *_saveNpmUser(username) {
   yield User.saveNpmUser(user);
 }
 
-SyncModuleWorker.prototype._sync = function *(name, pkg) {
+SyncModuleWorker.prototype._unpublished = function* (name, unpublishedInfo) {
+  var mods = yield Module.listByName(name);
+  this.log('  [%s] start unpublished %d versions from local cnpm registry',
+    name, mods.length);
+  if (this._isLocalModule(mods)) {
+    // publish on cnpm, dont sync this version package
+    this.log('  [%s] publish on local cnpm registry, don\'t sync', name);
+    return [];
+  }
+
+  var r = yield* ModuleUnpublished.add(name, unpublishedInfo);
+  this.log('    [%s] save unpublished info: %j to row#%s',
+    name, unpublishedInfo, r.insertId);
+  if (mods.length === 0) {
+    return;
+  }
+  yield [Module.removeByName(name), Module.removeTags(name)];
+  var keys = [];
+  for (var i = 0; i < mods.length; i++) {
+    var row = mods[i];
+    var dist = row.package.dist;
+    var key = dist.key;
+    if (!key) {
+      key = urlparse(dist.tarball).pathname;
+    }
+    key && keys.push(key);
+  }
+
+  if (keys.length === 0) {
+    return;
+  }
+
+  try {
+    yield keys.map(function (key) {
+      return nfs.remove(key);
+    });
+  } catch (err) {
+    // ignore error here
+    this.log('    [%s] delete nfs files: %j error: %s: %s',
+      name, keys, err.name, err.message);
+  }
+  this.log('    [%s] delete nfs files: %j success', name, keys);
+};
+
+SyncModuleWorker.prototype._isLocalModule = function (mods) {
+  for (var i = 0; i < mods.length; i++) {
+    var r = mods[i];
+    if (r.package && r.package._publish_on_cnpm) {
+      return true;
+    }
+  }
+  return false;
+};
+
+SyncModuleWorker.prototype._sync = function* (name, pkg) {
   var username = this.username;
   var that = this;
 
@@ -208,8 +297,15 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
   var tagRows = result[1];
   var existsStarUsers = result[2];
 
+  if (that._isLocalModule(moduleRows)) {
+    // publish on cnpm, dont sync this version package
+    that.log('  [%s] publish on local cnpm registry, don\'t sync', name);
+    return [];
+  }
+
   hasModules = moduleRows.length > 0;
   var map = {};
+  var localVersionNames = [];
   for (var i = 0; i < moduleRows.length; i++) {
     var r = moduleRows[i];
     if (!r.package || !r.package.dist) {
@@ -217,12 +313,6 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
       continue;
     }
 
-    if (r.package && r.package._publish_on_cnpm) {
-      // publish on cnpm, dont sync this version package
-      that.log('  [%s] publish on local cnpm, don\'t sync', name);
-      return [];
-    }
-
     if (r.version === 'next') {
       continue;
     }
@@ -230,6 +320,7 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
       map.latest = r;
     }
     map[r.version] = r;
+    localVersionNames.push(r.version);
   }
 
   var tags = {};
@@ -255,11 +346,15 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
 
     var maintainers = p.maintainers || [];
     if (maintainers && !Array.isArray(maintainers)) {
+      // http://r.cnpmjs.org/jasmine-node
+      // TODO: "maintainers": "Martin H膫陇ger <martin.haeger@gmail.com>",
       maintainers = [maintainers];
     }
 
     maintainers.forEach(function (m) {
-      npmUsernames[m.name.toLowerCase()] = 1;
+      if (m.name) {
+        npmUsernames[m.name.toLowerCase()] = 1;
+      }
     });
   }
 
@@ -275,11 +370,8 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
 
   var times = pkg.time || {};
   pkg.versions = pkg.versions || {};
-  var versionNames = Object.keys(times);
+  var remoteVersionNames = Object.keys(pkg.versions);
-  if (versionNames.length === 0) {
+  if (remoteVersionNames.length === 0) {
-    versionNames = Object.keys(pkg.versions);
-  }
-  if (versionNames.length === 0) {
     that.log('  [%s] no times and no versions, hasModules: %s', name, hasModules);
     if (!hasModules) {
       // save a next module
@@ -313,9 +405,12 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
     }
   }
 
-  var versions = [];
+  var remoteVersionNameMap = {};
-  for (var i = 0; i < versionNames.length; i++) {
+
-    var v = versionNames[i];
+  // find out missing versions
+  for (var i = 0; i < remoteVersionNames.length; i++) {
+    var v = remoteVersionNames[i];
+    remoteVersionNameMap[v] = v;
     var exists = map[v] || {};
     var version = pkg.versions[v];
     if (!version || !version.dist || !version.dist.tarball) {
@@ -360,9 +455,19 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
         continue;
       }
     }
-    versions.push(version);
+    missingVersions.push(version);
   }
 
+  // find out deleted versions
+  var deletedVersionNames = [];
+  for (var i = 0; i < localVersionNames.length; i++) {
+    var v = localVersionNames[i];
+    if (!remoteVersionNameMap[v]) {
+      deletedVersionNames.push(v);
+    }
+  }
+
+  // find out missing tags
   var sourceTags = pkg['dist-tags'] || {};
   for (var t in sourceTags) {
     var sourceTagVersion = sourceTags[t];
@@ -370,18 +475,25 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
       missingTags.push([t, sourceTagVersion]);
     }
   }
-
+  // find out deleted tags
-  if (versions.length === 0) {
+  var deletedTags = [];
-    that.log('  [%s] all versions are exists', name);
+  for (var t in tags) {
-  } else {
+    if (!sourceTags[t]) {
-    versions.sort(function (a, b) {
+      // not in remote tags, delete it from local registry
-      return a.publish_time - b.publish_time;
+      deletedTags.push(t);
-    });
+    }
-    that.log('  [%s] %d versions need to sync', name, versions.length);
   }
 
-  missingVersions = versions;
+  if (missingVersions.length === 0) {
-  var versionNames = [];
+    that.log('  [%s] all versions are exists', name);
+  } else {
+    missingVersions.sort(function (a, b) {
+      return a.publish_time - b.publish_time;
+    });
+    that.log('  [%s] %d versions need to sync', name, missingVersions.length);
+  }
+
+  var syncedVersionNames = [];
   var syncIndex = 0;
 
   // sync missing versions
@@ -393,13 +505,27 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
     }
     try {
       var result = yield that._syncOneVersion(index, syncModule);
-      versionNames.push(syncModule.version);
+      syncedVersionNames.push(syncModule.version);
     } catch (err) {
-      that.log('    [%s:%d] error, version: %s, %s: %s',
+      that.log('    [%s:%d] sync error, version: %s, %s: %s',
         syncModule.name, index, syncModule.version, err.name, err.message);
     }
   }
 
+
+  if (deletedVersionNames.length === 0) {
+    that.log('  [%s] no versions need to deleted', name);
+  } else {
+    that.log('  [%s] %d versions: %j need to deleted',
+      name, deletedVersionNames.length, deletedVersionNames);
+
+    try {
+      yield Module.removeByNameAndVersions(name, deletedVersionNames);
+    } catch (err) {
+      that.log('    [%s] delete error, %s: %s', name, err.name, err.message);
+    }
+  }
+
   // sync missing descriptions
   function *syncDes() {
     if (missingDescriptions.length === 0) {
@@ -424,7 +550,13 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
   }
 
   // sync missing tags
-  function *syncTag() {
+  function* syncTag() {
+    if (deletedTags.length > 0) {
+      yield* Module.removeTagsByNames(name, deletedTags);
+      that.log('  [%s] deleted %d tags: %j',
+        name, deletedTags.length, deletedTags);
+    }
+
     if (missingTags.length === 0) {
       return;
     }
@@ -527,7 +659,7 @@ SyncModuleWorker.prototype._sync = function *(name, pkg) {
   }
 
   yield [syncDes(), syncTag(), syncReadme(), syncMissingStarUsers(), syncMissingUsers()];
-  return versionNames;
+  return syncedVersionNames;
 };
 
 SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePackage) {
@@ -590,12 +722,14 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     // get tarball
     var r = yield *urllib.request(downurl, options);
     var statusCode = r.status || -1;
-    if (statusCode === 404) {
+    // https://github.com/cnpm/cnpmjs.org/issues/325
-      shasum = sourcePackage.dist.shasum;
+    // if (statusCode === 404) {
-      return yield afterUpload({
+    //   shasum = sourcePackage.dist.shasum;
-        url: downurl
+    //   return yield *afterUpload({
-      });
+    //     url: downurl
-    }
+    //   });
+    // }
+
     if (statusCode !== 200) {
       var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
       err.name = 'DownloadTarballError';
@@ -612,6 +746,13 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     var end = thunkify.event(rs);
     yield end(); // after end event emit
 
+    if (dataSize === 0) {
+      var err = new Error('Download ' + downurl + ' file size is zero');
+      err.name = 'DownloadTarballSizeZeroError';
+      err.data = sourcePackage;
+      throw err;
+    }
+
     // check shasum
     shasum = shasum.digest('hex');
     if (shasum !== sourcePackage.dist.shasum) {
@@ -629,7 +770,7 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     };
     // upload to NFS
     var result = yield nfs.upload(filepath, options);
-    return yield afterUpload(result);
+    return yield *afterUpload(result);
   } finally {
     // remove tmp file whatever
     fs.unlink(filepath, utility.noop);
@@ -674,27 +815,35 @@ SyncModuleWorker.prototype._syncOneVersion = function *(versionIndex, sourcePack
     mod.package.dist = dist;
     var r = yield Module.add(mod);
 
-    that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, ' +
+    that.log('    [%s:%s] done, insertId: %s, author: %s, version: %s, '
-    'size: %d, publish_time: %j, publish on cnpm: %s',
+      + 'size: %d, publish_time: %j, publish on cnpm: %s',
-    sourcePackage.name, versionIndex,
+      sourcePackage.name, versionIndex,
-    r.id,
+      r.id,
-    author, mod.version, dataSize,
+      author, mod.version, dataSize,
-    new Date(mod.publish_time),
+      new Date(mod.publish_time),
-    that._publish);
+      that._publish);
 
     return r;
   }
 };
 
-SyncModuleWorker.sync = function *(name, username, options) {
+SyncModuleWorker.sync = function* (name, username, options) {
   options = options || {};
-  var pkg = yield npm.get(name);
+  var result = yield npm.request('/' + name);
-  if (!pkg || !pkg._rev) {
+  var pkg = result.data;
+  if (result.status === 404 &&
+      (!pkg.time || !pkg.time.unpublished || !pkg.time.unpublished.time)) {
+    pkg = null;
+  }
+
+  if (!pkg || !pkg._id) {
     return {
       ok: false,
-      pkg: pkg
+      pkg: pkg,
+      statusCode: 404
     };
   }
+
   var result = yield Log.create({name: name, username: username});
   var worker = new SyncModuleWorker({
     logId: result.id,

proxy/user.js

@@ -124,7 +124,7 @@ exports.update = function (user, callback) {
     if (err) {
       return callback(err);
     }
-    callback(null, {rev: newRev});
+    callback(null, {rev: newRev, result: data});
   });
 };
 

routes/registry.js

@@ -45,6 +45,8 @@ function routes(app) {
   app.put('/:name/sync', sync.sync);
   app.get('/:name/sync/log/:id', sync.getSyncLog);
 
+  app.put('/:name/:tag', login, mod.updateTag);
+
   // need limit by ip
   app.get('/:name/download/:filename', limit, mod.download);
 

routes/web.js

@@ -36,7 +36,7 @@ function routes(app) {
 
   app.get('/_list/search/search', pkg.rangeSearch);
 
-  app.get(/^\/dist(\/.+)?/, dist.redirect);
+  app.get(/^\/dist(\/.*)?/, dist.list);
 }
 
 module.exports = routes;

servers/registry.js

@@ -18,29 +18,30 @@
 var koa = require('koa');
 var app = module.exports = koa();
 var http = require('http');
-var microtime = require('microtime');
 var middlewares = require('koa-middlewares');
 var routes = require('../routes/registry');
 var logger = require('../common/logger');
 var config = require('../config');
 var session = require('../common/session');
 var auth = require('../middleware/auth');
+var staticCache = require('../middleware/static');
 var notFound = require('../middleware/registry_not_found');
 
-app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
-
+app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
-app.use(middlewares.rewrite('/favicon.ico', '/public/favicon.ico'));
+app.use(staticCache);
 
 app.keys = ['todokey', config.sessionSecret];
-app.outputErrors = true;
 app.proxy = true;
 app.use(session);
 app.use(middlewares.bodyParser({jsonLimit: config.jsonLimit}));
 app.use(auth());
 app.use(notFound);
 
-app.use(middlewares.gzip());
+if (config.enableCompress) {
-app.use(middlewares.fresh());
+  app.use(middlewares.compress({threshold: 150}));
+}
+app.use(middlewares.conditional());
 app.use(middlewares.etag());
 
 /**

servers/web.js

@@ -1,4 +1,4 @@
-/*!
+/**!
  * cnpmjs.org - servers/web.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
@@ -18,13 +18,13 @@
 var path = require('path');
 var http = require('http');
 var fs = require('fs');
-var microtime = require('microtime');
 var koa = require('koa');
 var middlewares = require('koa-middlewares');
 var markdown = require('koa-markdown');
 var session = require('../common/session');
 var opensearch = require('../middleware/opensearch');
 var notFound = require('../middleware/web_not_found');
+var staticCache = require('../middleware/static');
 var auth = require('../middleware/auth');
 var routes = require('../routes/web');
 var logger = require('../common/logger');
@@ -34,23 +34,23 @@ var app = koa();
 
 var rootdir = path.dirname(__dirname);
 
-app.use(middlewares.rt({headerName: 'X-ReadTime', timer: microtime}));
+app.use(middlewares.rt({headerName: 'X-ReadTime'}));
-app.use(middlewares.staticCache(path.join(__dirname, '..', 'public'), {
+app.use(middlewares.rewrite('/favicon.ico', '/favicon.png'));
-  buffer: !config.debug,
+app.use(staticCache);
-  maxAge: config.debug ? 0 : 60 * 60 * 24 * 7,
+
-  dir: path.join(rootdir, 'public')
-}));
 app.use(opensearch);
 app.keys = ['todokey', config.sessionSecret];
-app.outputErrors = true;
 app.proxy = true;
 app.use(session);
 app.use(middlewares.bodyParser());
 app.use(auth());
 app.use(notFound);
 
-app.use(middlewares.gzip());
+if (config.enableCompress) {
-app.use(middlewares.fresh());
+  app.use(middlewares.compress({threshold: 150}));
+}
+
+app.use(middlewares.conditional());
 app.use(middlewares.etag());
 
 var viewDir = path.join(rootdir, 'view', 'web');
@@ -63,20 +63,30 @@ var layout = fs.readFileSync(path.join(viewDir, 'layout.html'), 'utf8')
   .replace('{{logoURL}}', config.logoURL);
 fs.writeFileSync(layoutFile, layout);
 
+// custom web readme home page support
+var readmeFile = path.join(docDir, '_readme.md');
+var readmeContent;
+if (config.customReadmeFile) {
+  readmeContent = fs.readFileSync(config.customReadmeFile, 'utf8');
+} else {
+  readmeContent = fs.readFileSync(path.join(docDir, 'readme.md'), 'utf8');
+}
+fs.writeFileSync(readmeFile, readmeContent);
+
 app.use(markdown({
   baseUrl: '/',
   root: docDir,
   layout: layoutFile,
   titleHolder: '<%- locals.title %>',
   bodyHolder: '<%- locals.body %>',
-  indexName: 'readme'
+  indexName: '_readme'
 }));
 
 var locals = {
   config: config
 };
 
-middlewares.render(app, {
+middlewares.ejs(app, {
   root: viewDir,
   viewExt: 'html',
   layout: '_layout',

services/package.js

@@ -0,0 +1,60 @@
+/**!
+ * cnpmjs.org - services/package.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var Module = require('../proxy/module');
+var ModuleMaintainer = require('../proxy/module_maintainer');
+var User = require('../proxy/user');
+
+exports.listMaintainers = function* (name) {
+  var names = yield* ModuleMaintainer.get(name);
+  if (names.length === 0) {
+    return names;
+  }
+  var users = yield* User.listByNames(names);
+  return users.map(function (user) {
+    return {
+      name: user.name,
+      email: user.email
+    };
+  });
+};
+
+exports.updateMaintainers = function* (name, usernames) {
+  var rs = yield [
+    ModuleMaintainer.update(name, usernames),
+    Module.updateLastModified(name),
+  ];
+  return rs[0];
+};
+
+exports.isMaintainer = function* (name, username) {
+  var maintainers = yield* ModuleMaintainer.get(name);
+  if (maintainers.length === 0) {
+    // if not found maintainers, try to get from latest module package info
+    var latestMod = yield Module.getLatest(name);
+    var ms = latestMod && latestMod.package && latestMod.package.maintainers;
+    if (ms && ms.length > 0) {
+      maintainers = ms.map(function (user) {
+        return user.name;
+      });
+    }
+  }
+  if (maintainers.length === 0) {
+    // no maintainers, meaning this module is free for everyone
+    return true;
+  }
+  return maintainers.indexOf(username) >= 0;
+};

sync/index.js

@@ -14,15 +14,16 @@
  * Module dependencies.
  */
 
-var config = require('../config');
+var debug = require('debug')('cnpmjs.org:sync:index');
+var co = require('co');
 var ms = require('ms');
-var mail = require('../common/mail');
 var util = require('util');
 var utility = require('utility');
-var debug = require('debug')('cnpmjs.org:sync:index');
+var config = require('../config');
+var mail = require('../common/mail');
 var Total = require('../proxy/total');
 var logger = require('../common/logger');
-var co = require('co');
+var syncDistWorker = require('./sync_dist');
 
 var sync = null;
 
@@ -73,7 +74,34 @@ var handleSync = co(function *() {
 
 if (sync) {
   handleSync();
-  setInterval(handleSync, ms('30m'));
+  setInterval(handleSync, ms(config.syncInterval));
+}
+
+var syncingDist = false;
+var syncDist = co(function* syncDist() {
+  if (syncingDist) {
+    return;
+  }
+  syncingDist = true;
+  logger.info('Start syncing dist...');
+  try {
+    yield* syncDistWorker();
+    yield* syncDistWorker.syncPhantomjsDir();
+  } catch (err) {
+    err.message += ' (sync dist error)';
+    logger.warn('Sync dist error: %s: %s\n%s', err.name, err.message, err.stack);
+    if (config.noticeSyncDistError) {
+      sendMailToAdmin(err, null, new Date());
+    }
+  }
+  syncingDist = false;
+});
+
+if (config.syncDist) {
+  syncDist();
+  setInterval(syncDist, ms(config.syncInterval));
+} else {
+  logger.info('sync dist disable');
 }
 
 function sendMailToAdmin(err, result, syncTime) {
@@ -91,7 +119,7 @@ function sendMailToAdmin(err, result, syncTime) {
     subject = 'Sync Error';
     type = 'error';
     html = util.format('Sync packages from official registry failed.\n' +
-      'Start sync time is %s.\nError message is %s.', syncTime, err.stack);
+      'Start sync time is %s.\nError message is %s: %s\n%s.', syncTime, err.name, err.message, err.stack);
   } else if (result.fails && result.fails.length) {
     subject = 'Sync Finished But Some Packages Failed';
     type = 'warn';

sync/status.js

@@ -47,6 +47,13 @@ Status.prototype.start = function () {
   this.timer = setInterval(this.log.bind(this), 30000);
 };
 
+Status.prototype.stop = function () {
+  this.log(true);
+  clearInterval(this.timer);
+  this.timer = null;
+  this.started = false;
+};
+
 Status.init = function (options, worker) {
   var status = new Status(options);
   status.start();
@@ -65,9 +72,7 @@ Status.init = function (options, worker) {
   });
 
   worker.on('end', function () {
-    status.started = false;
+    status.stop();
-    status.log(true);
-    clearInterval(status.timer);
   });
 };
 

sync/sync_all.js

@@ -24,7 +24,6 @@ var Npm = require('../proxy/npm');
 var Total = require('../proxy/total');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var Module = require('../proxy/module');
-var co = require('co');
 var thunkify = require('thunkify-wrap');
 
 function subtract(subtracter, minuend) {

sync/sync_dist.js

@@ -0,0 +1,340 @@
+/**!
+ * cnpmjs.org - sync/sync_dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug')('cnpmjs.org:sync:sync_dist');
+var fs = require('fs');
+var urllib = require('co-urllib');
+var co = require('co');
+var bytes = require('bytes');
+var crypto = require('crypto');
+var utility = require('utility');
+var thunkify = require('thunkify-wrap');
+var cheerio = require('cheerio');
+var urlResolve = require('url').resolve;
+var common = require('../lib/common');
+var Dist = require('../proxy/dist');
+var config = require('../config');
+var nfs = require('../common/nfs');
+var logger = require('../common/logger');
+
+var disturl = config.disturl;
+var USER_AGENT = 'distsync.cnpmjs.org/' + config.version + ' ' + urllib.USER_AGENT;
+
+module.exports = sync;
+
+function* sync(name) {
+  name = name || '/';
+  yield* syncDir(name);
+}
+
+function* syncDir(fullname, info) {
+  var news = yield* sync.listdiff(fullname);
+  var files = [];
+  var dirs = [];
+
+  for (var i = 0; i < news.length; i++) {
+    var item = news[i];
+    if (item.type === 'dir') {
+      dirs.push(item);
+    } else {
+      files.push(item);
+    }
+  }
+
+  logger.info('sync remote:%s got %d new items, %d dirs, %d files to sync',
+    fullname, news.length, dirs.length, files.length);
+
+  for (var i = 0; i < files.length; i++) {
+    yield* syncFile(files[i]);
+  }
+
+  for (var i = 0; i < dirs.length; i++) {
+    var dir = dirs[i];
+    yield* syncDir(dir.parent + dir.name, dir);
+  }
+
+  if (info) {
+    logger.info('Save dir:%s %j to database', fullname, info);
+    yield* Dist.savedir(info);
+  }
+
+  logger.info('Sync %s finished, %d dirs, %d files',
+    fullname, dirs.length, files.length);
+}
+
+function* syncFile(info) {
+  var name = info.parent + info.name;
+  name = process.pid + name.replace(/\//g, '_'); // make sure no parent dir
+  var isPhantomjsURL = false;
+  var downurl = disturl + info.parent + info.name;
+  if (info.downloadURL) {
+    downurl = info.downloadURL;
+    isPhantomjsURL = true;
+  }
+  var filepath = common.getTarballFilepath(name);
+  var ws = fs.createWriteStream(filepath);
+
+  var options = {
+    writeStream: ws,
+    followRedirect: true,
+    timeout: 6000000, // 100 minutes download
+    headers: {
+      'user-agent': USER_AGENT
+    }
+  };
+
+  try {
+    logger.info('downloading %s %s to %s, isPhantomjsURL: %s',
+      bytes(info.size), downurl, filepath, isPhantomjsURL);
+    // get tarball
+    var r = yield *urllib.request(downurl, options);
+    var statusCode = r.status || -1;
+    logger.info('download %s got status %s, headers: %j',
+      downurl, statusCode, r.headers);
+    if (statusCode !== 200) {
+      var err = new Error('Download ' + downurl + ' fail, status: ' + statusCode);
+      err.name = 'DownloadDistFileError';
+      throw err;
+    }
+
+    var shasum = crypto.createHash('sha1');
+    var dataSize = 0;
+    var rs = fs.createReadStream(filepath);
+    rs.on('data', function (data) {
+      shasum.update(data);
+      dataSize += data.length;
+    });
+    var end = thunkify.event(rs);
+    yield end(); // after end event emit
+
+    if (dataSize === 0) {
+      var err = new Error('Download ' + downurl + ' file size is zero');
+      err.name = 'DownloadDistFileZeroSizeError';
+      throw err;
+    }
+
+    if (isPhantomjsURL) {
+      debug('real size: %s, expect size: %s', dataSize, info.size);
+      if (dataSize < info.size) {
+        // phantomjs download page only show `6.7 MB`
+        var err = new Error('Download ' + downurl + ' file size is '
+          + dataSize + ' not match ' + info.size);
+        err.name = 'DownloadDistFileSizeError';
+        throw err;
+      }
+      info.size = dataSize;
+    } else if (dataSize !== info.size) {
+      var err = new Error('Download ' + downurl + ' file size is '
+        + dataSize + ' not match ' + info.size);
+      err.name = 'DownloadDistFileSizeError';
+      throw err;
+    }
+
+    shasum = shasum.digest('hex');
+    var args = {
+      key: '/dist' + info.parent + info.name,
+      size: info.size,
+      shasum: shasum,
+    };
+
+    // upload to NFS
+    logger.info('uploading %s to nfs:%s', filepath, args.key);
+    var result = yield nfs.upload(filepath, args);
+    info.url = result.url || result.key;
+    info.sha1 = shasum;
+
+    logger.info('upload %s to nfs:%s with size:%d, sha1:%s',
+      args.key, info.url, info.size, info.sha1);
+  } finally {
+    // remove tmp file whatever
+    fs.unlink(filepath, utility.noop);
+  }
+
+  logger.info('Sync dist file: %j done', info);
+  yield* Dist.savefile(info);
+}
+
+// <a href="latest/">latest/</a>                                            02-May-2014 14:45                   -
+// <a href="node-v0.4.10.tar.gz">node-v0.4.10.tar.gz</a>                                26-Aug-2011 16:22            12410018
+var FILE_RE = /^<a[^>]+>([^<]+)<\/a>\s+(\d+\-\w+\-\d+ \d+\:\d+)\s+([\-\d]+)/;
+
+function* listdir(fullname) {
+  var url = disturl + fullname;
+  var result = yield* urllib.request(url, {
+    timeout: 60000,
+  });
+  debug('listdir %s got %s, %j', url, result.status, result.headers);
+  var html = result.data && result.data.toString() || '';
+  var lines = html.split('\n');
+  var items = [];
+  for (var i = 0; i < lines.length; i++) {
+    var m = FILE_RE.exec(lines[i].trim());
+    if (!m) {
+      continue;
+    }
+    var itemName = m[1].replace(/^\/+/, '');
+    if (!itemName) {
+      continue;
+    }
+
+    // filter /nightlies/*
+    if (itemName.indexOf('nightlies/') === 0) {
+      continue;
+    }
+
+    items.push({
+      name: itemName, // 'SHASUMS.txt', 'x64/'
+      date: m[2],
+      size: m[3] === '-' ? '-' : parseInt(m[3]),
+      type: m[3] === '-' ? 'dir' : 'file',
+      parent: fullname, // '/', '/v0.10.28/'
+    });
+  }
+  return items;
+}
+
+sync.listdiff = function* (fullname) {
+  var items = yield* listdir(fullname);
+  if (items.length === 0) {
+    return items;
+  }
+  var exists = yield* Dist.listdir(fullname);
+  debug('listdiff %s got %s exists items', fullname, exists.length);
+  var map = {};
+  for (var i = 0; i < exists.length; i++) {
+    var item = exists[i];
+    map[item.name] = item;
+  }
+  var news = [];
+  for (var i = 0; i < items.length; i++) {
+    var item = items[i];
+    var exist = map[item.name];
+    if (!exist || exist.date !== item.date) {
+      news.push(item);
+      continue;
+    }
+
+    if (item.size !== '-' && item.size !== exist.size) {
+      news.push(item);
+      continue;
+    }
+
+    debug('skip %s', item.name);
+  }
+  return news;
+};
+
+function* syncPhantomjsDir() {
+  var fullname = '/phantomjs/';
+  var files = yield* sync.listPhantomjsDiff(fullname);
+
+  logger.info('sync remote:%s got %d files to sync',
+    fullname, files.length);
+
+  for (var i = 0; i < files.length; i++) {
+    yield* syncFile(files[i]);
+  }
+
+  logger.info('SyncPhantomjsDir %s finished, %d files',
+    fullname, files.length);
+}
+sync.syncPhantomjsDir = syncPhantomjsDir;
+
+// <tr class="iterable-item" id="download-301626">
+//   <td class="name"><a class="execute" href="/ariya/phantomjs/downloads/phantomjs-1.9.7-windows.zip">phantomjs-1.9.7-windows.zip</a></td>
+//   <td class="size">6.7 MB</td>
+//   <td class="uploaded-by"><a href="/Vitallium">Vitallium</a></td>
+//   <td class="count">122956</td>
+//   <td class="date">
+//     <div>
+//       <time datetime="2014-01-27T18:29:53.706942" data-title="true">2014-01-27</time>
+//     </div>
+//   </td>
+//   <td class="delete">
+//
+//   </td>
+// </tr>
+
+function* listPhantomjsDir(fullname) {
+  var url = 'https://bitbucket.org/ariya/phantomjs/downloads';
+  var result = yield* urllib.request(url, {
+    timeout: 60000,
+  });
+  debug('listPhantomjsDir %s got %s, %j', url, result.status, result.headers);
+  var html = result.data && result.data.toString() || '';
+  var $ = cheerio.load(html);
+  var items = [];
+  $('tr.iterable-item').each(function (i, el) {
+    var $el = $(this);
+    var $link = $el.find('.name a');
+    var name = $link.text();
+    var downloadURL = $link.attr('href');
+    if (!name || !downloadURL || !/\.(zip|bz2|gz)$/.test(downloadURL)) {
+      return;
+    }
+    downloadURL = urlResolve(url, downloadURL);
+    var size = parseInt(bytes($el.find('.size').text().toLowerCase().replace(/\s/g, '')));
+    if (size > 1024 * 1024) {
+      size -= 1024 * 1024;
+    } else if (size > 1024) {
+      size -= 1024;
+    } else {
+      size -= 10;
+    }
+    var date = $el.find('.date time').text();
+    items.push({
+      name: name, // 'SHASUMS.txt', 'x64/'
+      date: date,
+      size: size,
+      type: 'file',
+      parent: fullname,
+      downloadURL: downloadURL,
+    });
+  });
+  return items;
+}
+sync.listPhantomjsDir = listPhantomjsDir;
+
+sync.listPhantomjsDiff = function* (fullname) {
+  var items = yield* listPhantomjsDir(fullname);
+  if (items.length === 0) {
+    return items;
+  }
+  var exists = yield* Dist.listdir(fullname);
+  debug('listdiff %s got %s exists items', fullname, exists.length);
+  var map = {};
+  for (var i = 0; i < exists.length; i++) {
+    var item = exists[i];
+    map[item.name] = item;
+  }
+  var news = [];
+  for (var i = 0; i < items.length; i++) {
+    var item = items[i];
+    var exist = map[item.name];
+    if (!exist || exist.date !== item.date) {
+      news.push(item);
+      continue;
+    }
+
+    // if (item.size !== exist.size) {
+    //   news.push(item);
+    //   continue;
+    // }
+
+    debug('skip %s', item.name);
+  }
+  return news;
+};

test/controllers/registry/module.test.js

@@ -21,25 +21,37 @@ var thunkify = require('thunkify-wrap');
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var pedding = require('pedding');
 var config = require('../../../config');
 var app = require('../../../servers/registry');
 var Module = require('../../../proxy/module');
 var Npm = require('../../../proxy/npm');
 var controller = require('../../../controllers/registry/module');
 var ModuleDeps = require('../../../proxy/module_deps');
+var SyncModuleWorker = require('../../../proxy/sync_module_worker');
 
 var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
 
 describe('controllers/registry/module.test.js', function () {
+  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
+  var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
+
   before(function (done) {
-    app.listen(0, done);
+    app.listen(0, function () {
+      // name: mk2testmodule
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      pkg.maintainers[0].name = 'cnpmjstest10';
+      pkg.versions['0.0.1'].maintainers[0].name = 'cnpmjstest10';
+      request(app)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(201, done);
+    });
   });
 
   afterEach(mm.restore);
 
-  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
-  var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
-
   describe('sync source npm package', function () {
     var logId;
     it('should put /:name/sync success', function (done) {
@@ -66,13 +78,84 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
+  describe('GET /:name unpublished', function () {
+    before(function (done) {
+      var worker = new SyncModuleWorker({
+        name: ['tnpm'],
+        username: 'fengmk2'
+      });
+
+      worker.start();
+      worker.on('end', function () {
+        var names = worker.successes.concat(worker.fails);
+        names.sort();
+        names.should.eql(['tnpm']);
+        done();
+      });
+    });
+
+    it('should show unpublished info', function (done) {
+      request(app)
+      .get('/tnpm')
+      .expect('content-type', 'application/json; charset=utf-8')
+      .expect(404, function (err, res) {
+        should.not.exist(err);
+        res.body.should.eql({
+          _id: 'tnpm',
+         name: 'tnpm',
+         time: {
+           modified: '2014-06-05T01:33:59.668Z',
+           unpublished:
+          { name: 'fengmk2',
+           time: '2014-06-05T01:33:59.668Z',
+           tags: { latest: '0.3.10' },
+           maintainers:
+           [ { name: 'fengmk2', email: 'fengmk2@gmail.com' },
+            { name: 'dead_horse', email: 'dead_horse@qq.com' } ],
+           description: 'npm client for alibaba private npm registry',
+           versions:
+           [ '0.0.1',
+            '0.0.2',
+            '0.0.3',
+            '0.0.4',
+            '0.1.0',
+            '0.1.1',
+            '0.1.2',
+            '0.1.3',
+            '0.1.4',
+            '0.1.5',
+            '0.1.8',
+            '0.1.9',
+            '0.2.0',
+            '0.2.1',
+            '0.2.2',
+            '0.2.3',
+            '0.2.4',
+            '0.3.0',
+            '0.3.1',
+            '0.3.2',
+            '0.3.3',
+            '0.3.4',
+            '0.3.5',
+            '0.3.6',
+            '0.3.7',
+            '0.3.8',
+            '0.3.9',
+            '0.3.10' ] } },
+          _attachments: {}
+        });
+        done();
+      });
+    });
+  });
+
   describe('GET /:name get module package info', function () {
     var etag;
 
     it('should return module info and etag', function (done) {
       request(app)
-      .get('/cnpmjs.org')
+      .get('/mk2testmodule')
-      .expect('content-type', 'application/json')
+      .expect('content-type', 'application/json; charset=utf-8')
       .expect(200, function (err, res) {
         should.not.exist(err);
         // should have etag
@@ -81,11 +164,17 @@ describe('controllers/registry/module.test.js', function () {
         etag.should.match(/^"\d{13}"$/);
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
-          'time', 'author', 'repository', '_attachments',
+          'time', 'author',
-          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
+          // 'repository',
-        res.body.name.should.equal('cnpmjs.org');
+          '_attachments',
+          'users',
+          // 'readmeFilename',
+          // 'homepage',
+          // 'bugs',
+          'license');
+        res.body.name.should.equal('mk2testmodule');
         res.body.versions[Object.keys(res.body.versions)[0]]
-          .dist.tarball.should.include('/cnpmjs.org/download');
+          .dist.tarball.should.containEql('/mk2testmodule/download');
         res.body.time.should.have.property('modified');
         res.body.time.modified.should.be.a.String;
         res.body.time.should.have.property('created');
@@ -98,9 +187,9 @@ describe('controllers/registry/module.test.js', function () {
 
     it('should return module info and gzip when accept-encoding=gzip', function (done) {
       request(app)
-      .get('/cnpmjs.org')
+      .get('/mk2testmodule')
       .set('accept-encoding', 'gzip')
-      .expect('content-encoding', 'gzip')
+      // .expect('content-encoding', 'gzip')
       .expect(200, function (err, res) {
         // console.log(res.headers)
         should.not.exist(err);
@@ -109,10 +198,16 @@ describe('controllers/registry/module.test.js', function () {
         etag = res.headers.etag;
         res.body.should.have.keys('_id', '_rev', 'name', 'description',
           'versions', 'dist-tags', 'readme', 'maintainers',
-          'time', 'author', 'repository', '_attachments',
+          'time', 'author',
-          'users', 'readmeFilename', 'homepage', 'bugs', 'license');
+          // 'repository',
-        res.body.name.should.equal('cnpmjs.org');
+          '_attachments',
-        res.body.versions[Object.keys(res.body.versions)[0]].dist.tarball.should.include('/cnpmjs.org/download');
+          'users',
+          // 'readmeFilename',
+          // 'homepage', 'bugs',
+          'license');
+        res.body.name.should.equal('mk2testmodule');
+        res.body.versions[Object.keys(res.body.versions)[0]]
+          .dist.tarball.should.containEql('/mk2testmodule/download');
         should.not.exist(res.headers['set-cookie']);
         done();
       });
@@ -120,7 +215,7 @@ describe('controllers/registry/module.test.js', function () {
 
     it('should 304 when etag match', function (done) {
       request(app)
-      .get('/cnpmjs.org')
+      .get('/mk2testmodule')
       .set('If-None-Match', etag)
       .expect(304, done);
     });
@@ -129,14 +224,14 @@ describe('controllers/registry/module.test.js', function () {
   describe('GET /:name/:(version|tag)', function () {
     it('should return module@version info', function (done) {
       request(app)
-      .get('/cnpmjs.org/0.3.9')
+      .get('/mk2testmodule/0.0.1')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
-        body.name.should.equal('cnpmjs.org');
+        body.name.should.equal('mk2testmodule');
         body.version.should.match(/\d+\.\d+\.\d+/);
-        body._id.should.match(/cnpmjs\.org@\d+\.\d+\.\d+/);
+        body._id.should.match(/mk2testmodule@\d+\.\d+\.\d+/);
-        body.dist.tarball.should.match(/cnpmjs\.org-\d+\.\d+\.\d+\.tgz/);
+        body.dist.tarball.should.match(/mk2testmodule\-\d+\.\d+\.\d+\.tgz/);
         body.should.have.property('_cnpm_publish_time');
         body._cnpm_publish_time.should.be.a.Number;
         body.should.have.property('_publish_on_cnpm', true);
@@ -146,36 +241,23 @@ describe('controllers/registry/module.test.js', function () {
 
     it('should return module@tag info', function (done) {
       request(app)
-      .get('/cutter/latest')
+      .get('/mk2testmodule/latest')
       .expect(200, function (err, res) {
         should.not.exist(err);
         var body = res.body;
-        body.name.should.equal('cutter');
+        body.name.should.equal('mk2testmodule');
-        body.version.should.equal('0.0.3');
+        body.version.should.equal('0.0.1');
-        body._id.should.equal('cutter@0.0.3');
+        body._id.should.equal('mk2testmodule@0.0.1');
-        body.dist.tarball.should.include('/cutter/download/cutter-0.0.3.tgz');
+        body.dist.tarball.should.containEql('/mk2testmodule/download/mk2testmodule-0.0.1.tgz');
-        done();
-      });
-    });
-
-    it('should get cnpmjs.org@latest with _publish_on_cnpm=true', function (done) {
-      request(app)
-      .get('/cnpmjs.org/latest')
-      .expect(200, function (err, res) {
-        should.not.exist(err);
-        var body = res.body;
-        body.name.should.equal('cnpmjs.org');
-        // body.version.should.equal('latest');
-        body._publish_on_cnpm.should.equal(true);
         done();
       });
     });
   });
 
-  describe('PUT /:name/-rev/id update maintainers', function () {
+  describe('PUT /:name/-rev/id updateMaintainers()', function () {
     before(function (done) {
       request(app)
-      .put('/cnpmjs.org/-rev/1')
+      .put('/mk2testmodule/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
@@ -183,29 +265,68 @@ describe('controllers/registry/module.test.js', function () {
         }]
       })
       .set('authorization', baseauth)
-      .expect('content-type', 'application/json', done);
+      .expect({"ok":true,"id":"mk2testmodule","rev":"1"}, done);
     });
 
     it('should add new maintainers', function (done) {
       request(app)
-      .put('/cnpmjs.org/-rev/1')
+      .put('/mk2testmodule/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
           email: 'cnpmjstest10@cnpmjs.org'
         }, {
-          name: 'fengmk2',
+          name: 'cnpmjstest101',
           email: 'fengmk2@cnpmjs.org'
         }]
       })
       .set('authorization', baseauth)
       .expect(201)
-      .expect('content-type', 'application/json', done);
+      .expect({
+        ok: true, id: 'mk2testmodule', rev: '1'
+      }, function (err) {
+        should.not.exist(err);
+        done = pedding(2, done);
+        // check maintainers update
+        request(app)
+        .get('/mk2testmodule')
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          var pkg = res.body;
+          pkg.maintainers.should.length(2);
+          pkg.maintainers.should.eql(pkg.versions['0.0.1'].maintainers);
+          pkg.maintainers.sort(function (a, b) {
+            return a.name > b.name ? 1 : -1;
+          });
+          pkg.maintainers.should.eql([
+            { name: 'cnpmjstest10', email: 'fengmk2@gmail.com' },
+            { name: 'cnpmjstest101', email: 'fengmk2@gmail.com' },
+          ]);
+          done();
+        });
+
+        // /pkg/0.0.1
+        request(app)
+        .get('/mk2testmodule/0.0.1')
+        .expect(200, function (err, res) {
+          should.not.exist(err);
+          var pkg = res.body;
+          pkg.maintainers.should.length(2);
+          pkg.maintainers.sort(function (a, b) {
+            return a.name > b.name ? 1 : -1;
+          });
+          pkg.maintainers.should.eql([
+            { name: 'cnpmjstest10', email: 'fengmk2@gmail.com' },
+            { name: 'cnpmjstest101', email: 'fengmk2@gmail.com' },
+          ]);
+          done();
+        });
+      });
     });
 
     it('should add again new maintainers', function (done) {
       request(app)
-      .put('/cnpmjs.org/-rev/1')
+      .put('/mk2testmodule/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
@@ -217,12 +338,12 @@ describe('controllers/registry/module.test.js', function () {
       })
       .set('authorization', baseauth)
       .expect(201)
-      .expect('content-type', 'application/json', done);
+      .expect('content-type', 'application/json; charset=utf-8', done);
     });
 
     it('should rm maintainers', function (done) {
       request(app)
-      .put('/cnpmjs.org/-rev/1')
+      .put('/mk2testmodule/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
@@ -231,12 +352,12 @@ describe('controllers/registry/module.test.js', function () {
       })
       .set('authorization', baseauth)
       .expect(201)
-      .expect('content-type', 'application/json', done);
+      .expect('content-type', 'application/json; charset=utf-8', done);
     });
 
     it('should rm again maintainers', function (done) {
       request(app)
-      .put('/cnpmjs.org/-rev/1')
+      .put('/mk2testmodule/-rev/1')
       .send({
         maintainers: [{
           name: 'cnpmjstest10',
@@ -245,11 +366,62 @@ describe('controllers/registry/module.test.js', function () {
       })
       .set('authorization', baseauth)
       .expect(201)
-      .expect('content-type', 'application/json', done);
+      .expect({
+        id: 'mk2testmodule',
+        rev: '1',
+        ok: true
+      }, done);
+    });
+
+    it('should rm all maintainers forbidden 403', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: []
+      })
+      .set('authorization', baseauth)
+      .expect(403)
+      .expect({error: 'invalid operation', reason: 'Can not remove all maintainers'})
+      .expect('content-type', 'application/json; charset=utf-8', done);
+    });
+
+    it('should 403 when not maintainer update in private mode', function (done) {
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauthOther)
+      .expect(403)
+      .expect({
+        error: 'no_perms',
+        reason: 'Private mode enable, only admin can publish this module'
+      }, done);
+    });
+
+    it('should 403 when not maintainer update in public mode', function (done) {
+      mm(config, 'enablePrivate', false);
+      request(app)
+      .put('/mk2testmodule/-rev/1')
+      .send({
+        maintainers: [{
+          name: 'cnpmjstest10',
+          email: 'cnpmjstest10@cnpmjs.org'
+        }]
+      })
+      .set('authorization', baseauthOther)
+      .expect(403)
+      .expect({
+        error: 'forbidden user',
+        reason: 'cnpmjstest101 not authorized to modify mk2testmodule'
+      }, done);
     });
   });
 
-  describe('PUT /:name', function () {
+  describe('PUT /:name old publish flow', function () {
     var pkg = {
       name: 'testputmodule',
       description: 'test put module',
@@ -303,7 +475,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, done);
     });
 
-    it('should try to add return 403 when not module user and only next module exists',
+    it.skip('should try to add return 403 when not module user and only next module exists',
     function (done) {
       mm(config, 'enablePrivate', false);
       request(app)
@@ -393,7 +565,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should upload tarball fail 404 when rev wrong', function (done) {
       var body = fs.readFileSync(path.join(fixtures, 'testputmodule-0.1.9.tgz'));
       request(app)
-      .put('/' + pkg.name + '/-/' + pkg.name + '-0.1.9.tgz/-rev/' + lastRev + '1')
+      .put('/' + pkg.name + '/-/' + pkg.name + '-0.1.9.tgz/-rev/' + '1231231')
       .set('authorization', baseauth)
       .set('content-type', 'application/octet-stream')
       .set('content-length', '' + body.length)
@@ -473,7 +645,7 @@ describe('controllers/registry/module.test.js', function () {
         should.not.exist(err);
         res.body.name.should.equal('testputmodule');
         res.body.version.should.equal('0.1.9');
-        res.body.dist.tarball.should.include('/testputmodule/download/testputmodule-0.1.9.tgz');
+        res.body.dist.tarball.should.containEql('/testputmodule/download/testputmodule-0.1.9.tgz');
         done();
       });
     });
@@ -497,16 +669,16 @@ describe('controllers/registry/module.test.js', function () {
           res.body.should.have.keys('ok', 'rev');
           res.body.ok.should.equal(true);
 
-          // upload again should 409
+          // upload again should 403
           request(app)
           .put('/' + pkg.name)
           .set('authorization', baseauth)
           .send(pkg)
-          .expect(409, function (err, res) {
+          .expect(403, function (err, res) {
             should.not.exist(err);
             res.body.should.eql({
-              error: 'conflict',
+              error: 'forbidden',
-              reason: 'Document update conflict.'
+              reason: 'cannot modify pre-existing version: 0.0.1'
             });
             done();
           });
@@ -551,6 +723,8 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .get('/-/all/since?stale=update_after&startkey=0')
       .expect(200, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.body);
         res.body.should.be.an.Object;
         res.body._updated.should.be.a.Number;
         var keys = Object.keys(res.body);
@@ -563,6 +737,8 @@ describe('controllers/registry/module.test.js', function () {
       request(app)
       .get('/-/all/since?stale=update_after&startkey=' + (Date.now() * 2))
       .expect(200, function (err, res) {
+        should.not.exist(err);
+        should.exist(res.body);
         res.body.should.be.an.Object;
         res.body._updated.should.be.a.Number;
         res.body.should.eql({
@@ -585,7 +761,7 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('PUT /:name/-rev/:rev', function () {
+  describe('PUT /:name/-rev/:rev removeWithVersions', function () {
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
     var lastRev;
@@ -623,7 +799,7 @@ describe('controllers/registry/module.test.js', function () {
       .expect(201, done);
     });
 
-    it('should remove version ok', function (done) {
+    it('should remove all version ok', function (done) {
       //do not really remove it here
       mm.empty(Module, 'removeByNameAndVersions');
       mm.empty(Module, 'removeTagsByIds');
@@ -631,8 +807,7 @@ describe('controllers/registry/module.test.js', function () {
       .put('/testputmodule/-rev/' + lastRev)
       .set('authorization', baseauth)
       .send({
-        versions: {
+        versions: {}
-        }
       })
       .expect(201, done);
     });
@@ -642,7 +817,7 @@ describe('controllers/registry/module.test.js', function () {
     it('should download a file with 302 redirect', function (done) {
       request(app)
       .get('/cutter/download/cutter-0.0.2.tgz')
-      .expect('Location', 'http://qtestbucket.qiniudn.com/cutter/-/cutter-0.0.2.tgz')
+      .expect('Location', config.qn.domain + '/cutter/-/cutter-0.0.2.tgz')
       .expect(302, done);
     });
   });
@@ -679,7 +854,67 @@ describe('controllers/registry/module.test.js', function () {
     });
   });
 
-  describe('DELETE /:name/-rev/:rev', function (done) {
+  describe('PUT /:name/:tag updateTag()', function () {
+    it('should create new tag ok', function (done) {
+      request(app)
+      .put('/mk2testmodule/newtag')
+      .set('content-type', 'application/json')
+      .set('authorization', baseauth)
+      .send('"0.0.1"')
+      .expect(201)
+      .expect({"ok":true}, done);
+    });
+
+    it('should override exist tag ok', function (done) {
+      request(app)
+      .put('/mk2testmodule/newtag')
+      .set('content-type', 'application/json')
+      .set('authorization', baseauth)
+      .send('"0.0.1"')
+      .expect(201, done);
+    });
+
+    it('should tag invalid version 403', function (done) {
+      request(app)
+      .put('/mk2testmodule/newtag')
+      .set('content-type', 'application/json')
+      .set('authorization', baseauth)
+      .send('"hello"')
+      .expect(403)
+      .expect({
+        error: 'forbidden',
+        reason: 'setting tag newtag to invalid version: hello: mk2testmodule/newtag'
+      }, done);
+    });
+
+    it('should tag not eixst version 403', function (done) {
+      request(app)
+      .put('/mk2testmodule/newtag')
+      .set('content-type', 'application/json')
+      .set('authorization', baseauth)
+      .send('"5.0.0"')
+      .expect(403)
+      .expect({
+        error: 'forbidden',
+        reason: 'setting tag newtag to unknown version: 5.0.0: mk2testmodule/newtag'
+      }, done);
+    });
+
+    it('should not maintainer tag return no permission 403', function (done) {
+      request(app)
+      .put('/mk2testmodule/newtag')
+      .set('content-type', 'application/json')
+      .set('authorization', baseauthOther)
+      .send('"0.0.1"')
+      .expect(403)
+      .expect({
+        error: 'forbidden user',
+        reason: 'cnpmjstest101 not authorized to modify mk2testmodule'
+      }, done);
+    });
+  });
+
+  describe('DELETE /:name/-rev/:rev', function () {
     var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
     var baseauthOther = 'Basic ' + new Buffer('cnpmjstest101:cnpmjstest101').toString('base64');
     var lastRev;

test/controllers/registry/user.test.js

@@ -35,17 +35,17 @@ describe('controllers/registry/user.test.js', function () {
   describe('GET /-/user/org.couchdb.user:name', function () {
     it('should return user info', function (done) {
       request(app)
-      .get('/-/user/org.couchdb.user:cnpmjstest1')
+      .get('/-/user/org.couchdb.user:cnpmjstest10')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('_id', '_rev', 'name', 'email', 'type',
           '_cnpm_meta', 'roles', 'date');
-        res.body.name.should.equal('cnpmjstest1');
+        res.body.name.should.equal('cnpmjstest10');
         done();
       });
     });
 
-    it('should return npm user info', function (done) {
+    it.skip('should return npm user info', function (done) {
       request(app)
       .get('/-/user/org.couchdb.user:fengmk2')
       .expect(200, function (err, res) {
@@ -167,7 +167,7 @@ describe('controllers/registry/user.test.js', function () {
       }, function (err, res) {
         should.not.exist(err);
         should.exist(res.headers['set-cookie']);
-        res.headers['set-cookie'].join(';').should.include('AuthSession=');
+        res.headers['set-cookie'].join(';').should.containEql('AuthSession=');
         done();
       });
     });
@@ -200,7 +200,7 @@ describe('controllers/registry/user.test.js', function () {
       .expect(500, done);
     });
 
-    it('should 409 when req.body.rev error', function (done) {
+    it('should 201 when req.body.rev error', function (done) {
       request(app)
       .put('/-/user/org.couchdb.user:cnpmjstest10/-rev/:1-123')
       .set('authorization', 'basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64'))
@@ -210,7 +210,7 @@ describe('controllers/registry/user.test.js', function () {
         email: 'cnpmjstest10@cnpmjs.org',
         rev: '1-123'
       })
-      .expect(409, done);
+      .expect(201, done);
     });
 
     it('should 201 update ok', function (done) {

test/controllers/web/dist.test.js

@@ -16,22 +16,126 @@
 
 var should = require('should');
 var request = require('supertest');
+var pedding = require('pedding');
+var mm = require('mm');
 var app = require('../../../servers/web');
+var Dist = require('../../../proxy/dist');
 
 describe('controllers/web/dist.test.js', function () {
   before(function (done) {
     app.listen(0, done);
   });
+
   after(function (done) {
     app.close(done);
   });
 
-  describe('GET /dist', function (done) {
+  afterEach(mm.restore);
-    it('should 302 to config.disturl', function (done) {
+
+  describe('GET /dist/*', function (done) {
+    it('should GET /dist redirect to /dist/', function (done) {
       request(app)
       .get('/dist')
-      .expect('Location', 'http://dist.u.qiniudn.com/')
+      .expect(302)
-      .expect(302, done);
+      .expect('Location', '/dist/', done);
+    });
+
+    it('should GET /dist/ show file list', function (done) {
+      request(app)
+      .get('/dist/')
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
+        done();
+      });
+    });
+
+    it('should mock return files and dirs', function (done) {
+      mm(Dist, 'listdir', function* () {
+        return [
+          {name: 'ooxx/', date: '02-May-2014 00:54'},
+          {name: 'foo.txt', size: 1024, date: '02-May-2014 00:54'},
+        ];
+      });
+      request(app)
+      .get('/dist/v1.0.0/')
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/v1.0.0/</title>');
+        res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/v1.0.0/">http://nodejs.org/dist/v1.0.0/</a></h1>');
+        res.text.should.containEql('<a href="ooxx/">ooxx/</a>                                             02-May-2014 00:54                   -\n');
+        res.text.should.containEql('<a href="foo.txt">foo.txt</a>                                           02-May-2014 00:54                1024\n');
+        done();
+      });
+    });
+
+    it('should list files and dirs', function (done) {
+      mm(Dist, 'listdir', function* () {
+        return [
+          {name: 'npm/', date: '02-May-2014 00:54'},
+          {name: 'npm-versions.txt', size: 1676, date: '02-May-2014 00:54'},
+        ];
+      });
+      request(app)
+      .get('/dist/')
+      .expect('Content-Type', 'text/html; charset=utf-8')
+      .expect(200, function (err, res) {
+        should.not.exist(err);
+        res.text.should.containEql('<title>Mirror index of http://nodejs.org/dist/</title>');
+        res.text.should.containEql('<h1>Mirror index of <a target="_blank" href="http://nodejs.org/dist/">http://nodejs.org/dist/</a></h1>');
+        res.text.should.containEql('<a href="npm/">npm/</a>                                              02-May-2014 00:54                   -\n');
+        res.text.should.containEql('<a href="npm-versions.txt">npm-versions.txt</a>                                  02-May-2014 00:54                1676\n');
+        done();
+      });
+    });
+  });
+
+  describe('GET /dist/ files', function () {
+    it('should redirect to nfs url', function (done) {
+      mm(Dist, 'getfile', function* () {
+        return {
+          name: 'foo.txt', size: 1024, date: '02-May-2014 00:54',
+          url: 'http://mock.com/dist/v0.10.28/SHASUMS.txt'
+        };
+      });
+
+      request(app)
+      .get('/dist/v0.10.28/SHASUMS.txt')
+      .expect(302)
+      .expect('Location', 'http://mock.com/dist/v0.10.28/SHASUMS.txt', done);
+    });
+
+    it('should GET /dist/npm-versions.txt redirect to nfs url', function (done) {
+      mm(Dist, 'getfile', function* (fullname) {
+        fullname.should.equal('/npm-versions.txt');
+        return {
+          name: 'npm-versions.txt', size: 1024, date: '02-May-2014 00:54',
+          url: 'http://mock.com/dist/npm-versions.txt'
+        };
+      });
+
+      request(app)
+      .get('/dist/npm-versions.txt')
+      .expect(302)
+      .expect('Location', 'http://mock.com/dist/npm-versions.txt', done);
+    });
+
+    it('should download nfs file and send it', function (done) {
+      mm(Dist, 'getfile', function* () {
+        return {
+          name: 'foo.txt',
+          size: 1264,
+          date: '02-May-2014 00:54',
+          url: '/dist/v0.10.28/SHASUMS.txt'
+        };
+      });
+
+      request(app)
+      .get('/dist/v0.10.28/SHASUMS.txt')
+      .expect(200)
+      .expect(/6eff580cc8460741155d42ef1ef537961194443f/, done);
     });
   });
 });

test/controllers/web/package.test.js

@@ -17,14 +17,30 @@
 var should = require('should');
 var request = require('supertest');
 var mm = require('mm');
+var path = require('path');
 var mysql = require('../../../common/mysql');
 var app = require('../../../servers/web');
+var registry = require('../../../servers/registry');
 var pkg = require('../../../controllers/web/package');
 
+var fixtures = path.join(path.dirname(path.dirname(__dirname)), 'fixtures');
+
 describe('controllers/web/package.test.js', function () {
+  var baseauth = 'Basic ' + new Buffer('cnpmjstest10:cnpmjstest10').toString('base64');
+
   before(function (done) {
-    app.listen(0, done);
+    registry.listen(0, function () {
+      var pkg = require(path.join(fixtures, 'package_and_tgz.json'));
+      request(registry)
+      .put('/' + pkg.name)
+      .set('authorization', baseauth)
+      .send(pkg)
+      .expect(201, function () {
+        app.listen(0, done);
+      });
+    });
   });
+
   after(function (done) {
     app.close(done);
   });
@@ -32,10 +48,10 @@ describe('controllers/web/package.test.js', function () {
   afterEach(mm.restore);
 
   describe('GET /_list/search/search', function () {
-    it('should search with "c"', function (done) {
+    it('should search with "m"', function (done) {
       request(app)
-      .get('/_list/search/search?startkey="c"&limit=2')
+      .get('/_list/search/search?startkey="m"&limit=2')
-      .expect('content-type', 'application/json')
+      .expect('content-type', 'application/json; charset=utf-8')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('rows');
@@ -48,9 +64,9 @@ describe('controllers/web/package.test.js', function () {
       });
     });
 
-    it('should search with c', function (done) {
+    it('should search with m', function (done) {
       request(app)
-      .get('/_list/search/search?startkey=c&limit=2')
+      .get('/_list/search/search?startkey=m&limit=2')
       .expect(200, function (err, res) {
         should.not.exist(err);
         res.body.should.have.keys('rows');
@@ -77,16 +93,16 @@ describe('controllers/web/package.test.js', function () {
   describe('GET /package/:name', function (done) {
     it('should get 200', function (done) {
       request(app)
-      .get('/package/cutter')
+      .get('/package/mk2testmodule')
       .expect(200)
-      .expect('content-encoding', 'gzip')
+      // .expect('content-encoding', 'gzip')
       .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
       .expect(/<th>Version<\/th>/, function (err, res) {
         should.not.exist(err);
         res.should.have.header('etag');
-        res.text.should.include('<meta charset="utf-8">');
+        res.text.should.containEql('<meta charset="utf-8">');
         done();
       });
     });
@@ -101,7 +117,7 @@ describe('controllers/web/package.test.js', function () {
   describe('GET /package/:name/:version', function (done) {
     it('should 200 when get by version', function (done) {
       request(app)
-      .get('/package/cutter/0.0.2')
+      .get('/package/mk2testmodule/0.0.1')
       .expect(200)
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
@@ -110,21 +126,22 @@ describe('controllers/web/package.test.js', function () {
 
     it('should 200 when get by tag', function (done) {
       request(app)
-      .get('/package/cutter/latest')
+      .get('/package/mk2testmodule/latest')
       .expect(200)
       .expect(/<div id="package">/)
       .expect(/<th>Maintainers<\/th>/)
       .expect(/<th>Version<\/th>/, done);
     });
+
     it('should 404 when get by version not exist', function (done) {
       request(app)
-      .get('/package/cutter/1.1.2')
+      .get('/package/mk2testmodule/1.1.2')
       .expect(404, done);
     });
 
     it('should 404 when get by tag', function (done) {
       request(app)
-      .get('/package/cutter/notexisttag')
+      .get('/package/mk2testmodule/notexisttag')
       .expect(404, done);
     });
   });
@@ -132,14 +149,14 @@ describe('controllers/web/package.test.js', function () {
   describe('GET　/browse/keyword/:word', function () {
     it('should list by keyword ok', function (done) {
       request(app)
-      .get('/browse/keyword/cnpm')
+      .get('/browse/keyword/mk2testmodule')
       .expect(200)
       .expect(/Packages match/, done);
     });
 
     it('should list by keyword with json ok', function (done) {
       request(app)
-      .get('/browse/keyword/cnpm?type=json')
+      .get('/browse/keyword/mk2testmodule?type=json')
       .expect(200)
       .expect('content-type', 'application/json; charset=utf-8', done);
     });

test/controllers/web/user.test.js

@@ -29,7 +29,7 @@ describe('controllers/web/user.test.js', function () {
   describe('GET /~:name', function (done) {
     it('should get 200', function (done) {
       request(app)
-      .get('/~dead_horse')
+      .get('/~cnpmjstest10')
       .expect(200)
       .expect('content-type', 'text/html; charset=utf-8')
       .expect(/<div id="profile">/)
@@ -44,7 +44,7 @@ describe('controllers/web/user.test.js', function () {
 
     it('should get not eixst user but have modules 200', function (done) {
       request(app)
-      .get('/~tjholowaychuk')
+      .get('/~cnpmjstest101')
       .expect(200)
       .expect(/<div id="profile">/)
       .expect(/Packages by/, done);

test/fixtures/cnpmtest-package.json

@@ -0,0 +1 @@
+{"_id":"cnpmtest-package","_rev":"382002","name":"cnpmtest-package","description":"cnpmtest-package","dist-tags":{"latest":"0.0.0"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"time":{"modified":"2014-04-09T10:54:45.000Z","created":"2014-04-09T10:54:39.425Z","0.0.0":"2014-04-09T10:54:39.425Z"},"users":{},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"versions":{"0.0.0":{"name":"cnpmtest-package","version":"0.0.0","description":"cnpmtest-package","main":"index.js","scripts":{"test":"make test-all"},"config":{"cov":{"threshold":100}},"dependencies":{},"devDependencies":{"autod":"*","contributors":"*","should":"*","jshint":"*","cov":"*","istanbul-harmony":"*","mocha":"*"},"homepage":"https://github.com/{{group}}/cnpmtest-package","repository":{"type":"git","url":"git://github.com/{{group}}/cnpmtest-package.git","web":"https://github.com/{{group}}/cnpmtest-package"},"bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"keywords":["cnpmtest-package"],"engines":{"node":">= 0.10.0"},"author":{"name":"fengmk2","email":"fengmk2@gmail.com","url":"http://fengmk2.github.com"},"license":"MIT","contributors":[],"_id":"cnpmtest-package@0.0.0","dist":{"shasum":"81ea57c24a7f9f6e7263385116fc433f0ad9e179","size":1839,"noattachment":false,"tarball":"http://r.cnpmjs.org/cnpmtest-package/download/cnpmtest-package-0.0.0.tgz"},"_from":".","_npmVersion":"1.4.6","_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"directories":{},"publish_time":1397040879425,"_cnpm_publish_time":1397040879425}},"readme":"cnpmtest-package\n=======\n\n[![Build Status](https://secure.travis-ci.org/{{group}}/cnpmtest-package.png)](http://travis-ci.org/{{group}}/cnpmtest-package)\n[![Dependency Status](https://gemnasium.com/{{group}}/cnpmtest-package.png)](https://gemnasium.com/{{group}}/cnpmtest-package)\n\n[![NPM](https://nodei.co/npm/cnpmtest-package.png?downloads=true&stars=true)](https://nodei.co/npm/cnpmtest-package/)\n\n![logo](https://raw.github.com/{{group}}/cnpmtest-package/master/logo.png)\n\ncnpmtest-package desc\n\n## Install\n\n```bash\n$ npm install cnpmtest-package\n```\n\n## Usage\n\n```js\nvar cnpmtest-package = require('cnpmtest-package');\n\ncnpmtest-package.foo(function (err) {\n\n});\n```\n\n## License\n\n(The MIT License)\n\nCopyright (c) 2014 fengmk2 <fengmk2@gmail.com> and other contributors\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n'Software'), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n","_attachments":{},"homepage":"https://github.com/{{group}}/cnpmtest-package","bugs":{"url":"https://github.com/{{group}}/cnpmtest-package/issues","email":"fengmk2@gmail.com"},"license":"MIT"}

test/init.js

@@ -0,0 +1,41 @@
+/**!
+ * cnpmjs.org - test/init.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var crypto = require('crypto');
+var utility = require('utility');
+var User = require('../proxy/user');
+
+var usernames = [
+  'cnpmjstest101',
+  'cnpmjstest10'
+];
+
+usernames.forEach(function (name) {
+  var user = {
+    name: name,
+    email: 'fengmk2@gmail.com',
+    // password: 'cnpmjstest10',
+    ip: '127.0.0.1'
+  };
+  user.salt = crypto.randomBytes(30).toString('hex');
+  user.password_sha = utility.sha1(user.name + user.salt);
+
+  User.add(user, function (err, result) {
+    if (err) {
+      throw err;
+    }
+  });
+});

test/middleware/auth.test.js

@@ -34,7 +34,7 @@ describe('middleware/auth.test.js', function () {
   describe('auth()', function () {
     it('should pass if no authorization', function (done) {
       request(app)
-      .get('/-/user/org.couchdb.user:cnpmjstest1')
+      .get('/-/user/org.couchdb.user:cnpmjstest10')
       .expect(200, done);
     });
 

test/middleware/static.test.js

@@ -0,0 +1,60 @@
+/**!
+ * cnpmjs.org - test/middleware/static.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var request = require('supertest');
+var registry = require('../../servers/registry');
+var web = require('../../servers/registry');
+
+describe('middleware/static.test.js', function () {
+  before(function (done) {
+    registry = registry.listen(0, function () {
+      web = web.listen(0, done);
+    });
+  });
+
+  describe('registry', function () {
+    it('should /favicon.ico rewrite to /favicon.png', function (done) {
+      request(registry)
+      .get('/favicon.ico')
+      // .expect('content-type', 'image/png')
+      .expect(200, done);
+    });
+
+    it('should 200 /favicon.png', function (done) {
+      request(registry)
+      .get('/favicon.png')
+      .expect('content-type', 'image/png')
+      .expect(200, done);
+    });
+  });
+
+  describe('web', function () {
+    it('should /favicon.ico rewrite to /favicon.png', function (done) {
+      request(registry)
+      .get('/favicon.ico')
+      // .expect('content-type', 'image/png')
+      .expect(200, done);
+    });
+
+    it('should 200 /favicon.png', function (done) {
+      request(registry)
+      .get('/favicon.png')
+      .expect('content-type', 'image/png')
+      .expect(200, done);
+    });
+  });
+});

test/middleware/web_not_found.test.js

@@ -28,17 +28,17 @@ describe('middleware/web_not_found.test.js', function () {
   });
 
   describe('web_not_found()', function () {
-    it('should redirect /byte to /package/byte', function (done) {
+    it('should redirect /mk2testmodule to /package/mk2testmodule', function (done) {
       request(app)
-      .get('/byte')
+      .get('/mk2testmodule')
-      .expect('Location', '/package/byte')
+      .expect('Location', '/package/mk2testmodule')
       .expect(302, done);
     });
 
-    it('should redirect /byte/ to /package/byte', function (done) {
+    it('should redirect /mk2testmodule/ to /package/mk2testmodule', function (done) {
       request(app)
-      .get('/byte/')
+      .get('/mk2testmodule/')
-      .expect('Location', '/package/byte')
+      .expect('Location', '/package/mk2testmodule')
       .expect(302, done);
     });
 
@@ -48,9 +48,9 @@ describe('middleware/web_not_found.test.js', function () {
       .expect(404, done);
     });
 
-    it('should 200 /package/byte', function (done) {
+    it('should 200 /package/mk2testmodule', function (done) {
       request(app)
-      .get('/package/byte')
+      .get('/package/mk2testmodule')
       .expect(200, done);
     });
 

test/proxy/dist.test.js

@@ -0,0 +1,52 @@
+/**!
+ * cnpmjs.org - test/proxy/dist.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var Dist = require('../../proxy/dist');
+
+describe('proxy/dist.test.js', function () {
+  describe('savefile() and getfile', function () {
+    it('should save and get /npm-versions.txt', function* () {
+      var info = {
+        name: 'npm-versions.txt',
+        parent: '/',
+        date: '15-Sep-2011 23:48',
+        size: 1676,
+        url: 'http://cnpmjs.org/dist/npm-versions.txt',
+        sha1: '104731881047318810473188'
+      };
+      yield* Dist.savefile(info);
+      var got = yield* Dist.getfile('/npm-versions.txt');
+      should.exist(got);
+      got.should.eql(info);
+    });
+
+    it('should save and get /v1.0.0/npm-versions.txt', function* () {
+      var info = {
+        name: 'npm-versions.txt',
+        parent: '/v1.0.0/',
+        date: '15-Sep-2011 23:48',
+        size: 1676,
+        url: 'http://cnpmjs.org/dist/v1.0.0/npm-versions.txt',
+        sha1: '104731881047318810473188'
+      };
+      yield* Dist.savefile(info);
+      var got = yield* Dist.getfile('/v1.0.0/npm-versions.txt');
+      should.exist(got);
+      got.should.eql(info);
+    });
+  });
+});

test/proxy/module.test.js

@@ -38,8 +38,17 @@ describe('proxy/module.test.js', function () {
   });
 
   describe('search()', function () {
-    it('should search modules', function (done) {
+    before(function (done) {
-      Module.search('as', function (err, data) {
+      Module.addKeywords('aaaa', 'mock aaaaaa', ['aa', 'bb', 'cc'], function (err, results) {
+        should.not.exist(err);
+        results.should.be.an.Array;
+        results.should.length(3);
+        done();
+      });
+    });
+
+    it.skip('should search modules', function (done) {
+      Module.search('mock', function (err, data) {
         should.not.exist(err);
         data.should.have.keys('keywordMatchs', 'searchMatchs');
         data.searchMatchs.length.should.above(0);
@@ -74,13 +83,8 @@ describe('proxy/module.test.js', function () {
     });
   });
 
+  var mockName = 'aa' + Date.now();
   describe('addKeywords()', function () {
-    var mockName = 'aa' + Date.now();
-
-    after(function (done) {
-      mysql.query('DELETE FROM module_keyword WHERE name=?', [mockName], done);
-    });
-
     it('should add diff keywords to module', function (done) {
       Module.addKeywords(mockName, mockName, ['aa', 'bb', 'cc'], function (err, results) {
         should.not.exist(err);
@@ -91,10 +95,11 @@ describe('proxy/module.test.js', function () {
     });
 
     it('should add same keywords to module', function (done) {
-      Module.addKeywords('aa', 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
+      Module.addKeywords(mockName, 'desc aa', ['aa', 'bb', 'cc'], function (err, results) {
         should.not.exist(err);
         results.should.be.an.Array;
-        results.should.length(0);
+        results.should.length(3);
+        // results.should.length(0);
         done();
       });
     });
@@ -102,7 +107,7 @@ describe('proxy/module.test.js', function () {
 
   describe('getKeywords()', function () {
     it('should get aa module keywords', function (done) {
-      Module.getKeywords('aa', function (err, keywords) {
+      Module.getKeywords(mockName, function (err, keywords) {
         should.not.exist(err);
         keywords.should.eql(['aa', 'bb', 'cc']);
         done();
@@ -162,4 +167,10 @@ describe('proxy/module.test.js', function () {
       });
     });
   });
+
+  describe('removeTagsByNames()', function () {
+    it('should work', function* () {
+      yield* Module.removeTagsByNames('foo', ['latest', '1.0']);
+    });
+  });
 });

test/proxy/module_maintainer.test.js

@@ -0,0 +1,69 @@
+/**!
+ * cnpmjs.org - test/proxy/module_maintainer.test.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var ModuleMaintainer = require('../../proxy/module_maintainer');
+
+describe('proxy/module_maintainer.test.js', function () {
+  describe('update()', function () {
+    it('should update one maintainer', function* () {
+      var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2']);
+      rs.should.eql({
+        add: ['fengmk2'],
+        remove: []
+      });
+      // again should be fine
+      var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2']);
+      rs.should.eql({
+        add: ['fengmk2'],
+        remove: []
+      });
+      // remove the exists
+      var rs = yield* ModuleMaintainer.update('testfoo', ['fengmk2-1', 'foobar']);
+      rs.should.eql({
+        add: ['fengmk2-1', 'foobar'],
+        remove: ['fengmk2']
+      });
+    });
+
+    it('should update multi maintainers', function* () {
+      var rs = yield* ModuleMaintainer.update('testfoo2', ['fengmk2', 'ok', 'foobar']);
+      rs.should.eql({
+        add: ['fengmk2', 'ok', 'foobar'],
+        remove: []
+      });
+      // remove exists
+      var rs = yield* ModuleMaintainer.update('testfoo2', ['fengmk2']);
+      rs.should.eql({
+        add: ['fengmk2'],
+        remove: ['ok', 'foobar']
+      });
+      var rs = yield* ModuleMaintainer.update('testfoo3', ['fengmk2', 'ok', 'foobar']);
+      rs.should.eql({
+        add: ['fengmk2', 'ok', 'foobar'],
+        remove: []
+      });
+    });
+
+    it('should add empty maintainers do nothing', function* () {
+      var rs = yield* ModuleMaintainer.update('tesfoobar', []);
+      rs.should.eql({
+        add: [],
+        remove: []
+      });
+    });
+  });
+});

test/proxy/sync_module_worker.test.js

@@ -22,14 +22,14 @@ var Log = require('../../proxy/module_log');
 describe('proxy/sync_module_worker.test.js', function () {
   it('should start a sync worker', function (done) {
     Log.create({
-      name: 'cnpmjs.org',
+      name: 'mk2testmodule',
       username: 'fengmk2',
     }, function (err, result) {
       should.not.exist(err);
       result.id.should.above(0);
       var worker = new SyncModuleWorker({
         logId: result.id,
-        name: 'cnpmjs.org',
+        name: 'mk2testmodule',
         username: 'fengmk2'
       });
 
@@ -40,7 +40,7 @@ describe('proxy/sync_module_worker.test.js', function () {
 
   it('should start a sync worker with names and noDep', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['cnpmjs.org', 'cutter'],
+      name: ['mk2testmodule'],
       noDep: true,
       username: 'fengmk2'
     });
@@ -49,18 +49,45 @@ describe('proxy/sync_module_worker.test.js', function () {
     worker.on('end', function () {
       var names = worker.successes.concat(worker.fails);
       names.sort();
-      names.should.eql(['cnpmjs.org', 'cutter']);
+      names.should.eql(['mk2testmodule']);
       done();
     });
   });
 
   it('should start a sync worker with names', function (done) {
     var worker = new SyncModuleWorker({
-      name: ['cnpmjs.org', 'cutter'],
+      name: ['mk2testmodule'],
       username: 'fengmk2'
     });
 
     worker.start();
     worker.on('end', done);
   });
+
+  it('should sync unpublished module by name', function* () {
+    var result = yield* SyncModuleWorker.sync('tnpm', 'fengmk2');
+    result.ok.should.equal(true);
+    result.should.have.property('logId');
+  });
+
+  it('should not sync not exists module', function* () {
+    var result = yield* SyncModuleWorker.sync('tnpm-not-exists', 'fengmk2');
+    result.ok.should.equal(false);
+    result.should.not.have.property('logId');
+  });
+
+  it('should sync unpublished info', function (done) {
+    var worker = new SyncModuleWorker({
+      name: ['tnpm'],
+      username: 'fengmk2'
+    });
+
+    worker.start();
+    worker.on('end', function () {
+      var names = worker.successes.concat(worker.fails);
+      names.sort();
+      names.should.eql(['tnpm']);
+      done();
+    });
+  });
 });

test/proxy/user.test.js

@@ -127,11 +127,12 @@ describe('proxy/user.test.js', function () {
 
   describe('update()', function () {
     before(initUser);
+
     it('should update ok', function (done) {
       user.update(mockUser, function (err, data) {
         should.not.exist(err);
         should.exist(data);
-        data.should.have.keys(['rev']);
+        data.should.have.keys('rev', 'result');
         done();
       });
     });
@@ -140,7 +141,8 @@ describe('proxy/user.test.js', function () {
       mockUser.rev = '1-error';
       user.update(mockUser, function (err, data) {
         should.not.exist(err);
-        should.not.exist(data);
+        should.exist(data);
+        data.result.affectedRows.should.equal(0);
         done();
       });
     });
@@ -176,7 +178,7 @@ describe('proxy/user.test.js', function () {
       yield user.saveNpmUser(existUser);
       var r = yield mysql.queryOne('select rev, json, npm_user from user where name=?', existUser.name);
       should.exist(r);
-      r.npm_user.should.equal(0);
+      // r.npm_user.should.equal(0);
       r.rev.should.equal(existUser._rev);
       JSON.parse(r.json).should.eql(existUser);
     });

test/sync.js

@@ -14,12 +14,14 @@
  * Module dependencies.
  */
 
+var debug = require('debug');
+debug.enable('cnpmjs.org*');
 var SyncModuleWorker = require('../proxy/sync_module_worker');
 var mysql = require('../common/mysql');
 var Log = require('../proxy/module_log');
 var config = require('../config');
 
-config.sourceNpmRegistry = 'http://r.cnpmjs.org';
+config.sourceNpmRegistry = 'http://registry.npmjs.org';
 
 var names = process.argv[2] || 'byte';
 names = names.split(',');

test/sync/sync_all.test.js

@@ -1,5 +1,5 @@
 /*!
- * cnpmjs.org - test/sync/sync_all.js
+ * cnpmjs.org - test/sync/sync_all.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -20,31 +20,32 @@ var Total = require('../../proxy/total');
 var should = require('should');
 var Module = require('../../proxy/module');
 
-describe('sync/sync_all.js', function () {
+describe('sync/sync_all.test.js', function () {
   describe('sync()', function () {
     afterEach(mm.restore);
 
     it('should sync first time ok', function *() {
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+      mm.data(Npm, 'getShort', ['mk2testmodule', 'mk2testmodule-not-exists']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: 0});
       var data = yield sync;
-      data.successes.should.eql(['cnpmjs.org', 'cutter']);
+      data.successes.should.eql(['mk2testmodule', 'mk2testmodule-not-exists']);
       mm.restore();
       var result = yield Total.getTotalInfo();
-      result.last_sync_module.should.equal('cutter');
+      should.exist(result);
+      result.last_sync_module.should.equal('mk2testmodule-not-exists');
     });
 
     it('should sync common ok', function *() {
       mm.data(Npm, 'getAllSince', {
         _updated: Date.now(),
-        'cnpmjs.org': {},
+        'mk2testmodule': {},
-        cutter: {}
+        // cutter: {}
       });
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter', 'cnpm']);
+      mm.data(Npm, 'getShort', ['mk2testmodule']);
       mm.data(Total, 'getTotalInfo', {last_sync_time: Date.now()});
-      mm.data(Module, 'listAllModuleNames', [{name: 'cnpmjs.org'}, {name: 'cutter'}]);
+      mm.data(Module, 'listAllModuleNames', [{name: 'mk2testmodule'}]);
       var data = yield sync;
-      data.successes.should.eql(['cnpmjs.org', 'cutter']);
+      data.successes.should.eql(['mk2testmodule']);
       mm.restore();
     });
   });

test/sync/sync_dist.test.js

@@ -0,0 +1,129 @@
+/**!
+ * cnpmjs.org - test/sync/sync_dist.test.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var should = require('should');
+var mm = require('mm');
+var urllib = require('co-urllib');
+var Dist = require('../../proxy/dist');
+var distsync = require('../../sync/sync_dist');
+
+describe('sync/sync_dist.test.js', function () {
+  afterEach(mm.restore);
+
+  describe('listPhantomjsDir()', function () {
+    it('should list all phantomjs download infos', function* () {
+      var items = yield* distsync.listPhantomjsDir('/phantomjs');
+      items.length.should.above(1);
+      items.forEach(function (item) {
+        item.should.have.keys('name', 'date', 'size', 'type', 'parent', 'downloadURL');
+      });
+    });
+  });
+
+  describe('listdiff()', function () {
+    it('should got all news', function* () {
+      mm(urllib, 'request', function* () {
+        return {
+          status: 200,
+          data: '<a href="npm/">npm/</a>   06-May-2014 01:18     -\n<a href="npm-versions.txt">npm-versions.txt</a>  27-Feb-2014 00:01         1676',
+          headers: {},
+        };
+      });
+
+      mm(Dist, 'listdir', function* () {
+        return [];
+      });
+
+      var items = yield* distsync.listdiff('/');
+      items.should.eql([
+        { name: 'npm/',
+          date: '06-May-2014 01:18',
+          size: '-',
+          type: 'dir',
+          parent: '/' },
+        { name: 'npm-versions.txt',
+          date: '27-Feb-2014 00:01',
+          size: 1676,
+          type: 'file',
+          parent: '/' }
+      ]);
+    });
+
+    it('should got empty when all exists', function* () {
+      mm(urllib, 'request', function* () {
+        return {
+          status: 200,
+          data: '<a href="npm/">npm/</a>   06-May-2014 01:18     -\n<a href="npm-versions.txt">npm-versions.txt</a>  27-Feb-2014 00:01         1676',
+          headers: {},
+        };
+      });
+
+      mm(Dist, 'listdir', function* () {
+        return [
+          {
+            name: 'npm/',
+            date: '06-May-2014 01:18',
+            parent: '/'
+          },
+          {
+            name: 'npm-versions.txt',
+            date: '27-Feb-2014 00:01',
+            size: 1676,
+            parent: '/'
+          },
+        ];
+      });
+
+      var items = yield* distsync.listdiff('/');
+      items.should.length(0);
+    });
+
+    it('should got date change dir', function* () {
+      mm(urllib, 'request', function* () {
+        return {
+          status: 200,
+          data: '<a href="npm/">npm/</a>   06-May-2014 01:18     -\n<a href="npm-versions.txt">npm-versions.txt</a>  27-Feb-2014 00:01         1676',
+          headers: {},
+        };
+      });
+
+      mm(Dist, 'listdir', function* () {
+        return [
+          {
+            name: 'npm/',
+            date: '06-May-2014 01:17',
+            parent: '/'
+          },
+          {
+            name: 'npm-versions.txt',
+            date: '27-Feb-2014 00:01',
+            size: 1676,
+            parent: '/'
+          },
+        ];
+      });
+
+      var items = yield* distsync.listdiff('/');
+      items.should.eql([
+        { name: 'npm/',
+          date: '06-May-2014 01:18',
+          size: '-',
+          type: 'dir',
+          parent: '/' }
+      ]);
+    });
+  });
+});

test/sync/sync_exist.test.js

@@ -1,5 +1,5 @@
 /*!
- * cnpmjs.org - test/sync/sync_exist.js
+ * cnpmjs.org - test/sync/sync_exist.test.js
  *
  * Copyright(c) cnpmjs.org and other contributors.
  * MIT Licensed
@@ -13,32 +13,32 @@
 /**
  * Module dependencies.
  */
-var sync = require('../../sync/sync_exist');
+
+var should = require('should');
 var mm = require('mm');
+var sync = require('../../sync/sync_exist');
 var Npm = require('../../proxy/npm');
 var Total = require('../../proxy/total');
-var should = require('should');
 
-describe('sync/sync_exist.js', function () {
+describe('sync/sync_exist.test.js', function () {
   describe('sync()', function () {
     afterEach(mm.restore);
 
     it('should sync first time ok', function *() {
-      mm.data(Npm, 'getShort', ['cnpmjs.org', 'cutter']);
+      mm.data(Npm, 'getShort', ['mk2testmodule']);
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: 0});
       var data = yield sync();
-      data.successes.should.eql(['cnpmjs.org', 'cutter']);
+      data.successes.should.eql(['mk2testmodule']);
     });
 
     it('should sync common ok', function *() {
       mm.data(Npm, 'getAllSince', {
         _updated: Date.now(),
-        'cnpmjs.org': {},
+        'mk2testmodule': {},
-        cutter: {}
       });
       mm.data(Total, 'getTotalInfo', {last_exist_sync_time: Date.now()});
       var data = yield sync();
-      data.successes.should.eql(['cnpmjs.org', 'cutter']);
+      data.successes.should.eql(['mk2testmodule']);
     });
   });
 });

test/sync_dist.js

@@ -0,0 +1,25 @@
+/**!
+ * cnpmjs.org - test/sync_dist.js
+ *
+ * Copyright(c) fengmk2 and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var debug = require('debug');
+debug.enable('cnpmjs.org*');
+var co = require('co');
+var syncdist = require('../sync/sync_dist');
+
+co(function* () {
+  // yield* syncdist('/v0.10.28/');
+  yield* syncdist.syncPhantomjsDir();
+})();

test/zlib_write_after_close.js

@@ -0,0 +1,9 @@
+var zlib = require('zlib');
+
+var stream = zlib.createGzip();
+
+stream.write('foo');
+stream.close();
+
+// Assertion failed: (!ctx->pending_close_ && "close is pending"), function Write, file ../src/node_zlib.cc, line 150.
+stream.write('again');

view/web/404.html

@@ -0,0 +1,5 @@
+<div class="alert alert-warning">
+  Can not found package match <%= name %>. You can
+  <a href="/sync/<%= name %>" target="_blank">SYNC</a> from official npm registry or
+  <a href="https://npmjs.org/search?q=<%= name %>" target="_blank">SEARCH</a> in official npm website.
+</div>

view/web/dist.html

@@ -0,0 +1,9 @@
+<h1>Mirror index of <a target="_blank" href="<%= disturl %><%= dirname %>"><%= disturl %><%= dirname %></a></h1>
+<hr>
+<pre><a href="../">../</a><% for (var i = 0; i < items.length; i++) {
+  var item = items[i];
+  var sizestr = '' + (item.size || '-');
+%>
+<a href="<%= item.name %>"><%= item.name %></a><%- padding(50, item.name.length, " ") %><%= item.date %><%- padding(20, sizestr.length, " ") %><%= sizestr %><% } %>
+</pre>
+<hr>

view/web/layout.html

@@ -4,12 +4,12 @@
     <meta charset="utf-8">
     <title><%- locals.title %></title>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link rel="shortcut icon" href="/public/favicon.png">
+    <link rel="shortcut icon" href="/favicon.png">
     <!-- Bootstrap -->
-    <link href="http://cdn.staticfile.org/twitter-bootstrap/3.0.0-rc2/css/bootstrap.min.css" rel="stylesheet" media="screen">
+    <link href="//dn-staticfile.qbox.me/twitter-bootstrap/3.0.0-rc2/css/bootstrap.min.css" rel="stylesheet" media="screen">
-    <link href="http://cdn.staticfile.org/prettify/r298/prettify.min.css" rel="stylesheet" media="screen">
+    <link href="//dn-staticfile.qbox.me/prettify/r298/prettify.min.css" rel="stylesheet" media="screen">
     <!-- JavaScript plugins (requires jQuery) -->
-    <script src="http://cdn.staticfile.org/jquery/2.0.3/jquery.min.js"></script>
+    <script src="//dn-staticfile.qbox.me/jquery/2.0.3/jquery.min.js"></script>
     <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="CNPM" />
     <style>
       a{color:#09f;}
@@ -72,17 +72,17 @@
         {{footer}}
       </p>
       <a href="https://github.com/cnpm/cnpmjs.org" id="fork" target="_blank">
-        <img alt="Fork me on GitHub" src="http://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
+        <img alt="Fork me on GitHub" src="//s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png">
     </a>
     </div>
     </div>
     <!-- Include all compiled plugins (below), or include individual files as needed -->
-    <script src="http://cdn.staticfile.org/twitter-bootstrap/3.0.0-rc2/js/bootstrap.min.js"></script>
+    <script src="//dn-staticfile.qbox.me/twitter-bootstrap/3.0.0-rc2/js/bootstrap.min.js"></script>
 
     <!-- Enable responsive features in IE8 with Respond.js (https://github.com/scottjehl/Respond) -->
-    <script src="http://cdn.staticfile.org/respond.js/1.2.0/respond.min.js"></script>
+    <script src="//dn-staticfile.qbox.me/respond.js/1.2.0/respond.min.js"></script>
 
-    <script src="http://cdn.staticfile.org/prettify/r298/prettify.min.js"></script>
+    <script src="//dn-staticfile.qbox.me/prettify/r298/prettify.min.js"></script>
 
     <!-- Specific to this page -->
     <script>

view/web/package.html

@@ -252,6 +252,7 @@
             <a class="downloadlink" target="_blank" href="<%= package.dist.tarball %>">
               <%= package.dist.tarball %>
             </a>
+            (<%- package.dist.size %>)
           </td>
         </tr>
         <% } %>

worker.js

@@ -21,12 +21,14 @@ var config = require('./config');
 var registry = require('./servers/registry');
 var web = require('./servers/web');
 
-registry.listen(config.registryPort);
+registry.listen(config.registryPort, config.bindingHost);
-web.listen(config.webPort);
+web.listen(config.webPort, config.bindingHost);
 
-console.log('[%s] [worker:%d] Server started, registry server listen at %d, web listen at %d, cluster: %s',
+console.log('[%s] [worker:%d] Server started, registry server listen at %s:%d, web listen at %s:%d, cluster: %s',
   new Date(), process.pid,
-  config.registryPort, config.webPort, config.enableCluster);
+  config.bindingHost, config.registryPort,
+  config.bindingHost, config.webPort,
+  config.enableCluster);
 
 graceful({
   server: [registry, web],